diff --git a/.Rbuildignore b/.Rbuildignore
index 3b6f456cb..4b3815dd8 100644
--- a/.Rbuildignore
+++ b/.Rbuildignore
@@ -2,16 +2,14 @@
 ^renv\.lock$
 ^.*\.Rproj$
 ^\.Rproj\.user$
-.travis.yml
 standalone
-^\.travis\.yml$
 deploy.sh
 extras/*
 man-roxygen
-xgboost
 compare_versions
 .github
 docs/*
 _pkgdown.yml
-
-
+^vignettes/articles$
+^doc$
+^Meta$
diff --git a/.github/workflows/R_CMD_check_Hades.yaml b/.github/workflows/R_CMD_check_Hades.yaml
index f85ed6abb..a0baf21b2 100644
--- a/.github/workflows/R_CMD_check_Hades.yaml
+++ b/.github/workflows/R_CMD_check_Hades.yaml
@@ -22,7 +22,7 @@ jobs:
         config:
           - {os: windows-latest, r: 'release'}  # Does not appear to have Java 32-bit, hence the --no-multiarch
           - {os: macOS-latest, r: 'release'}
-          - {os: ubuntu-20.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"}
+          - {os: ubuntu-22.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/jammy/latest"}
 
     env:
       GITHUB_PAT: ${{ secrets.GH_TOKEN }}
@@ -45,98 +45,56 @@ jobs:
       CDM5_SQL_SERVER_USER: ${{ secrets.CDM5_SQL_SERVER_USER }}
 
     steps:
-      - uses: actions/checkout@v2
-    
-      - uses: conda-incubator/setup-miniconda@v2
-        with:
-          python-version: "3.7"
-          activate-environment: r-reticulate
-      
-      - name: python main dependencies  
-        run: conda install --name r-reticulate numpy scipy scikit-learn pandas pydotplus joblib
-      
-      - name: python json dependencies  
-        run: conda install --name r-reticulate -c conda-forge sklearn-json 
-      
-      - name: python scikit-survival dependencies    
-        run: conda install --name r-reticulate -c sebp scikit-survival
+      - uses: actions/checkout@v4
 
-      - uses: r-lib/actions/setup-r@v1
+      - uses: r-lib/actions/setup-r@v2
         with:
           r-version: ${{ matrix.config.r }}
 
-      - uses: r-lib/actions/setup-tinytex@v1
+      - uses: r-lib/actions/setup-tinytex@v2
 
-      - uses: r-lib/actions/setup-pandoc@v1
+      - uses: r-lib/actions/setup-pandoc@v2
 
-      - name: Query dependencies
-        run: |
-          install.packages('remotes')
-          saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
-          writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
-        shell: Rscript {0}
-
-      - name: Cache R packages
-        if: runner.os != 'Windows'
-        uses: actions/cache@v2
-        with:
-          path: ${{ env.R_LIBS_USER }}
-          key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-3-${{ hashFiles('.github/depends.Rds') }}
-          restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-3-
-  
-      - name: Install system dependencies
+      - name: Install system requirements
         if: runner.os == 'Linux'
         run: |
+          sudo apt-get install -y libssh-dev
+          Rscript -e 'install.packages("remotes")'
           while read -r cmd
           do
             eval sudo $cmd
-          done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "20.04"))')
-
-      - name: Install dependencies
-        run: |
-          remotes::install_deps(dependencies = TRUE, INSTALL_opts=c("--no-multiarch"))
-          remotes::install_cran("rcmdcheck")
-        shell: Rscript {0}
-
-      - name: Install covr
-        if: runner.os == 'Windows'
-        run: |
-          remotes::install_cran("covr")
-        shell: Rscript {0}
-
-      - name: Remove check folder if exists
-        if: runner.os == 'macOS'
-        run: unlink("check", recursive = TRUE)
-        shell: Rscript {0}
-
-      - name: use r-reticulate environment
-        run: |
-          reticulate::use_condaenv("r-reticulate", required = TRUE)
-        shell: Rscript {0}
-
-      - name: Check
-        env:
-          _R_CHECK_CRAN_INCOMING_REMOTE_: false
-        run: rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran", "--no-multiarch"), error_on = "warning", check_dir = "check")
-        shell: Rscript {0}
+          done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "22.04"))')
+          
+      - name: Setup conda
+        uses: conda-incubator/setup-miniconda@v3
+        
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          extra-packages: any::rcmdcheck
+          needs: check
 
-      - name: Upload check results
-        if: failure()
-        uses: actions/upload-artifact@main
+      - uses: r-lib/actions/check-r-package@v2
         with:
-          name: ${{ runner.os }}-r${{ matrix.config.r }}-results
-          path: check
+          args: 'c("--no-manual", "--as-cran")'
+          error-on: '"warning"'
+          check-dir: '"check"'
 
       - name: Upload source package
         if: success() && runner.os == 'macOS' && github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: package_tarball
           path: check/*.tar.gz
-
+          
+      - name: Install covr
+        if: runner.os == 'Linux'
+        run: |
+          remotes::install_cran("covr")
+        shell: Rscript {0}
+        
       - name: Test coverage
-        if: runner.os == 'Windows'
-        run: covr::codecov()
+        if: runner.os == 'Linux'
+        run: covr::codecov(token = "${{ secrets.CODECOV_TOKEN }}")
         shell: Rscript {0}
 
 
@@ -152,7 +110,7 @@ jobs:
 
     steps:
       
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0    
 
@@ -178,7 +136,7 @@ jobs:
           draft: false
           prerelease: false
           
-      - uses: r-lib/actions/setup-r@v1
+      - uses: r-lib/actions/setup-r@v2
         if: ${{ env.new_version != '' }}
         
       - name: Install drat
@@ -194,7 +152,7 @@ jobs:
           
       - name: Download package tarball
         if: ${{ env.new_version != '' }}
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: package_tarball    
           
diff --git a/.github/workflows/R_CMD_check_main_weekly.yaml b/.github/workflows/R_CMD_check_main_weekly.yaml
new file mode 100644
index 000000000..45bbb548b
--- /dev/null
+++ b/.github/workflows/R_CMD_check_main_weekly.yaml
@@ -0,0 +1,72 @@
+on:
+  schedule:
+    - cron: '0 12 * * 6' # every Saturday at noon UTC
+  
+  workflow_dispatch:
+    
+name: 'R check'
+
+jobs:
+  R-CMD-check-main:
+    runs-on: ${{ matrix.config.os }}
+
+    name: ${{ matrix.config.os }} (${{ matrix.config.r }})
+
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - {os: macOS-latest, r: 'release'}
+
+    env:
+      GITHUB_PAT: ${{ secrets.GH_TOKEN }}
+      R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
+      RSPM: ${{ matrix.config.rspm }}
+      CDM5_ORACLE_CDM_SCHEMA: ${{ secrets.CDM5_ORACLE_CDM_SCHEMA }}
+      CDM5_ORACLE_OHDSI_SCHEMA: ${{ secrets.CDM5_ORACLE_OHDSI_SCHEMA }}
+      CDM5_ORACLE_PASSWORD: ${{ secrets.CDM5_ORACLE_PASSWORD }}
+      CDM5_ORACLE_SERVER: ${{ secrets.CDM5_ORACLE_SERVER }}
+      CDM5_ORACLE_USER: ${{ secrets.CDM5_ORACLE_USER }}
+      CDM5_POSTGRESQL_CDM_SCHEMA: ${{ secrets.CDM5_POSTGRESQL_CDM_SCHEMA }}
+      CDM5_POSTGRESQL_OHDSI_SCHEMA: ${{ secrets.CDM5_POSTGRESQL_OHDSI_SCHEMA }}
+      CDM5_POSTGRESQL_PASSWORD: ${{ secrets.CDM5_POSTGRESQL_PASSWORD }}
+      CDM5_POSTGRESQL_SERVER: ${{ secrets.CDM5_POSTGRESQL_SERVER }}
+      CDM5_POSTGRESQL_USER: ${{ secrets.CDM5_POSTGRESQL_USER }}
+      CDM5_SQL_SERVER_CDM_SCHEMA: ${{ secrets.CDM5_SQL_SERVER_CDM_SCHEMA }}
+      CDM5_SQL_SERVER_OHDSI_SCHEMA: ${{ secrets.CDM5_SQL_SERVER_OHDSI_SCHEMA }}
+      CDM5_SQL_SERVER_PASSWORD: ${{ secrets.CDM5_SQL_SERVER_PASSWORD }}
+      CDM5_SQL_SERVER_SERVER: ${{ secrets.CDM5_SQL_SERVER_SERVER }}
+      CDM5_SQL_SERVER_USER: ${{ secrets.CDM5_SQL_SERVER_USER }}
+      CDM5_REDSHIFT_CDM_SCHEMA: ${{ secrets.CDM5_REDSHIFT_CDM_SCHEMA }}
+      CDM5_REDSHIFT_OHDSI_SCHEMA: ${{ secrets.CDM5_REDSHIFT_OHDSI_SCHEMA }}
+      CDM5_REDSHIFT_PASSWORD: ${{ secrets.CDM5_REDSHIFT_PASSWORD }}
+      CDM5_REDSHIFT_SERVER: ${{ secrets.CDM5_REDSHIFT_SERVER }}
+      CDM5_REDSHIFT_USER: ${{ secrets.CDM5_REDSHIFT_USER }}
+      CDM5_SPARK_USER: ${{ secrets.CDM5_SPARK_USER }}
+      CDM5_SPARK_PASSWORD: ${{ secrets.CDM5_SPARK_PASSWORD }}
+      CDM5_SPARK_CONNECTION_STRING: ${{ secrets.CDM5_SPARK_CONNECTION_STRING }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          r-version: ${{ matrix.config.r }}
+
+      - uses: r-lib/actions/setup-tinytex@v2
+
+      - uses: r-lib/actions/setup-pandoc@v2
+
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          extra-packages: any::rcmdcheck
+          needs: check
+          
+      - name: Setup conda
+        uses: conda-incubator/setup-miniconda@v3
+
+      - uses: r-lib/actions/check-r-package@v2
+        with:
+          args: 'c("--no-manual", "--as-cran")'
+          error-on: '"warning"'
+          check-dir: '"check"'
diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml
new file mode 100644
index 000000000..7a5e8ac76
--- /dev/null
+++ b/.github/workflows/pkgdown.yaml
@@ -0,0 +1,46 @@
+# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
+# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
+on:
+  push:
+    branches: [main, develop]
+  release:
+    types: [published]
+  workflow_dispatch:
+
+name: pkgdown
+
+jobs:
+  pkgdown:
+    runs-on: ubuntu-latest
+    # Only restrict concurrency for non-PR jobs
+    concurrency:
+      group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }}
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: r-lib/actions/setup-pandoc@v2
+
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          use-public-rspm: true
+
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          extra-packages: any::pkgdown, ohdsi/OhdsiRTools
+          needs: website
+
+      - name: Build site
+        run: Rscript -e 'pkgdown::build_site_github_pages(new_process = FALSE, install = TRUE)'
+        
+      - name: Fix Hades Logo
+        run: Rscript -e 'OhdsiRTools::fixHadesLogo()'
+
+      - name: Deploy to GitHub pages 🚀
+        if: github.event_name != 'pull_request'
+        uses: JamesIves/github-pages-deploy-action@4.1.4
+        with:
+          clean: false
+          branch: gh-pages
+          folder: docs
diff --git a/.gitignore b/.gitignore
index c7ce48620..cb6091781 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,8 @@
 *-Ex.R
 # R data files from past sessions
 .Rdata
+# R environ
+.Renviron
 # RStudio files
 .Rproj.user/
 .Rproj.user
@@ -20,4 +22,6 @@ standalone/build/*
 /plpmodels/*
 /python_models/*
 /mycache/*
-/inst/shiny/DiagnosticsExplorer/rsconnect/*
\ No newline at end of file
+/inst/shiny/DiagnosticsExplorer/rsconnect/*
+/doc/
+/Meta/
diff --git a/DESCRIPTION b/DESCRIPTION
index 75919f642..0011aef4b 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -2,17 +2,18 @@ Package: PatientLevelPrediction
 Type: Package
 Title: Developing patient level prediction using data in the OMOP Common Data
     Model
-Version: 5.0.5
-Date: 2022-03-16
+Version: 6.3.9
+Date: 2024-08-21
 Authors@R: c(
     person("Jenna", "Reps", email = "jreps@its.jnj.com", role = c("aut", "cre")),
     person("Martijn", "Schuemie", role = c("aut")),
     person("Marc", "Suchard", role = c("aut")),
     person("Patrick", "Ryan", role = c("aut")),
-    person("Peter", "Rijnbeek", role = c("aut"))
+    person("Peter", "Rijnbeek", role = c("aut")),
+    person("Egill", "Fridgeirsson", role = c("aut"))
     )
 Maintainer: Jenna Reps <jreps@its.jnj.com>
-Description: A package for creating patient level prediction models. Given a
+Description: A user friendly way to create patient level prediction models using the OMOP common data model. Given a
     cohort of interest and an outcome of interest, the package can use data in the
     OMOP Common Data Model to build a large set of features. These features can then
     be assessed to fit a predictive model using a number of machine learning algorithms.
@@ -22,69 +23,56 @@ URL: https://ohdsi.github.io/PatientLevelPrediction, https://github.com/OHDSI/Pa
 BugReports: https://github.com/OHDSI/PatientLevelPrediction/issues
 VignetteBuilder: knitr
 Depends:
-    R (>= 3.3.0),
-    FeatureExtraction (>= 3.0.0)
+    R (>= 4.0.0)
 Imports:
     Andromeda,
     Cyclops (>= 3.0.0),
-    DatabaseConnector (>= 5.0.0),
+    DatabaseConnector (>= 6.0.0),
     dplyr,
+    FeatureExtraction (>= 3.0.0),
     ggplot2,
     gridExtra,
-    Hmisc,
-    jsonlite,
     Matrix,
     memuse,
+    mgcv,
     ParallelLogger (>= 2.0.0),
+    polspline,
     pROC,
     PRROC,
-    reticulate (> 1.16),
+    reticulate (>= 1.30),
     rlang,
-    rms,
-    RSQLite,
     SqlRender (>= 1.1.3),
     survival,
-    tibble,
     tidyr,
     utils
 Suggests:
     AUC,
     BigKnn (>= 1.0.0),
-    DBI,
     devtools,
-    diagram, 
-    DT,
     Eunomia,
-    gnm,
-    htmlwidgets (> 0.8),
-    htmltools,
     IterativeHardThresholding,
     knitr,
     markdown,
     Metrics,
     parallel,
-    plotly,
     plyr, 
     pool,
-    R6,
+    readr,
     ResourceSelection,
+    ResultModelManager (>= 0.2.0),
     rmarkdown,
+    RSQLite,
     scoring,
-    shiny,
-    shinycssloaders,
-    shinydashboard,
-    shinyWidgets,
-    SparseM,
-    survAUC,
+    ShinyAppBuilder (>= 1.1.1),
     survminer,
     testthat,
     withr,
-    xgboost (> 1.3.2.1)
+    xgboost (> 1.3.2.1),
+    lightgbm
 Remotes:
     ohdsi/BigKnn,
-    ohdsi/Eunomia,
     ohdsi/FeatureExtraction,
-    ohdsi/IterativeHardThresholding,
-    ohdsi/ParallelLogger
-RoxygenNote: 7.1.2
+    ohdsi/ShinyAppBuilder,
+    ohdsi/ResultModelManager,
+RoxygenNote: 7.3.1
 Encoding: UTF-8
diff --git a/NAMESPACE b/NAMESPACE
index 3aed99048..38cfb743f 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -3,14 +3,22 @@
 S3method(print,plpData)
 S3method(print,summary.plpData)
 S3method(summary,plpData)
+export(MapIds)
+export(addDiagnosePlpToDatabase)
+export(addMultipleDiagnosePlpToDatabase)
+export(addMultipleRunPlpToDatabase)
+export(addRunPlpToDatabase)
 export(averagePrecision)
 export(brierScore)
 export(calibrationLine)
 export(computeAuc)
+export(computeGridPerformance)
 export(configurePython)
 export(covariateSummary)
 export(createCohortCovariateSettings)
 export(createDatabaseDetails)
+export(createDatabaseList)
+export(createDatabaseSchemaSettings)
 export(createDefaultExecuteSettings)
 export(createDefaultSplitSetting)
 export(createExecuteSettings)
@@ -23,13 +31,19 @@ export(createPreprocessSettings)
 export(createRandomForestFeatureSelection)
 export(createRestrictPlpDataSettings)
 export(createSampleSettings)
+export(createSplineSettings)
+export(createStratifiedImputationSettings)
 export(createStudyPopulation)
 export(createStudyPopulationSettings)
+export(createTempModelLoc)
 export(createUnivariateFeatureSelection)
+export(createValidationDesign)
 export(createValidationSettings)
-export(diagnostic)
+export(diagnoseMultiplePlp)
+export(diagnosePlp)
 export(evaluatePlp)
 export(externalValidateDbPlp)
+export(extractDatabaseToCsv)
 export(fitPlp)
 export(getCalibrationSummary)
 export(getCohortCovariateData)
@@ -38,14 +52,18 @@ export(getPlpData)
 export(getPredictionDistribution)
 export(getThresholdSummary)
 export(ici)
-export(launchDiagnosticsExplorer)
+export(insertCsvToDatabase)
+export(insertModelDesignInDatabase)
+export(insertResultsToSqlite)
 export(listAppend)
+export(listCartesian)
 export(loadPlpAnalysesJson)
 export(loadPlpData)
 export(loadPlpModel)
 export(loadPlpResult)
 export(loadPlpShareable)
 export(loadPrediction)
+export(migrateDataModel)
 export(modelBasedConcordance)
 export(outcomeSurvivalPlot)
 export(pfi)
@@ -63,7 +81,6 @@ export(plotSparseCalibration)
 export(plotSparseCalibration2)
 export(plotSparseRoc)
 export(plotVariableScatterplot)
-export(populatePlpResultTables)
 export(predictCyclops)
 export(predictPlp)
 export(recalibratePlp)
@@ -83,18 +100,21 @@ export(setGradientBoostingMachine)
 export(setIterativeHardThresholding)
 export(setKNN)
 export(setLassoLogisticRegression)
+export(setLightGBM)
 export(setMLP)
 export(setNaiveBayes)
 export(setPythonEnvironment)
 export(setRandomForest)
 export(setSVM)
 export(simulatePlpData)
+export(sklearnFromJson)
+export(sklearnToJson)
 export(splitData)
 export(toSparseM)
+export(validateExternal)
 export(validateMultiplePlp)
 export(viewDatabaseResultPlp)
 export(viewMultiplePlp)
 export(viewPlp)
-import(FeatureExtraction)
 importFrom(dplyr,"%>%")
 importFrom(rlang,.data)
diff --git a/NEWS.md b/NEWS.md
index f0eedf6cc..4de122e95 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,3 +1,196 @@
+PatientLevelPrediction 6.3.9
+======================
+- Hotfix adding schema to DatabaseConnector::getTableNames when creating results tables
+
+PatientLevelPrediction 6.3.8
+======================
+- Add support for R4.4 
+- Fix notes around documentation (vignette engine and brackets in itemize)
+- Use webp image format where possible (not in pdfs) for smaller size
+- Make sure random table names are unique in tests
+- Remove remote info for Eunomia since it's in CRAN
+
+PatientLevelPrediction 6.3.7
+======================
+- Clean up dependencies, tibble removed and IHT and ParallelLogger from CRAN
+- Use cohortIds for cohortCovariates to comply with FeatureExtraction
+- Add cdmDatabaseName from DatabaseDetails to model output
+- Fix bug when attributes weren't preserved on trainData$covariateData after split
+- Fix warnings in tests and speed them up
+- Fix bug in assignment operator in configurePython
+- Delay evaluation of plpData when using do.call like in learningCurves and 
+runMultiplePlp
+- Speed up population generation when subjectId's are distinct
+- Fix bug when population was still generated when provided to runPlp
+
+PatientLevelPrediction 6.3.6
+======================
+- fix bug with ohdsi shiny modules version check (issue 415)
+
+PatientLevelPrediction 6.3.5
+======================
+- Fix sklearnToJson to be compatible with scikit-learn>=1.3
+- Fix github actions so it's not hardcoded to use python 3.7
+
+
+PatientLevelPrediction 6.3.4
+======================
+- added spline feature engineering 
+- added age/sex stratified imputation feature engineering
+- changed result table execution date types to varchar
+- updated covariateSummary to use feature engineering
+
+PatientLevelPrediction 6.3.3
+======================
+- fixed bug introduced with new reticulate update in model saving to json tests
+
+
+PatientLevelPrediction 6.3.2
+======================
+- fixed bug with database insert if result is incomplete
+- updated/fixed documentation (Egill)
+- added model path to models (Henrik)
+- updated hyper-parameter saving to data.frame and made consistent 
+
+PatientLevelPrediction 6.3.1
+======================
+- fixed bug with multiple covariate settings in diagnose plp
+- added min cell count when exporting database results to csv files
+- light GBM added (thanks Jin Choi and Chungsoo Kim)
+- fixed minor bugs when uploading results to database
+
+PatientLevelPrediction 6.2.1
+======================
+- added ensure_installed("ResultModelManager") to getDataMigrator()
+
+PatientLevelPrediction 6.1.0
+======================
+- shiny app is now using ShinyAppBuilder with a config saved in the /inst folder
+
+PatientLevelPrediction 6.0.11
+======================
+- fixed bugs introduced when sklearn inputs changed
+- added sklearn model being saved as jsons
+- made changes around the DatabaseConnection get table names function to make it work for the updated DatabaseConnection
+- removed check RAM stop (now it just warns)
+
+PatientLevelPrediction 6.0.10
+======================
+- Updated test to skip test for FE setting if the model does not fit (this was causing occasional test fail)
+- replaced .data$ with "" for all dplyr::select to remove warnings
+
+PatientLevelPrediction 6.0.9
+======================
+- Fix bug with python type being required to be int
+
+PatientLevelPrediction 6.0.8
+======================
+- Allow priorType to be passed down to getCV function in case prior is not 'laplace'
+- Seed specified in Cyclops model wasn't passed to Cyclops
+
+PatientLevelPrediction 6.0.7
+======================
+- fixed issue with shiny viewer converting connection details to large json
+
+PatientLevelPrediction 6.0.6
+======================
+- added check for cdmDatabaseId into createDatabaseDetails
+- added test for check for cdmDatabaseId into createDatabaseDetails to error when NULL
+- removed session$onSessionEnded(shiny::stopApp)  from shiny server
+
+PatientLevelPrediction 6.0.5
+======================
+- fixing cox predictions
+
+PatientLevelPrediction 6.0.4
+======================
+- forcing cdmDatabaseId to be a string if integer is input
+
+PatientLevelPrediction 6.0.3
+======================
+- replaced utils::read.csv with readr::read_csv when inserting results from csv
+
+PatientLevelPrediction 6.0.2
+======================
+- replaced gsub with sub when inserting csvs to database
+
+PatientLevelPrediction 6.0.1
+======================
+- saved result specification csv in windows to fix odd formating issue
+
+PatientLevelPrediction 6.0.0
+======================
+- fixed sample data bugs
+- updated to use v1.0.0 of OhdsiShinyModules
+- updated plp database result tables to use the same structure for cohort and database as other HADES packages
+- added function to insert csv results into plp database result tables
+- added input for databaseId (database and version) when extracting data to be consistent with other HADES packages.  This is saved in plp objects.
+
+PatientLevelPrediction 5.4.4
+======================
+- fixed issue with 'preprocess' vs 'preprocessing' inconsistently used across models
+- added metaData tracking for feature engineering or preprocessing when predicting
+- fixed issue with FE using trainData$covariateData metaData rather than trainData
+- fixed bug when using sameData for FE
+
+PatientLevelPrediction 5.4.3
+======================
+- pulled in multiple bug fixes and test improvements from Egill
+- pulled in fix for learning curves from Henrik
+- Pulled in fix for feature engineering from Solomon
+- Cleaned check messages about comparing class(x) with a string by changing to inherits()
+
+PatientLevelPrediction 5.4.2
+======================
+- removed json saving for sklearn models since sklearn-json is no longer working for the latest sklearn
+
+
+PatientLevelPrediction 5.4.1
+======================
+- renamed the input corresponding to the string that gets appended to the results table names to tablePrefix
+- fixed issues with system.file() from SqlRender code breaking the tests
+- added an input fileAppend to the function that exports the database tables to csv files
+- moved the plp model (including preprocessing details) outside of the result database (into a specified folder) due to the size of the objects (too large to insert into the database). 
+
+PatientLevelPrediction 5.4.0
+======================
+- added saving of plp models into the result database 
+- added default cohortDefinitions in runMultiplePlp
+
+PatientLevelPrediction 5.3.3
+======================
+- added modelType to all models for database upload
+
+PatientLevelPrediction 5.3.2
+======================
+- moved FeatureExtraction to depends 
+- fixed using inherits()
+
+PatientLevelPrediction 5.3.1
+======================
+- moved most of the shiny app code into OhdsiShinyModules
+- removed shiny dependencies and added OhdsiShinyModules to suggests
+- fixed bug with linux sklearn saving
+
+PatientLevelPrediction 5.1.1
+======================
+- replaced cohortId to targetId for consistency throughout code
+
+PatientLevelPrediction 5.1.0
+======================
+- replaced targetId in model design to cohortId for consistency throughout code
+- replaced plpDataSettings to restrictPlpDataSettings to improve naming consistency 
+- added ability to use initial population in runPlp by adding the population to plpData$population
+- added splitSettings into modelDesign
+- replaced saving json settings with ParallelLogger function
+- updated database result schema (removed researcher_id from tables - if desired a new table with the setting_ids and researcher_id could be added, removed study tables and revised results table to performances table with a reference to model_design_id and development_database_id to enable validation results without a model to be inserted)
+- added diagnostic code based on PROBAST
+- added diagnostic shiny module
+- added code to create sqlite database and populate in uploadToDatabase
+- add code to convert runPlp+val to sqlite database when viewing shiny
+- added code to extract database results into csv files: extractDatabaseToCsv()
+
+
 PatientLevelPrediction 5.0.5
 ======================
 - pulled in GBM update (default hyper-parameters and variable importance fix) work done by Egill (egillax)
diff --git a/R/AdditionalCovariates.R b/R/AdditionalCovariates.R
index 2548eaf12..f19f30803 100644
--- a/R/AdditionalCovariates.R
+++ b/R/AdditionalCovariates.R
@@ -30,8 +30,9 @@
 #' @param cohortTable  the table name that contains the target population cohort
 #' @param rowIdField  string representing the unique identifier in the target population cohort
 #' @param aggregated  whether the covariate should be aggregated
-#' @param cohortId  cohort id for the target population cohort
+#' @param cohortIds  cohort id for the target cohort
 #' @param covariateSettings  settings for the covariate cohorts and time periods
+#' @param ...  additional arguments from FeatureExtraction
 #'
 #' @return
 #' The models will now be in the package
@@ -45,8 +46,9 @@ getCohortCovariateData <- function(
   cohortTable = "#cohort_person",
   rowIdField = "row_id",
   aggregated,
-  cohortId,
-  covariateSettings
+  cohortIds,
+  covariateSettings,
+  ...
   ){
   
   # Some SQL to construct the covariate:
@@ -58,23 +60,23 @@ getCohortCovariateData <- function(
     "}} as covariate_value",
     "from @cohort_temp_table a inner join @covariate_cohort_schema.@covariate_cohort_table b",
     " on a.subject_id = b.subject_id and ",
-    " b.cohort_start_date <= dateadd(day, @endDay, a.cohort_start_date) and ",
+    " b.cohort_start_date <= dateadd(day, @endDays, a.cohort_start_date) and ",
     " b.cohort_end_date >= dateadd(day, @startDay, a.cohort_start_date) ",
     "{@ageInteraction | @lnAgeInteraction}?{inner join @cdm_database_schema.person p on p.person_id=a.subject_id}",
     "where b.cohort_definition_id = @covariate_cohort_id
-    group by a.@row_id_field "
+    group by a.@row_id_field; "
   )
   
   sql <- SqlRender::render(
     sql,
     covariate_cohort_schema = covariateSettings$cohortDatabaseSchema,
     covariate_cohort_table = covariateSettings$cohortTable,
-    covariate_cohort_id = covariateSettings$cohortId,
+    covariate_cohort_id = covariateSettings$cohortIds,
     cohort_temp_table = cohortTable,
     row_id_field = rowIdField,
     startDay = covariateSettings$startDay,
     covariate_id = covariateSettings$covariateId,
-    endDay = covariateSettings$endDay,
+    endDays = covariateSettings$endDays,
     countval = covariateSettings$count,
     ageInteraction = covariateSettings$ageInteraction,
     lnAgeInteraction = covariateSettings$lnAgeInteraction,
@@ -94,7 +96,7 @@ getCohortCovariateData <- function(
   colnames(covariates) <- SqlRender::snakeCaseToCamelCase(colnames(covariates))
   # Construct covariate reference:
   sql <- "select @covariate_id as covariate_id, '@concept_set' as covariate_name,
-  @analysis_id as analysis_id, -1 as concept_id"
+  @analysis_id as analysis_id, -1 as concept_id;"
   sql <- SqlRender::render(
     sql = sql, 
     covariate_id = covariateSettings$covariateId,
@@ -102,7 +104,7 @@ getCohortCovariateData <- function(
     concept_set = paste('Cohort_covariate during day',
       covariateSettings$startDay,
       'through',
-      covariateSettings$endDay,
+      covariateSettings$endDays,
       'days relative to index:',
       ifelse(covariateSettings$count, 'Number of', ''),
       covariateSettings$covariateName,
@@ -191,9 +193,9 @@ createCohortCovariateSettings <- function(
     covariateId = cohortId*100000+settingId*1000+analysisId,
     cohortDatabaseSchema = cohortDatabaseSchema,
     cohortTable = cohortTable,
-    cohortId = cohortId,
+    cohortIds = cohortId,
     startDay = startDay,
-    endDay = endDay,
+    endDays = endDay,
     count = count,
     ageInteraction = ageInteraction,
     lnAgeInteraction = lnAgeInteraction,
diff --git a/R/AndromedaHelperFunctions.R b/R/AndromedaHelperFunctions.R
index 7a97b41b3..b6182610c 100644
--- a/R/AndromedaHelperFunctions.R
+++ b/R/AndromedaHelperFunctions.R
@@ -55,40 +55,28 @@ batchRestrict <- function(covariateData, population, sizeN = 10000000){
   newCovariateData <- Andromeda::andromeda(covariateRef = covariateData$covariateRef,
                                            analysisRef = covariateData$analysisRef)
   
-  maxRows <- RSQLite::dbGetQuery(covariateData, 
-                                 "SELECT count(*) as n FROM covariates;")
+  Andromeda::batchApply(covariateData$covariates, function(tempData) {
   
-  steps <- ceiling(maxRows$n/sizeN)
-  
-  
-  pb <- utils::txtProgressBar(style = 3)
-  
-  for(i in 1:steps){
-    utils::setTxtProgressBar(pb, i/steps)
-    
-    offset <- ((i-1)*sizeN)
-    limit <- sizeN
+    filtered <- dplyr::inner_join(tempData, population, by = 'rowId')
     
-    tempData <- RSQLite::dbGetQuery(covariateData, 
-                                    paste0("SELECT * FROM covariates LIMIT ",limit," OFFSET ",offset," ;"))
-    
-    filtered <- tempData %>% dplyr::inner_join(population, by = 'rowId')
-    
-    if(i==1){
+    if ("covariates" %in% names(newCovariateData)) {
+      Andromeda::appendToTable(newCovariateData$covariates, data = filtered)
+    } else {
       newCovariateData$covariates <- filtered
-    } else{
-      Andromeda::appendToTable(tbl = newCovariateData$covariates, 
-                               data = filtered)
     }
-  }
-  close(pb)
+  },
+  progressBar = TRUE,
+  batchSize = sizeN)
   
-  Andromeda::createIndex(tbl = newCovariateData$covariates, columnNames = 'covariateId', 
-    indexName = 'covariates_ncovariateIds')
-  Andromeda::createIndex(tbl = newCovariateData$covariates, c('rowId'),
-    indexName = 'covariates_rowId')
-  Andromeda::createIndex(tbl = newCovariateData$covariates, c('covariateId', 'covariateValue'),
-    indexName = 'covariates_covariateId_value')
+  Andromeda::createIndex(tbl = newCovariateData$covariates,
+                         columnNames = 'covariateId',
+                         indexName = 'covariates_ncovariateIds')
+  Andromeda::createIndex(tbl = newCovariateData$covariates,
+                         columnNames = 'rowId',
+                         indexName = 'covariates_rowId')
+  Andromeda::createIndex(tbl = newCovariateData$covariates,
+                         columnNames = c('covariateId', 'covariateValue'),
+                         indexName = 'covariates_covariateId_value')
   
   metaData$populationSize <- nrow(population)
   attr(newCovariateData, 'metaData') <- metaData
@@ -109,8 +97,9 @@ calculatePrevs <- function(plpData, population){
   #===========================
   
   # add population to sqllite
-  population <- tibble::as_tibble(population)
-  plpData$covariateData$population <- population %>% dplyr::select(.data$rowId, .data$outcomeCount)
+  population <- dplyr::as_tibble(population)
+  plpData$covariateData$population <- population %>% 
+    dplyr::select("rowId", "outcomeCount")
   
   outCount <- nrow(plpData$covariateData$population %>% dplyr::filter(.data$outcomeCount == 1))
   nonOutCount <- nrow(plpData$covariateData$population %>% dplyr::filter(.data$outcomeCount == 0))
@@ -120,7 +109,7 @@ calculatePrevs <- function(plpData, population){
     dplyr::group_by(.data$covariateId) %>% 
     dplyr::summarise(prev.out = 1.0*sum(.data$outcomeCount==1, na.rm = TRUE)/outCount,
               prev.noout = 1.0*sum(.data$outcomeCount==0, na.rm = TRUE)/nonOutCount) %>%
-    dplyr::select(.data$covariateId, .data$prev.out, .data$prev.noout)
+    dplyr::select("covariateId", "prev.out", "prev.noout")
   
   #clear up data
   ##plpData$covariateData$population <- NULL
diff --git a/R/CalibrationSummary.R b/R/CalibrationSummary.R
index fe8d297c2..e8094350b 100644
--- a/R/CalibrationSummary.R
+++ b/R/CalibrationSummary.R
@@ -146,7 +146,7 @@ getCalibrationSummary_survival <- function(
   } else{
     gval <- numberOfStrata
   }
-  groups<-Hmisc::cut2(predictionOfInterest$value,g=gval)
+  groups<-cut2(predictionOfInterest$value,g=gval)
   n.groups<-length(levels(groups))
   pred<-tapply(predictionOfInterest$value,groups,mean)
   sizesN<-tapply(predictionOfInterest$value,groups,length)
@@ -199,3 +199,4 @@ getCalibrationSummary_survival <- function(
   return(result)
   
 }
+
diff --git a/R/CovariateSummary.R b/R/CovariateSummary.R
index b344caa7d..6972441a5 100644
--- a/R/CovariateSummary.R
+++ b/R/CovariateSummary.R
@@ -66,6 +66,28 @@ covariateSummary <- function(
     strata = strata
   )
   
+  # apply feature engineering
+  if(!is.null(featureEngineering)){
+    
+    # create copy of covariateData
+    newCovariateData <- Andromeda::andromeda(
+      covariateRef = covariateData$covariateRef,
+      analysisRef = covariateData$analysisRef,
+      covariates = covariateData$covariates
+      )
+    covariateData <- newCovariateData
+    
+    if(!is.null(featureEngineering$funct)){
+      featureEngineering <- list(featureEngineering)
+    }
+    
+    for(fe in featureEngineering){
+      feSettings <- fe$settings
+      feSettings$trainData = list(covariateData = covariateData)
+      covariateData <- do.call(fe$funct, feSettings)$covariateData
+    }
+  }
+  
   # make this run in parallel for big speed improvements..
   covariateSummariesPerStrata <- lapply(subsetList, 
     function(x){
@@ -114,10 +136,10 @@ aggregateCovariateSummaries <- function(
     ParallelLogger::logInfo('Aggregating with no labels or strata')
     result <- covariateSummariesPerStrata %>% 
       dplyr::select(
-        .data$covariateId, 
-        .data$CovariateCount, 
-        .data$CovariateMean, 
-        .data$CovariateStDev, 
+        "covariateId", 
+        "CovariateCount", 
+        "CovariateMean", 
+        "CovariateStDev", 
       )
   }
   
@@ -126,11 +148,11 @@ aggregateCovariateSummaries <- function(
     ParallelLogger::logInfo('Aggregating with only labels or strata')
     resultLabels <- covariateSummariesPerStrata %>% 
       dplyr::select(
-        .data$group,
-        .data$covariateId, 
-        .data$CovariateCount, 
-        .data$CovariateMean, 
-        .data$CovariateStDev, 
+        "group",
+        "covariateId", 
+        "CovariateCount", 
+        "CovariateMean", 
+        "CovariateStDev", 
       )
 
     resultLabels <- tidyr::pivot_longer(
@@ -142,7 +164,7 @@ aggregateCovariateSummaries <- function(
     
     resultLabels <- resultLabels %>% 
       dplyr::mutate(group_variable = paste(.data$group, .data$variable, sep ='_')) %>%
-      dplyr::select(-.data$group, -.data$variable)
+      dplyr::select(-"group", -"variable")
     
     resultLabels <- tidyr::pivot_wider(
       data = resultLabels, 
@@ -176,11 +198,11 @@ aggregateCovariateSummaries <- function(
     # labels and strata
     resultLabelStratas <- covariateSummariesPerStrata %>% 
       dplyr::select(
-        .data$group,
-        .data$covariateId, 
-        .data$CovariateCount, 
-        .data$CovariateMean, 
-        .data$CovariateStDev, 
+        "group",
+        "covariateId", 
+        "CovariateCount", 
+        "CovariateMean", 
+        "CovariateStDev", 
       )
     
     resultLabelStratas <- tidyr::pivot_longer(
@@ -192,7 +214,7 @@ aggregateCovariateSummaries <- function(
     
     resultLabelStratas <- resultLabelStratas %>% 
       dplyr::mutate(group_variable = paste(.data$group, .data$variable, sep ='_')) %>%
-      dplyr::select(-.data$group, -.data$variable)
+      dplyr::select(-"group", -"variable")
     
     resultLabelStratas <- tidyr::pivot_wider(
       data = resultLabelStratas, 
@@ -220,11 +242,11 @@ aggregateCovariateSummaries <- function(
       CovariateStDev = sqrt(sum(.data$sumSquares)/sum(.data$N) - (sum(.data$sumVal)/sum(.data$N))^2 )
     ) %>% 
       dplyr::select(
-        .data$groupLabel,
-        .data$covariateId, 
-        .data$CovariateCount, 
-        .data$CovariateMean, 
-        .data$CovariateStDev
+        "groupLabel",
+        "covariateId", 
+        "CovariateCount", 
+        "CovariateMean", 
+        "CovariateStDev"
       )
     
     resultLabels <- tidyr::pivot_longer(
@@ -236,7 +258,7 @@ aggregateCovariateSummaries <- function(
     
     resultLabels <- resultLabels %>% 
       dplyr::mutate(group_variable = paste(.data$groupLabel, .data$variable, sep ='_')) %>%
-      dplyr::select(-.data$groupLabel, -.data$variable)
+      dplyr::select(-"groupLabel", -"variable")
     
     resultLabels <- tidyr::pivot_wider(
       data = resultLabels, 
@@ -304,7 +326,7 @@ createCovariateSubsets <- function(
     ParallelLogger::logInfo(paste0('calculating subset of strata ',i))
     subset <- cohort %>% 
       dplyr::filter(.data$finalStrata == finalStratas[[i]]) %>%
-      dplyr::select(.data$rowId)
+      dplyr::select("rowId")
     
     result[[i]] <- list(
       subset = subset, 
diff --git a/R/CyclopsModels.R b/R/CyclopsModels.R
index 54f6065f8..16f035164 100644
--- a/R/CyclopsModels.R
+++ b/R/CyclopsModels.R
@@ -19,11 +19,13 @@
 
 fitCyclopsModel <- function(
   trainData, 
-  param, 
+  modelSettings, # old:param, 
   search='adaptive',
   analysisId,
   ...){
   
+  param <- modelSettings$param
+  
   # check plpData is coo format:
   if (!FeatureExtraction::isCovariateData(trainData$covariateData)){
     stop("Needs correct covariateData")
@@ -36,11 +38,26 @@ fitCyclopsModel <- function(
       y = sapply(.data$outcomeCount, function(x) min(1,x)),
       time = .data$survivalTime
     )
-    
-  covariates <- filterCovariateIds(param, trainData$covariateData)
   
-  start <- Sys.time() 
+  covariates <- filterCovariateIds(param, trainData$covariateData)
   
+  if (!is.null(param$priorCoefs)) {
+    sourceCoefs <- param$priorCoefs %>%
+      dplyr::filter(abs(.data$betas)>0 & .data$covariateIds != "(Intercept)")
+    
+    newCovariates <- covariates %>%
+      dplyr::filter(.data$covariateId %in% !!sourceCoefs$covariateIds) %>%
+      dplyr::mutate(newCovariateId = .data$covariateId*-1) %>%
+      dplyr::select(-"covariateId") %>%
+      dplyr::rename(covariateId = .data$newCovariateId) %>%
+      dplyr::collect()
+    
+    Andromeda::appendToTable(covariates, newCovariates)
+    
+  }
+
+  start <- Sys.time()
+
   cyclopsData <- Cyclops::convertToCyclopsData(
     outcomes = trainData$covariateData$labels,
     covariates = covariates,
@@ -50,6 +67,20 @@ fitCyclopsModel <- function(
     normalize = NULL,
     quiet = TRUE
     )
+  
+  if (!is.null(param$priorCoefs)) {
+    fixedCoefficients <- c(FALSE,
+                           rep(TRUE, nrow(sourceCoefs)),
+                           rep(FALSE, length(cyclopsData$coefficientNames)-(nrow(sourceCoefs)+1)))
+    
+    startingCoefficients <- rep(0, length(fixedCoefficients))
+    
+    # skip intercept index
+    startingCoefficients[2:(nrow(sourceCoefs)+1)] <- sourceCoefs$betas
+  } else {
+    startingCoefficients <- NULL
+    fixedCoefficients <- NULL 
+  }
 
   if(settings$crossValidationInPrior){  
     param$priorParams$useCrossValidation <- max(trainData$folds$index)>1
@@ -69,7 +100,8 @@ fitCyclopsModel <- function(
       selectorType = settings$selectorType,
       noiseLevel = "silent",
       threads = settings$threads,
-      maxIterations = settings$maxIterations
+      maxIterations = settings$maxIterations,
+      seed = settings$seed
       )
     
     fit <- tryCatch({
@@ -77,7 +109,9 @@ fitCyclopsModel <- function(
       Cyclops::fitCyclopsModel(
         cyclopsData = cyclopsData, 
         prior = prior, 
-        control = control
+        control = control,
+        fixedCoefficients = fixedCoefficients,
+        startingCoefficients = startingCoefficients
         )}, 
       finally = ParallelLogger::logInfo('Done.')
       )
@@ -88,15 +122,21 @@ fitCyclopsModel <- function(
       finally = ParallelLogger::logInfo('Done.')) 
   }
   
+  
   modelTrained <- createCyclopsModel(
     fit = fit, 
     modelType = settings$modelType, 
     useCrossValidation = max(trainData$folds$index)>1, 
     cyclopsData = cyclopsData, 
     labels = trainData$covariateData$labels,
-    folds = trainData$folds
+    folds = trainData$folds,
+    priorType = param$priorParams$priorType
     )
-
+  
+  if (!is.null(param$priorCoefs)) {
+    modelTrained$coefficients <- reparamTransferCoefs(modelTrained$coefficients)
+  }
+  
   # TODO get optimal lambda value
   ParallelLogger::logTrace('Returned from fitting to LassoLogisticRegression')
   comp <- Sys.time() - start
@@ -116,7 +156,7 @@ fitCyclopsModel <- function(
   prediction$evaluationType <- 'Train'
   
   # get cv AUC if exists
-  cvPerFold <- c()
+  cvPerFold <- data.frame()
   if(!is.null(modelTrained$cv)){
     cvPrediction  <- do.call(rbind, lapply(modelTrained$cv, function(x){x$predCV}))
     cvPrediction$evaluationType <- 'CV'
@@ -127,7 +167,17 @@ fitCyclopsModel <- function(
     
     cvPerFold <-  unlist(lapply(modelTrained$cv, function(x){x$out_sample_auc}))
     if(length(cvPerFold)>0){
-      names(cvPerFold) <- paste0('fold_auc', 1:length(cvPerFold))
+      cvPerFold <- data.frame(
+        metric = 'AUC',
+        fold = 1:length(cvPerFold),
+        value = cvPerFold,
+        startingVariance = ifelse(is.null(param$priorParams$variance), 'NULL', param$priorParams$variance),
+        lowerLimit = ifelse(is.null(param$lowerLimit), 'NULL', param$lowerLimit),
+        upperLimit = ifelse(is.null(param$upperLimit), 'NULL', param$upperLimit),
+        tolerance  = ifelse(is.null(settings$tolerance), 'NULL', settings$tolerance)
+      )
+    } else{
+      cvPerFold <- data.frame()
     }
     
     # remove the cv from the model:
@@ -136,39 +186,41 @@ fitCyclopsModel <- function(
   
   result <- list(
     model = modelTrained,
+    
+    preprocessing = list(
+      featureEngineering = attr(trainData, "metaData")$featureEngineering,#learned mapping
+      tidyCovariates = attr(trainData$covariateData, "metaData")$tidyCovariateDataSettings,  #learned mapping
+      requireDenseMatrix = F
+    ),
+    
     prediction = prediction,
     
-    settings = list(
-      plpDataSettings = attr(trainData, "metaData")$plpDataSettings,
+    modelDesign = PatientLevelPrediction::createModelDesign(
+      targetId = attr(trainData, "metaData")$targetId, # added
+      outcomeId = attr(trainData, "metaData")$outcomeId, # added
+      restrictPlpDataSettings = attr(trainData, "metaData")$restrictPlpDataSettings, # made this restrictPlpDataSettings
       covariateSettings = attr(trainData, "metaData")$covariateSettings,
-      featureEngineering = attr(trainData$covariateData, "metaData")$featureEngineering,
-      tidyCovariates = attr(trainData$covariateData, "metaData")$tidyCovariateDataSettings, 
-      covariateMap = NULL,
-      requireDenseMatrix = F,
-      populationSettings = attr(trainData, "metaData")$populationSettings,
-      modelSettings = list(
-        model = settings$modelType, 
-        param = param,
-        finalModelParameters = list(
-          variance = modelTrained$priorVariance,
-          log_likelihood = modelTrained$log_likelihood
-        ),
-        extraSettings = attr(param, 'settings')
-      ),
+      populationSettings = attr(trainData, "metaData")$populationSettings, 
+      featureEngineeringSettings = attr(trainData, "metaData")$featureEngineeringSettings,
+      preprocessSettings = attr(trainData$covariateData, "metaData")$preprocessSettings,
+      modelSettings = modelSettings, #modified
       splitSettings = attr(trainData, "metaData")$splitSettings,
       sampleSettings = attr(trainData, "metaData")$sampleSettings
-      
-      
     ),
     
     trainDetails = list(
-      analysisId = analysisId,
-      cdmDatabaseSchema = attr(trainData, "metaData")$cdmDatabaseSchema,
-      outcomeId = attr(trainData, "metaData")$outcomeId,
-      cohortId = attr(trainData, "metaData")$cohortId,
+      analysisId = analysisId, 
+      analysisSource = '', #TODO add from model
+      developmentDatabase = attr(trainData, "metaData")$cdmDatabaseName,
+      developmentDatabaseSchema = attr(trainData, "metaData")$cdmDatabaseSchema, 
       attrition = attr(trainData, "metaData")$attrition, 
-      trainingTime = comp,
+      trainingTime =  paste(as.character(abs(comp)), attr(comp,'units')),
       trainingDate = Sys.Date(),
+      modelName = settings$modelType,
+      finalModelParameters = list(
+        variance = modelTrained$priorVariance,
+        log_likelihood = modelTrained$log_likelihood
+      ),
       hyperParamSearch = cvPerFold
     ),
     
@@ -213,20 +265,20 @@ predictCyclops <- function(plpModel, data, cohort ) {
   
   # survival cyclops use baseline hazard to convert to risk from exp(LP) to 1-S^exp(LP)
   if(attr(plpModel, 'modelType') == 'survival'){
-    if(!is.null(plpModel$model$baselineHazard)){
+    if(!is.null(plpModel$model$baselineSurvival)){
       if(is.null(attr(cohort, 'timepoint'))){
         timepoint <- attr(cohort,'metaData')$populationSettings$riskWindowEnd
       } else{
         timepoint <- attr(cohort, 'timepoint')
       }
-      bhind <- which.min(abs(plpModel$model$baselineHazard$time-timepoint))
-      #prediction$value <- 1-plpModel$model$baselineHazard$surv[bhind]^prediction$value
-      prediction$value <- (1-plpModel$model$baselineHazard$surv[bhind])*prediction$value
+      bhind <- which.min(abs(plpModel$model$baselineSurvival$time-timepoint))
+      # 1- baseline survival(time)^ (exp(betas*values))
+      prediction$value <- 1-plpModel$model$baselineSurvival$surv[bhind]^prediction$value
       
       
       metaData <- list()
-      metaData$baselineHazardTimepoint <- plpModel$model$baselineHazard$time[bhind]
-      metaData$baselineHazard <- plpModel$model$baselineHazard$surv[bhind]
+      metaData$baselineSurvivalTimepoint <- plpModel$model$baselineSurvival$time[bhind]
+      metaData$baselineSurvival <- plpModel$model$baselineSurvival$surv[bhind]
       metaData$offset <- 0
       
       attr(prediction, 'metaData') <- metaData
@@ -246,11 +298,11 @@ predictCyclopsType <- function(coefficients, population, covariateData, modelTyp
     stop("Needs correct covariateData")
   }
   
-  intercept <- coefficients[names(coefficients)%in%'(Intercept)']
+  intercept <- coefficients$betas[coefficients$covariateId%in%'(Intercept)']
   if(length(intercept)==0) intercept <- 0
-  coefficients <- coefficients[!names(coefficients)%in%'(Intercept)']
-  coefficients <- data.frame(beta = as.numeric(coefficients),
-    covariateId = as.numeric(names(coefficients)) #!@ modified 
+  betas <- coefficients$betas[!coefficients$covariateIds%in%'(Intercept)']
+  coefficients <- data.frame(beta = betas,
+    covariateId = coefficients$covariateIds[coefficients$covariateIds!='(Intercept)']
   )
   coefficients <- coefficients[coefficients$beta != 0, ]
   if(sum(coefficients$beta != 0)>0){
@@ -262,7 +314,7 @@ predictCyclopsType <- function(coefficients, population, covariateData, modelTyp
       dplyr::mutate(values = .data$covariateValue*.data$beta) %>%
       dplyr::group_by(.data$rowId) %>%
       dplyr::summarise(value = sum(.data$values, na.rm = TRUE)) %>%
-      dplyr::select(.data$rowId, .data$value)
+      dplyr::select("rowId", "value")
     
     prediction <- as.data.frame(prediction)
     prediction <- merge(population, prediction, by ="rowId", all.x = TRUE, fill = 0)
@@ -294,17 +346,21 @@ predictCyclopsType <- function(coefficients, population, covariateData, modelTyp
 }
 
 
-createCyclopsModel <- function(fit, modelType, useCrossValidation, cyclopsData, labels, folds){
+createCyclopsModel <- function(fit, modelType, useCrossValidation, cyclopsData, labels, folds,
+                               priorType){
 
   if (is.character(fit)) {
     coefficients <- c(0)
+    names(coefficients) <- ''
     status <- fit
   } else if (fit$return_flag == "ILLCONDITIONED") {
     coefficients <- c(0)
+    names(coefficients) <- ''
     status <- "ILL CONDITIONED, CANNOT FIT"
     ParallelLogger::logWarn(paste("GLM fitting issue: ", status))
   } else if (fit$return_flag == "MAX_ITERATIONS") {
     coefficients <- c(0)
+    names(coefficients) <- ''
     status <- "REACHED MAXIMUM NUMBER OF ITERATIONS, CANNOT FIT"
     ParallelLogger::logWarn(paste("GLM fitting issue: ", status))
   } else {
@@ -313,27 +369,33 @@ createCyclopsModel <- function(fit, modelType, useCrossValidation, cyclopsData,
     ParallelLogger::logInfo(paste("GLM fit status: ", status))
   }
   
+  # use a dataframe for the coefficients
+  betas <- as.numeric(coefficients)
+  betaNames <- names(coefficients)
+  coefficients <- data.frame(betas=betas, covariateIds=betaNames)
+  
   outcomeModel <- list(
-    coefficients = coefficients, 
     priorVariance = fit$variance,
     log_likelihood = fit$log_likelihood,
     modelType = modelType,
-    modelStatus = status
+    modelStatus = status,
+    coefficients = coefficients
   )
 
   if(modelType == "cox" || modelType == "survival") {
-    baselineHazard <- tryCatch({survival::survfit(fit, type = "aalen")},
+    baselineSurvival <- tryCatch({survival::survfit(fit, type = "aalen")},
       error = function(e) {ParallelLogger::logInfo(e); return(NULL)})
-    if(is.null(baselineHazard)){
+    if(is.null(baselineSurvival)){
       ParallelLogger::logInfo('No baseline hazard function returned')
     }
-    outcomeModel$baselineHazard <- baselineHazard
+    outcomeModel$baselineSurvival <- baselineSurvival
   }
   class(outcomeModel) <- "plpModel"
   
-  #get CV
-  if(modelType == "logistic" && useCrossValidation){
-    outcomeModel$cv <- getCV(cyclopsData, labels, cvVariance = fit$variance, folds = folds)
+  #get CV - added && status == "OK" to only run if the model fit sucsessfully 
+  if(modelType == "logistic" && useCrossValidation && status == "OK"){
+    outcomeModel$cv <- getCV(cyclopsData, labels, cvVariance = fit$variance, folds = folds,
+                             priorType = priorType)
   }
   
   return(outcomeModel)
@@ -366,10 +428,13 @@ getCV <- function(
   cyclopsData, 
   labels,
   cvVariance,
-  folds
+  folds,
+  priorType
 )
 {
-  fixed_prior <- Cyclops::createPrior("laplace", variance = cvVariance, useCrossValidation = FALSE)
+  fixed_prior <- Cyclops::createPrior(priorType = priorType, 
+                                      variance = cvVariance, 
+                                      useCrossValidation = FALSE)
   
   # add the index to the labels
   labels <- merge(labels, folds, by = 'rowId')
@@ -402,8 +467,8 @@ getCV <- function(
 
 getVariableImportance <- function(modelTrained, trainData){
   varImp <- data.frame(
-    covariateId = as.double(names(modelTrained$coefficients)[names(modelTrained$coefficients)!='(Intercept)']),
-    value = modelTrained$coefficients[names(modelTrained$coefficients)!='(Intercept)']
+    covariateId = as.double(modelTrained$coefficients$covariateIds[modelTrained$coefficients$covariateIds!='(Intercept)']),
+    value = modelTrained$coefficients$betas[modelTrained$coefficients$covariateIds!='(Intercept)']
   )
 
 if(sum(abs(varImp$value)>0)==0){
@@ -420,7 +485,7 @@ if(sum(abs(varImp$value)>0)==0){
     #dplyr::left_join(trainData$covariateData$varImp) %>%
     dplyr::left_join(varImp, by = 'covariateId') %>%
     dplyr::mutate(covariateValue = ifelse(is.na(.data$value), 0, .data$value)) %>%
-    dplyr::select(-.data$value) %>%
+    dplyr::select(-"value") %>%
     dplyr::arrange(-abs(.data$covariateValue)) %>%
     dplyr::collect()
 }
@@ -446,3 +511,19 @@ filterCovariateIds <- function(param, covariateData){
   }
   return(covariates)
 }
+
+reparamTransferCoefs <- function(inCoefs) {
+  transferCoefs <- inCoefs %>%
+    dplyr::filter(grepl("-", .data$covariateIds))
+  
+  transferCoefs$covariateIds <- substring(transferCoefs$covariateIds, 2)
+  
+  originalCoefs <- inCoefs %>%
+    dplyr::filter(!grepl("-", .data$covariateIds))
+  
+  coefs <- rbind(originalCoefs, transferCoefs)
+  coefs <- rowsum(coefs$betas, coefs$covariateIds)
+  coefs <- data.frame(betas = coefs, covariateIds = rownames(coefs), row.names = NULL)
+
+  return(coefs)
+}
\ No newline at end of file
diff --git a/R/CyclopsSettings.R b/R/CyclopsSettings.R
index 9bc3766d4..fb688f03e 100644
--- a/R/CyclopsSettings.R
+++ b/R/CyclopsSettings.R
@@ -10,6 +10,7 @@
 #' @param lowerLimit  	Numeric: Lower prior variance limit for grid-search
 #' @param tolerance   Numeric: maximum relative change in convergence criterion from successive iterations to achieve convergence
 #' @param maxIterations 	Integer: maximum iterations of Cyclops to attempt before returning a failed-to-converge error
+#' @param priorCoefs    Use coefficients from a previous model as starting points for model fit (transfer learning)
 #' 
 #' @examples
 #' model.lr <- setLassoLogisticRegression()
@@ -24,7 +25,9 @@ setLassoLogisticRegression<- function(
   upperLimit = 20, 
   lowerLimit = 0.01,
   tolerance = 2e-06,
-  maxIterations = 3000){
+  maxIterations = 3000,
+  priorCoefs = NULL
+  ){
   
   checkIsClass(seed, c('numeric','NULL','integer'))
   if(is.null(seed[1])){
@@ -48,7 +51,8 @@ setLassoLogisticRegression<- function(
       ),
     includeCovariateIds = includeCovariateIds, 
     upperLimit = upperLimit, 
-    lowerLimit = lowerLimit
+    lowerLimit = lowerLimit,
+    priorCoefs = priorCoefs
     )
   
   attr(param, 'settings') <- list(
@@ -107,8 +111,6 @@ setCoxModel <- function(
   maxIterations = 3000
 ){
   
-  ensure_installed("survAUC")
-
   checkIsClass(seed, c('numeric','NULL','integer'))
   if(is.null(seed[1])){
     seed <- as.integer(sample(100000000,1))
@@ -204,7 +206,7 @@ setIterativeHardThresholding<- function(
     stop("forceIntercept must be of type: logical")
   if(!is.logical(fitBestSubset))
     stop("fitBestSubset must be of type: logical")
-  if(!class(seed)%in%c('numeric','NULL','integer'))
+  if(!inherits(x = seed, what = c('numeric','NULL','integer')))
     stop('Invalid seed')
 
 
diff --git a/R/DataSplitting.R b/R/DataSplitting.R
index 0c5677759..93c7a060e 100644
--- a/R/DataSplitting.R
+++ b/R/DataSplitting.R
@@ -17,7 +17,7 @@
 
 
 #' Create the settings for defining how the plpData are split into test/validation/train sets using 
-#' default splitting functions (either random stratified by outcome, time or subject splitting).
+#' default splitting functions (either random stratified by outcome, time or subject splitting)
 #'
 #' @details
 #' Returns an object of class \code{splitSettings} that specifies the splitting function that will be called and the settings
@@ -28,9 +28,9 @@
 #' @param nfold             (numeric) An integer > 1 specifying the number of folds used in cross validation
 #' @param splitSeed         (numeric) A seed to use when splitting the data for reproducibility (if not set a random number will be generated)
 #' @param type              (character) Choice of:  \itemize{
-#'                                         \item{'stratified'}{ Each data point is randomly assigned into the test or a train fold set but this is done stratified such that the outcome rate is consistent in each partition }
-#'                                         \item{'time')}{ Older data are assigned into the training set and newer data are assigned into the test set}
-#'                                         \item{'subject'}{ Data are partitioned by subject, if a subject is in the data more than once, all the data points for the subject are assigned either into the test data or into the train data (not both).}
+#'                                         \item'stratified' Each data point is randomly assigned into the test or a train fold set but this is done stratified such that the outcome rate is consistent in each partition 
+#'                                         \item'time' Older data are assigned into the training set and newer data are assigned into the test set
+#'                                         \item'subject' Data are partitioned by subject, if a subject is in the data more than once, all the data points for the subject are assigned either into the test data or into the train data (not both).
 #'                                         } 
 #'
 #' @return
@@ -87,17 +87,17 @@ createDefaultSplitSetting <- function(testFraction=0.25,
 #'
 #' @details
 #' Returns a list containing the training data (Train) and optionally the test data (Test).  Train is an Andromeda object containing
-#' \itemize{\item{covariates}{ a table (rowId, covariateId, covariateValue) containing the covariates for each data point in the train data }
-#'          \item{covariateRef}{ a table with the covariate information}
-#'          \item{labels)}{ a table (rowId, outcomeCount, ...) for each data point in the train data (outcomeCount is the class label) }
-#'          \item{folds}{ a table (rowId, index) specifying which training fold each data point is in.}
+#' \itemize{\item covariates: a table (rowId, covariateId, covariateValue) containing the covariates for each data point in the train data
+#'          \item covariateRef: a table with the covariate information
+#'          \item labels: a table (rowId, outcomeCount, ...) for each data point in the train data (outcomeCount is the class label) 
+#'          \item folds: a table (rowId, index) specifying which training fold each data point is in.
 #'          } 
 #' Test is an Andromeda object containing
-#' \itemize{\item{covariates}{ a table (rowId, covariateId, covariateValue) containing the covariates for each data point in the test data }
-#'          \item{covariateRef}{ a table with the covariate information}
-#'          \item{labels)}{ a table (rowId, outcomeCount, ...) for each data point in the test data (outcomeCount is the class label) }
+#' \itemize{\item covariates: a table (rowId, covariateId, covariateValue) containing the covariates for each data point in the test data 
+#'          \item covariateRef: a table with the covariate information
+#'          \item labels: a table (rowId, outcomeCount, ...) for each data point in the test data (outcomeCount is the class label) 
 #'          } 
-#' 
+#'   
 #' 
 #'
 #' @param plpData                          An object of type \code{plpData} - the patient level prediction
@@ -146,9 +146,11 @@ splitData <- function(plpData = plpData,
     #trainData$covariateData$covariateRef <- plpData$covariateRef
     attr(trainData, "metaData") <- list(
       outcomeId = attr(population, "metaData")$outcomeId,
-      cohortId = attr(population, "metaData")$cohortId,
+      targetId = attr(population, "metaData")$targetId,
       cdmDatabaseSchema = plpData$metaData$databaseDetails$cdmDatabaseSchema,
-      plpDataSettings = attr(population, "metaData")$plpDataSettings,
+      cdmDatabaseName = plpData$metaData$databaseDetails$cdmDatabaseName,
+      cdmDatabaseId = plpData$metaData$databaseDetails$cdmDatabaseId,
+      restrictPlpDataSettings = attr(population, "metaData")$restrictPlpDataSettings,
       covariateSettings = plpData$metaData$covariateSettings,
       populationSettings = attr(population, "metaData")$populationSettings,
       attrition = attr(population, "metaData")$attrition,
@@ -183,9 +185,11 @@ splitData <- function(plpData = plpData,
     }
     attr(trainData, "metaData") <- list(
       outcomeId = attr(population, "metaData")$outcomeId,
-      cohortId = attr(population, "metaData")$cohortId,
+      targetId = attr(population, "metaData")$targetId,
       cdmDatabaseSchema = plpData$metaData$databaseDetails$cdmDatabaseSchema,
-      plpDataSettings = attr(population, "metaData")$plpDataSettings,
+      cdmDatabaseName = plpData$metaData$databaseDetails$cdmDatabaseName,
+      cdmDatabaseId = plpData$metaData$databaseDetails$cdmDatabaseId,
+      restrictPlpDataSettings = attr(population, "metaData")$restrictPlpDataSettings,
       covariateSettings = plpData$metaData$covariateSettings,
       populationSettings = attr(population, "metaData")$populationSettings,
       attrition = attr(population, "metaData")$attrition,
@@ -193,10 +197,6 @@ splitData <- function(plpData = plpData,
       populationSize = nrow(trainData$labels)
       )
     
-    # add pop size to covariateData as used in tidyCovariates
-    attr(trainData$covariateData, "metaData") <- list(populationSize = nrow(trainData$labels))
-    class(trainData$covariateData) <- "CovariateData"
-    
     testId <- splitId[splitId$index<0,]
     testData <- list()
     class(testData) <- 'plpData'
@@ -212,7 +212,6 @@ splitData <- function(plpData = plpData,
         data.frame(rowId = testId$rowId), 
         sizeN = 10000000)
     }
-    class(testData$covariateData) <- "CovariateData"
     
     result <- list(
       Train =  trainData,
diff --git a/R/DatabaseMigration.R b/R/DatabaseMigration.R
new file mode 100644
index 000000000..bc480ce32
--- /dev/null
+++ b/R/DatabaseMigration.R
@@ -0,0 +1,68 @@
+# @file DatabaseMigration.R
+#
+# Copyright 2023 Observational Health Data Sciences and Informatics
+#
+# This file is part of PatientLevelPrediction
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitatons under the License.
+#
+#' Migrate Data model
+#' @description
+#' Migrate data from current state to next state
+#'
+#' It is strongly advised that you have a backup of all data (either sqlite files, a backup database (in the case you
+#' are using a postgres backend) or have kept the csv/zip files from your data generation.
+#'
+#' @param connectionDetails             DatabaseConnector connection details object
+#' @param databaseSchema                String schema where database schema lives
+#' @param  tablePrefix                  (Optional) Use if a table prefix is used before table names (e.g. "cd_")
+#' 
+#' @export
+migrateDataModel <- function(connectionDetails, databaseSchema, tablePrefix = "") {
+  ParallelLogger::logInfo("Migrating data set")
+  
+  migrator <- getDataMigrator(
+    connectionDetails = connectionDetails, 
+    databaseSchema = databaseSchema, 
+    tablePrefix = tablePrefix
+    )
+  migrator$executeMigrations()
+  migrator$finalize()
+  
+  ParallelLogger::logInfo("Updating version number")
+  updateVersionSql <- SqlRender::loadRenderTranslateSql(
+    "UpdateVersionNumber.sql",
+    packageName = utils::packageName(),
+    database_schema = databaseSchema,
+    table_prefix = tablePrefix,
+    version_number = utils::packageVersion("PatientLevelPrediction"),
+    dbms = "sql server" # this is the same for all dbms so just using sql server
+  )
+  
+  connection <- DatabaseConnector::connect(connectionDetails = connectionDetails)
+  on.exit(DatabaseConnector::disconnect(connection))
+  DatabaseConnector::executeSql(connection, updateVersionSql)
+}
+
+
+getDataMigrator <- function(connectionDetails, databaseSchema, tablePrefix = "") {
+  ensure_installed("ResultModelManager")
+  
+  ResultModelManager::DataMigrationManager$new(
+    connectionDetails = connectionDetails,
+    databaseSchema = databaseSchema,
+    tablePrefix = tablePrefix,
+    migrationPath = "migrations",
+    packageName = utils::packageName()
+  )
+}
\ No newline at end of file
diff --git a/R/DemographicSummary.R b/R/DemographicSummary.R
index 908e5e2de..b3df14305 100644
--- a/R/DemographicSummary.R
+++ b/R/DemographicSummary.R
@@ -44,7 +44,7 @@ getDemographicSummary_binary <- function(prediction, evalColumn , ...){
         ageGroup = paste0('Age group: ', floor(.data$ageYear/5)*5, '-',floor(.data$ageYear/5)*5+4),
         genId = .data$gender,
         genGroup = ifelse(.data$gender==8507, 'Male', 'Female')) %>%
-      dplyr::select(.data$rowId,.data$ageId,.data$ageGroup,.data$genId,.data$genGroup ) %>%
+      dplyr::select("rowId","ageId","ageGroup","genId","genGroup") %>%
       dplyr::inner_join(predictionOfInterest[,colnames(predictionOfInterest)%in%c('rowId', 'value','outcomeCount','survivalTime')], by='rowId')
     
     demographicData <- demographicData %>%
@@ -88,7 +88,7 @@ getDemographicSummary_survival <- function(prediction, evalColumn, timepoint = N
         ageGroup = paste0('Age group: ', floor(.data$ageYear/5)*5, '-',floor(.data$ageYear/5)*5+4),
         genId = .data$gender,
         genGroup = ifelse(.data$gender==8507, 'Male', 'Female')) %>%
-      dplyr::select(.data$rowId,.data$ageId,.data$ageGroup,.data$genId,.data$genGroup ) %>%
+      dplyr::select("rowId","ageId","ageGroup","genId","genGroup" ) %>%
       dplyr::inner_join(predictionOfInterest[,colnames(predictionOfInterest)%in%c('rowId', 'value','outcomeCount','survivalTime')], by='rowId')
     
     
@@ -111,30 +111,30 @@ getDemographicSummary_survival <- function(prediction, evalColumn, timepoint = N
         tempDemo <- demographicSum %>% 
           dplyr::filter( .data$genGroup == gen & .data$ageGroup == age )
         
-        if(nrow(tempDemo)>0){
-          t1 <- tempDemo %>% dplyr::select(.data$t)
-          y1 <- tempDemo %>% dplyr::select(.data$y)
-          p1 <- tempDemo %>% dplyr::select(.data$value)
+        if (nrow(tempDemo) > 1 & length(unique(tempDemo$y)) > 1) {
+          t <- tempDemo$t
+          y <- tempDemo$y
+          value <- tempDemo$value
           
           out <- tryCatch(
             {
               summary(
-                survival::survfit(survival::Surv(t1$t, y1$y) ~ 1), 
+                survival::survfit(survival::Surv(t, y) ~ 1), 
                 times = timepoint
               )
             },
-            error = function(e){ParallelLogger::logError(e); return(NULL)}
+            error = function(e){ParallelLogger::logError(e);return(NULL)}
           )
           
           if(!is.null(out)){
             demoTemp <- c(
               genGroup = gen, 
               ageGroup = age, 
-              PersonCountAtRisk = length(p1$value),
-              PersonCountWithOutcome = round(length(p1$value)*(1-out$surv)),
+              PersonCountAtRisk = length(value),
+              PersonCountWithOutcome = round(length(value)*(1-out$surv)),
               observedRisk = 1-out$surv, 
-              averagePredictedProbability = mean(p1$value, na.rm = T),
-              StDevPredictedProbability = stats::sd(p1$value, na.rm = T)
+              averagePredictedProbability = mean(value, na.rm = T),
+              StDevPredictedProbability = stats::sd(value, na.rm = T)
               )
             
             demographicData <- rbind(demographicData, demoTemp)
diff --git a/R/DiagnosePlp.R b/R/DiagnosePlp.R
new file mode 100644
index 000000000..dee4bba82
--- /dev/null
+++ b/R/DiagnosePlp.R
@@ -0,0 +1,933 @@
+# @file Diagnostics.R
+#
+# Copyright 2021 Observational Health Data Sciences and Informatics
+#
+# This file is part of PatientLevelPrediction
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#' Run a list of predictions diagnoses
+#'
+#' @details
+#' This function will run all specified prediction design diagnoses as defined using . 
+#'
+#' @param databaseDetails               The database settings created using \code{createDatabaseDetails()}
+#' @param modelDesignList                A list of model designs created using \code{createModelDesign()}
+#' @param cohortDefinitions               A list of cohort definitions for the target and outcome cohorts
+#' @param logSettings                    The setting spexcifying the logging for the analyses created using \code{createLogSettings()}
+#' @param saveDirectory                   Name of the folder where all the outputs will written to.
+#' 
+#' @return
+#' A data frame with the following columns: \tabular{ll}{ \verb{analysisId} \tab The unique identifier
+#' for a set of analysis choices.\cr \verb{targetId} \tab The ID of the target cohort populations.\cr
+#' \verb{outcomeId} \tab The ID of the outcomeId.\cr \verb{dataLocation} \tab The location where the plpData was saved 
+#'  \cr \verb{the settings ids} \tab The ids for all other settings used for model development.\cr }
+#'
+#' @export
+diagnoseMultiplePlp <- function(
+  databaseDetails = createDatabaseDetails(),
+  modelDesignList = list(
+    createModelDesign(targetId = 1, outcomeId = 2, modelSettings = setLassoLogisticRegression()), 
+    createModelDesign(targetId = 1, outcomeId = 3, modelSettings = setLassoLogisticRegression())
+  ),
+  cohortDefinitions = NULL,
+  logSettings = createLogSettings(
+    verbosity = "DEBUG", 
+    timeStamp = T, 
+    logName = "diagnosePlp Log"
+  ),
+  saveDirectory = getwd()
+){
+  
+  #input checks
+  checkIsClass(databaseDetails, c('databaseDetails'))
+  checkIsClass(modelDesignList, c('list', 'modelDesign'))
+  checkIsClass(logSettings, 'logSettings')
+  checkIsClass(saveDirectory, 'character')
+  if(!dir.exists(saveDirectory)){
+    dir.create(saveDirectory, recursive = T)
+  }
+  
+  if(is.null(cohortDefinitions)){
+    
+    cohortIds <- unlist(
+      lapply(
+        X = 1:length(modelDesignList), 
+        FUN = function(i){
+          c(
+            modelDesignList[[i]]$targetId,
+            modelDesignList[[i]]$outcomeId
+          )
+        }
+      )
+    )
+    
+    cohortDefinitions <- lapply(
+      X = cohortIds, 
+      FUN = function(x){
+        list(
+          id = x, 
+          name = paste0('Cohort: ', x)
+        )
+      }
+    )
+    
+  }
+  
+  settingstable <- convertToJson(modelDesignList,cohortDefinitions) # from runMultiplePlp.R
+  
+  if(nrow(settingstable) != length(modelDesignList)){
+    stop('Error in settingstable')
+  }
+  
+  # save the settings: TODO fix
+  utils::write.csv(
+    x = settingstable %>% dplyr::select(
+      "analysisId",
+      "targetId", 
+      "targetName",
+      "outcomeId", 
+      "outcomeName",
+      "dataLocation"
+    ), 
+    file.path(saveDirectory,'settings.csv'), 
+    row.names = F
+  )
+
+  # group the outcomeIds per combination of data extraction settings
+  dataSettings <- settingstable %>% 
+    dplyr::group_by(
+      .data$targetId,
+      .data$covariateSettings,
+      .data$restrictPlpDataSettings,
+      .data$dataLocation
+    ) %>% 
+    dplyr::summarise(
+      outcomeIds = paste(unique(.data$outcomeId), collapse = ',')
+    )
+  
+  # extract data
+  for(i in 1:nrow(as.data.frame(dataSettings))){
+    dataExists <- length(dir(file.path(saveDirectory, dataSettings$dataLocation[i])))>0
+    if(!dataExists){
+      ParallelLogger::logInfo(paste('Extracting data for cohort', dataSettings$targetId[i], 'to', file.path(saveDirectory, dataSettings$dataLocation[i])))
+      
+      databaseDetails$targetId <- dataSettings$targetId[i]
+      databaseDetails$outcomeIds <- strsplit(dataSettings$outcomeIds[i], ',')[[1]]
+      
+      plpDataSettings <- list(
+        databaseDetails = databaseDetails,
+        covariateSettings = ParallelLogger::convertJsonToSettings(dataSettings$covariateSettings[i]),
+        restrictPlpDataSettings = ParallelLogger::convertJsonToSettings(dataSettings$restrictPlpDataSettings[i])
+      )
+      
+      plpData <- tryCatch(
+        {do.call(getPlpData, plpDataSettings)},
+        error = function(e){ParallelLogger::logInfo(e); return(NULL)}
+      )
+      if(!is.null(plpData)){
+        savePlpData(plpData, file.path(saveDirectory, dataSettings$dataLocation[i]))
+      }
+    } else{
+      ParallelLogger::logInfo(paste('Data for cohort', dataSettings$targetId[i], 'exists at', file.path(saveDirectory, dataSettings$dataLocation[i])))
+    }
+  }
+  
+  # diagnosePlp
+  for(i in 1:nrow(as.data.frame(settingstable))){
+    modelDesign <- modelDesignList[[i]]
+    settings <- settingstable[i,] # just the data locations?
+    
+    dataExists <- length(dir(file.path(saveDirectory, settings$dataLocation)))>0
+    
+    if(dataExists){
+      plpData <- PatientLevelPrediction::loadPlpData(file.path(saveDirectory, settings$dataLocation))
+      
+      diagnoseExists <- file.exists(file.path(saveDirectory, settings$analysisId, 'diagnosePlp.rds'))
+      if(!diagnoseExists){
+        
+        diagnosePlpSettings <- list(
+          plpData = plpData,
+          outcomeId = modelDesign$outcomeId,
+          analysisId = settings$analysisId,
+          populationSettings = modelDesign$populationSettings,
+          splitSettings = modelDesign$splitSettings,
+          sampleSettings = modelDesign$sampleSettings,
+          featureEngineeringSettings = modelDesign$featureEngineeringSettings,
+          preprocessSettings = modelDesign$preprocessSettings,
+          modelSettings = modelDesign$modelSettings,
+          logSettings = logSettings,
+          saveDirectory = saveDirectory
+        )
+        
+        result <- tryCatch(
+          {do.call(diagnosePlp, diagnosePlpSettings)},
+          error = function(e){ParallelLogger::logInfo(e); return(NULL)}
+        )
+      } else{
+        ParallelLogger::logInfo(paste('Diagnosis ', settings$analysisId, 'exists at', file.path(saveDirectory, settings$analysisId)))
+      }
+    } # end run per setting
+    
+  }
+  return(invisible(settingstable))
+}
+
+
+#' diagnostic - Investigates the prediction problem settings - use before training a model
+#'
+#' @description
+#' This function runs a set of prediction diagnoses to help pick a suitable T, O, TAR and determine 
+#' whether the prediction problem is worth executing.
+#' 
+#' @details
+#' Users can define set of Ts, Os, databases and population settings.  A list of data.frames containing details such as
+#' follow-up time distribution, time-to-event information, characteriszation details, time from last prior event, 
+#' observation time distribution. 
+#'
+#' @param plpData                    An object of type \code{plpData} - the patient level prediction
+#'                                   data extracted from the CDM.  Can also include an initial population as 
+#'                                   plpData$popualtion.
+#' @param outcomeId                  (integer) The ID of the outcome.                                       
+#' @param analysisId                 (integer) Identifier for the analysis. It is used to create, e.g., the result folder. Default is a timestamp.
+#' @param populationSettings         An object of type \code{populationSettings} created using \code{createStudyPopulationSettings} that
+#'                                   specifies how the data class labels are defined and addition any exclusions to apply to the 
+#'                                   plpData cohort
+#' @param splitSettings              An object of type \code{splitSettings} that specifies how to split the data into train/validation/test.  
+#'                                   The default settings can be created using \code{createDefaultSplitSetting}.                               
+#' @param sampleSettings             An object of type \code{sampleSettings} that specifies any under/over sampling to be done.
+#'                                   The default is none.
+#' @param featureEngineeringSettings An object of \code{featureEngineeringSettings} specifying any feature engineering to be learned (using the train data)                                                        
+#' @param preprocessSettings         An object of \code{preprocessSettings}. This setting specifies the minimum fraction of 
+#'                                   target population who must have a covariate for it to be included in the model training                            
+#'                                   and whether to normalise the covariates before training  
+#' @param modelSettings              An object of class \code{modelSettings} created using one of the function:
+#'                                         \itemize{
+#'                                         \item setLassoLogisticRegression() A lasso logistic regression model
+#'                                         \item setGradientBoostingMachine() A gradient boosting machine
+#'                                         \item setAdaBoost() An ada boost model
+#'                                         \item setRandomForest() A random forest model
+#'                                         \item setDecisionTree() A decision tree model
+#'                                         \item setKNN() A KNN model
+#'                                         
+#'                                         } 
+#' @param logSettings           An object of \code{logSettings} created using \code{createLogSettings} 
+#'                              specifying how the logging is done                                                                            
+#' @param saveDirectory         The path to the directory where the results will be saved (if NULL uses working directory)
+#' 
+#' @return
+#' An object containing the model or location where the model is save, the data selection settings, the preprocessing
+#' and training settings as well as various performance measures obtained by the model.
+#'
+#' \item{distribution}{list for each O of a data.frame containing: i) Time to observation end distribution, ii) Time from observation start distribution, iii) Time to event distribution and iv) Time from last prior event to index distribution (only for patients in T who have O before index) }
+#' \item{incident}{list for each O of incidence of O in T during TAR}
+#' \item{characterization}{list for each O of Characterization of T, TnO, Tn~O}
+#'
+#'
+#' @export
+#' @examples
+#' \dontrun{
+#' #******** EXAMPLE 1 ********* 
+#' } 
+diagnosePlp <- function(
+  plpData = NULL,
+  outcomeId,
+  analysisId,
+  populationSettings,
+  splitSettings = createDefaultSplitSetting(),
+  sampleSettings = createSampleSettings(), # default none
+  saveDirectory = NULL,
+  featureEngineeringSettings = createFeatureEngineeringSettings(), # default none
+  modelSettings = setLassoLogisticRegression(), # default to logistic regression
+  logSettings =  createLogSettings(
+    verbosity = 'DEBUG',
+    timeStamp = T,
+    logName = 'diagnosePlp Log'
+  ),
+  preprocessSettings = createPreprocessSettings()
+){
+  
+  # start log 
+  analysisPath <- file.path(saveDirectory, analysisId)
+  logSettings$saveDirectory <- analysisPath
+  logSettings$logFileName <- 'plpLog'
+  logger <- do.call(createLog,logSettings)
+  ParallelLogger::registerLogger(logger)
+  on.exit(closeLog(logger))
+  
+  participantsDiag <- probastParticipants(
+    plpData, 
+    outcomeId = outcomeId,
+    populationSettings = populationSettings
+  )
+  
+  predictorDiag <- probastPredictors(
+    plpData, 
+    outcomeId = outcomeId,
+    populationSettings = populationSettings
+  )
+  
+  outcomeDiag <- probastOutcome(
+    plpData, 
+    outcomeId = outcomeId,
+    populationSettings = populationSettings
+  )
+  
+  designDiag <- probastDesign(
+    plpData, 
+    outcomeId = outcomeId,
+    populationSettings = populationSettings
+  )
+  
+  # Question: what about 
+  # splitSettings, sampleSettings, 
+  # FeatureEngineeringSettings, modelSettings
+  
+  result <- list(
+    summary = rbind(
+      participantsDiag$diagnosticParticipantsAggregate,
+      predictorDiag$diagnosticPredictorsAggregate,
+      outcomeDiag$diagnosticOutcomeAggregate,
+      designDiag$diagnosticDesignAggregate
+    ),
+    participants = participantsDiag$diagnosticParticipantsFull,
+    predictors = predictorDiag$diagnosticPredictorsFull,
+    outcomes = outcomeDiag$diagnosticOutcomeFull,
+    designs = designDiag$diagnosticDesignFull,
+    modelDesign = PatientLevelPrediction::createModelDesign(
+      targetId = attr(plpData$cohorts, "metaData")$targetId,
+      outcomeId = outcomeId,
+      restrictPlpDataSettings = plpData$metaData$restrictPlpDataSettings,
+      covariateSettings = plpData$metaData$covariateSettings,
+      populationSettings = populationSettings,
+      featureEngineeringSettings = featureEngineeringSettings,
+      preprocessSettings = preprocessSettings,
+      modelSettings = modelSettings,
+      splitSettings = splitSettings,
+      sampleSettings = sampleSettings
+    ),
+    databaseSchema = plpData$metaData$databaseDetails$cdmDatabaseSchema,
+    databaseId = plpData$metaData$databaseDetails$cdmDatabaseId
+  )
+  
+  class(result) <- 'diagnosePlp'
+  
+  if(!is.null(saveDirectory)){
+    if(!dir.exists(file.path(saveDirectory, analysisId))){
+      dir.create(file.path(saveDirectory, analysisId), recursive = T)
+    }
+    saveLocation <- file.path(saveDirectory, analysisId, 'diagnosePlp.rds')
+    ParallelLogger::logInfo(paste0('Saving diagnosePlp to ', saveLocation))
+    saveRDS(result, saveLocation)
+  }
+  
+  return(result)
+}
+
+probastDesign <- function(
+  plpData, 
+  outcomeId,
+  populationSettings
+){
+  
+  diagnosticAggregate <- c()
+  diagnosticFull <- c()
+  
+  population <- PatientLevelPrediction::createStudyPopulation(
+    plpData = plpData, 
+    outcomeId = outcomeId, 
+    populationSettings = populationSettings
+  )
+  
+  probastId <- '4.1'
+  if(min(sum(population$outcomeCount > 0),nrow(population) - sum(population$outcomeCount > 0)) >= 1000){
+    diagnosticAggregate <- rbind(
+      diagnosticAggregate,
+      c(probastId, 'Pass')
+    )
+  }else if((min(sum(population$outcomeCount > 0),nrow(population) - sum(population$outcomeCount > 0)) >= 100)){
+    diagnosticAggregate <- rbind(
+      diagnosticAggregate,
+      c(probastId, 'Unknown')
+    )
+  } else{
+    diagnosticAggregate <- rbind(
+      diagnosticAggregate,
+      c(probastId, 'Fail')
+    )
+  }
+  
+  if(!is.null(dim(diagnosticAggregate))){
+    diagnosticAggregate <- as.data.frame(diagnosticAggregate) %>% 
+      dplyr::mutate_if(is.factor, as.character)
+    colnames(diagnosticAggregate) <- c('probastId','resultValue')
+  }
+  
+  return(
+    list
+    (
+      diagnosticDesignFull = diagnosticFull,
+      diagnosticDesignAggregate = diagnosticAggregate
+    )
+  )
+  
+  
+}
+
+probastParticipants <- function(
+  plpData, 
+  outcomeId,
+  populationSettings
+){
+  
+  diagnosticAggregate <- c()
+  diagnosticFull <- c()
+  
+  # true due to plp
+  probastId <- '1.1'
+  diagnosticAggregate <- rbind(
+    diagnosticAggregate,
+    c(probastId, 'Pass')
+  )
+  
+  # appropriate inclusions
+  ## 1.2.1 min prior observation
+  if(populationSettings$washoutPeriod != 0 ){
+    var <- 'washoutPeriod'
+    probastId <- '1.2.1'
+    
+    result <- getDiagnostic(
+      probastId,
+      plpData, 
+      outcomeId, 
+      populationSettings, 
+      var, 
+      0  
+    )
+    
+    diagnosticAggregate <- rbind(
+      diagnosticAggregate,
+      result$diagnosticAggregate
+    )
+    
+    diagnosticFull <- rbind(
+      diagnosticFull,
+      result$diagnosticFull
+    )
+    
+  }
+  
+  ## 1.2.2 min time-at-risk
+  if(populationSettings$requireTimeAtRisk & populationSettings$minTimeAtRisk > 0){
+    probastId <- '1.2.2'
+    var <- 'minTimeAtRisk'
+    
+    result <- getDiagnostic(
+      probastId,
+      plpData, 
+      outcomeId, 
+      populationSettings, 
+      var, 
+      0  
+    )
+    
+    diagnosticAggregate <- rbind(
+      diagnosticAggregate,
+      result$diagnosticAggregate
+    )
+    
+    diagnosticFull <- rbind(
+      diagnosticFull,
+      result$diagnosticFull
+    )
+    
+  }
+  
+  
+  ## 1.2.3 first exposure only
+  if(populationSettings$firstExposureOnly){
+    probastId <- '1.2.3'
+    var <- 'firstExposureOnly'
+    default <- F
+    
+    result <- getDiagnostic(
+      probastId,
+      plpData, 
+      outcomeId, 
+      populationSettings, 
+      var, 
+      default  
+    )
+    
+    diagnosticAggregate <- rbind(
+      diagnosticAggregate,
+      result$diagnosticAggregate
+    )
+    
+    diagnosticFull <- rbind(
+      diagnosticFull,
+      result$diagnosticFull
+    )
+    
+  }
+  
+  ## 1.2.4 prior observation 
+  if(populationSettings$removeSubjectsWithPriorOutcome & populationSettings$priorOutcomeLookback > 0){
+    probastId <- '1.2.4'
+    var <- 'priorOutcomeLookback'
+    default <- 0
+    
+    result <- getDiagnostic(
+      probastId,
+      plpData, 
+      outcomeId, 
+      populationSettings, 
+      var, 
+      default  
+    )
+    
+    diagnosticAggregate <- rbind(
+      diagnosticAggregate,
+      result$diagnosticAggregate
+    )
+    
+    diagnosticFull <- rbind(
+      diagnosticFull,
+      result$diagnosticFull
+    )
+    
+  }
+  
+  if(!is.null(dim(diagnosticAggregate))){
+    diagnosticAggregate <- as.data.frame(diagnosticAggregate) %>% 
+      dplyr::mutate_if(is.factor, as.character)
+    colnames(diagnosticAggregate) <- c('probastId','resultValue')
+  }
+  
+  return(
+    list
+    (
+      diagnosticParticipantsFull = diagnosticFull,
+      diagnosticParticipantsAggregate = diagnosticAggregate
+    )
+  )
+}
+
+
+getMaxEndDaysFromCovariates <- function(covariateSettings){
+  
+  if(inherits(covariateSettings, 'covariateSettings')){
+    covariateSettings <- list(covariateSettings)
+  }
+  
+  vals <- unlist(lapply(covariateSettings, function(x){x$endDays}))
+  
+  if(length(vals) == 0){
+    return(0)
+  } else{
+    return(max(vals))
+  }
+}
+
+
+
+probastPredictors <- function(
+  plpData, 
+  outcomeId,
+  populationSettings
+){
+  
+  diagnosticAggregate <- c()
+  diagnosticFull <- c()
+  
+  # true due to plp
+  probastId <- '2.1'
+  diagnosticAggregate <- rbind(
+    diagnosticAggregate,
+    c(probastId, 'Pass')
+  )
+  
+  # 2.2.1
+  # cov end date < tar_start
+  # covariate + outcome correlation; km of outcome (close to index or not)?
+  probastId <- '2.2'
+  if(populationSettings$startAnchor == 'cohort start'){
+    if(populationSettings$riskWindowStart > getMaxEndDaysFromCovariates(plpData$metaData$covariateSettings)){
+      diagnosticAggregate <- rbind(
+        diagnosticAggregate,
+        c(probastId, 'Pass')
+      )
+    } else{
+      diagnosticAggregate <- rbind(
+        diagnosticAggregate,
+        c(probastId, 'Fail')
+      )
+    }
+  } else{
+    diagnosticAggregate <- rbind(
+      diagnosticAggregate,
+      c(probastId, 'Unknown')
+    )
+  }
+  
+  # KM of outcome
+  populationSettingsFull <- populationSettings
+  populationSettingsFull$riskWindowEnd <- 10*365
+  
+  population <- PatientLevelPrediction::createStudyPopulation(
+    plpData = plpData, 
+    outcomeId = outcomeId, 
+    populationSettings = populationSettings
+  )
+  
+  populationFull <- PatientLevelPrediction::createStudyPopulation(
+    plpData = plpData, 
+    outcomeId = outcomeId, 
+    populationSettings = populationSettingsFull
+  )
+  
+  #dayOfEvent, outcomeAtTime, observedAtStartOfDay
+  kmObservation <- population %>% 
+    dplyr::group_by(.data$daysToEvent) %>% 
+    dplyr::summarise(
+      outcomeAtTime = sum(!is.na(.data$daysToEvent))
+    )
+  
+  kmObservation$observedAtStartOfDay <-  unlist(
+    lapply(
+      kmObservation$daysToEvent,
+      function(x){
+        population %>% 
+          dplyr::filter(.data$survivalTime >= x) %>%
+          dplyr::tally() %>% 
+          dplyr::select("n")
+      }
+    )
+  )
+  
+  kmObservation$probastId <- probastId
+  kmObservation$inputType <- 'populationSettings'
+  
+  kmObservationFull <- populationFull %>% 
+    dplyr::group_by(.data$daysToEvent) %>% 
+    dplyr::summarise(
+      outcomeAtTime = sum(!is.na(.data$daysToEvent))
+    )
+  
+  kmObservationFull$observedAtStartOfDay <-  unlist(
+    lapply(
+      kmObservationFull$daysToEvent,
+      function(x){
+        populationFull %>% 
+          dplyr::filter(.data$survivalTime >= x) %>%
+          dplyr::tally() %>% 
+          dplyr::select("n")
+      }
+    )
+  )
+  
+  kmObservationFull$probastId <- probastId
+  kmObservationFull$inputType <- '10-year'
+  
+  diagnosticFull <- rbind(kmObservation, kmObservationFull)
+  
+  
+  # 2.3.1
+  # cov end_date <=0
+  probastId <- '2.3'
+  if(getMaxEndDaysFromCovariates(plpData$metaData$covariateSettings) <= 0){
+    
+    if(getMaxEndDaysFromCovariates(plpData$metaData$covariateSettings) < 0){
+      diagnosticAggregate <- rbind(
+        diagnosticAggregate,
+        c(probastId, 'Pass')
+      )
+    }
+    else{
+      diagnosticAggregate <- rbind(
+        diagnosticAggregate,
+        c(probastId, 'Unknown')
+      )
+    }
+  } else{
+    diagnosticAggregate <- rbind(
+      diagnosticAggregate,
+      c(probastId, 'Fail')
+    )
+  }
+  
+  if(!is.null(dim(diagnosticAggregate))){
+    diagnosticAggregate <- as.data.frame(diagnosticAggregate) %>% 
+      dplyr::mutate_if(is.factor, as.character)
+    colnames(diagnosticAggregate) <- c('probastId','resultValue')
+  }
+  
+  return(
+    list
+    (
+      diagnosticPredictorsFull = diagnosticFull,
+      diagnosticPredictorsAggregate = diagnosticAggregate
+    )
+  )
+  
+}
+
+
+
+probastOutcome <- function(
+  plpData, 
+  outcomeId,
+  populationSettings
+){
+  
+  diagnosticAggregate <- c()
+  diagnosticFull <- c()
+  
+  # true due to plp
+  probastId <- '3.4'
+  diagnosticAggregate <- rbind(
+    diagnosticAggregate,
+    c(probastId, 'Pass')
+  )
+  
+  # 3.5 - check the outcome definition doesn't use things before index?
+  
+  # 3.6 - check tar after covariate end_days
+  probastId <- '3.6'
+  if(populationSettings$startAnchor == 'cohort start'){
+    if(populationSettings$riskWindowStart > getMaxEndDaysFromCovariates(plpData$metaData$covariateSettings)){
+      diagnosticAggregate <- rbind(
+        diagnosticAggregate,
+        c(probastId, 'Pass')
+      )
+    } else{
+      diagnosticAggregate <- rbind(
+        diagnosticAggregate,
+        c(probastId, 'Fail')
+      )
+    }
+  } else{
+    diagnosticAggregate <- rbind(
+      diagnosticAggregate,
+      c(probastId, 'Unknown')
+    )
+  }
+  
+  # 3.1.1 - check the outcome rate per gender/age/index year
+  probastId <- '3.1.1'
+  
+  # all cohort vs pop
+  pop <- PatientLevelPrediction::createStudyPopulation(
+    plpData = plpData, 
+    outcomeId = outcomeId, 
+    populationSettings = populationSettings
+  )
+  
+  popSum <- getOutcomeSummary(
+    type = 'population',
+    population = pop
+  )
+  
+  cohort <- PatientLevelPrediction::createStudyPopulation(
+    plpData = plpData, 
+    outcomeId = outcomeId, 
+    populationSettings = PatientLevelPrediction::createStudyPopulationSettings(
+      includeAllOutcomes = F, 
+      firstExposureOnly = F, 
+      washoutPeriod = 0, 
+      removeSubjectsWithPriorOutcome = F, 
+      priorOutcomeLookback = 0, 
+      requireTimeAtRisk = F, 
+      minTimeAtRisk = 0, 
+      riskWindowStart = populationSettings$riskWindowStart, 
+      startAnchor = populationSettings$startAnchor, 
+      riskWindowEnd = populationSettings$riskWindowEnd, 
+      endAnchor = populationSettings$endAnchor
+    )
+  )
+  
+  cohortSum <- getOutcomeSummary(
+    type = 'cohort',
+    population = cohort
+  )
+  
+  
+  
+  #dayOfEvent, outcomeAtTime, observedAtStartOfDay
+  diagnosticFull <- rbind(
+    do.call(rbind, popSum),
+    do.call(rbind, cohortSum)
+  )
+  
+  if(!is.null(dim(diagnosticAggregate))){
+    diagnosticAggregate <- as.data.frame(diagnosticAggregate) %>% 
+      dplyr::mutate_if(is.factor, as.character)
+    colnames(diagnosticAggregate) <- c('probastId','resultValue')
+  }
+  
+  return(
+    list
+    (
+      diagnosticOutcomeFull = diagnosticFull,
+      diagnosticOutcomeAggregate = diagnosticAggregate
+    )
+  )
+  
+}
+
+getOutcomeSummary <- function(
+  type = 'population',
+  population
+){
+  
+  res <- list()
+  length(res) <- 4
+  probastId <- '3'
+  
+  res[[1]] <- population %>% 
+    dplyr::group_by(.data$ageYear) %>%
+    dplyr::summarise(outcomePercent = sum(.data$outcomeCount>0)/length(.data$outcomeCount)) %>%
+    dplyr::mutate(
+      probastId = probastId,
+      aggregation = 'age',
+      inputType = type
+    ) %>% 
+    dplyr::rename(xvalue = "ageYear")
+  
+  res[[2]] <-  population %>% 
+    dplyr::group_by(.data$gender) %>%
+    dplyr::summarise(outcomePercent = sum(.data$outcomeCount>0)/length(.data$outcomeCount)) %>%
+    dplyr::mutate(
+      probastId = probastId,
+      aggregation = 'gender',
+      inputType = type
+    )%>% 
+    dplyr::rename(xvalue = "gender")
+  
+  res[[3]] <- population %>% 
+    dplyr::mutate(
+      year = substring(.data$cohortStartDate,1,4)
+    ) %>%
+    dplyr::group_by(.data$year) %>%
+    dplyr::summarise(outcomePercent = sum(.data$outcomeCount>0)/length(.data$outcomeCount)) %>%
+    dplyr::mutate(
+      probastId = probastId,
+      aggregation = 'year',
+      inputType = type
+    ) %>% 
+    dplyr::rename(xvalue = "year")
+  
+  res[[4]] <- population %>% 
+    dplyr::mutate(
+      year = substring(.data$cohortStartDate,6,7)
+    ) %>%
+    dplyr::group_by(.data$year) %>%
+    dplyr::summarise(outcomePercent = sum(.data$outcomeCount>0)/length(.data$outcomeCount)) %>%
+    dplyr::mutate(
+      probastId = probastId,
+      aggregation = 'month',
+      inputType = type
+    ) %>% 
+    dplyr::rename(xvalue = "year")
+  
+  return(res)
+}
+
+cos_sim <- function(a,b) 
+{
+  return( sum(a*b)/sqrt(sum(a^2)*sum(b^2)) )
+} 
+
+getDiagnostic <- function(
+  probastId,
+  plpData, 
+  outcomeId,
+  populationSettings, 
+  var,
+  defaultValue = 0
+){
+  ParallelLogger::logInfo(paste0('Diagnosing impact of ',var,' in populationSettings'))
+  
+  populationSettingsCheck <- populationSettings
+  populationSettingsCheck[var] <- defaultValue
+  
+  pop <- PatientLevelPrediction::createStudyPopulation(
+    plpData = plpData, 
+    outcomeId = outcomeId, 
+    populationSettings = populationSettings
+  )
+  
+  popCheck <- PatientLevelPrediction::createStudyPopulation(
+    plpData = plpData, 
+    outcomeId = outcomeId, 
+    populationSettings = populationSettingsCheck 
+  )
+  
+  #compare the populations:
+  diag <- rbind(
+    data.frame(
+      probastId = probastId,
+      design = paste0(var,': ', defaultValue),
+      metric = c(
+        'N', 'outcomePercent', 'minAge',
+        'meanAge', 'medianAge', 'maxAge',
+        'malePercent'
+      ),
+      value = c(
+        nrow(popCheck), sum(popCheck$outcomeCount>0)/nrow(popCheck)*100, min(popCheck$ageYear),
+        mean(popCheck$ageYear), stats::median(popCheck$ageYear), max(popCheck$ageYear),
+        sum(popCheck$gender == 8507)/nrow(popCheck)*100
+      )
+    ),
+    data.frame(
+      probastId = probastId,
+      design = paste0(var,': ',populationSettings[var]),
+      metric = c(
+        'N', 'outcomePercent', 'minAge',
+        'meanAge', 'medianAge', 'maxAge',
+        'malePercent'
+      ),
+      value = c(
+        nrow(pop), sum(pop$outcomeCount>0)/nrow(pop)*100, min(pop$ageYear),
+        mean(pop$ageYear), stats::median(pop$ageYear), max(pop$ageYear),
+        sum(pop$gender == 8507)/nrow(pop)*100
+      )
+    )
+  )
+  
+  diagSim <- cos_sim(
+    diag %>% 
+      dplyr::filter(.data$design == unique(diag$design)[1]) %>%
+      dplyr::filter(.data$metric != 'N') %>%
+      dplyr::arrange(.data$metric) %>%
+      dplyr::select("value")
+    , 
+    diag %>% 
+      dplyr::filter(.data$design == unique(diag$design)[2]) %>%
+      dplyr::filter(.data$metric != 'N') %>%
+      dplyr::arrange(.data$metric) %>%
+      dplyr::select("value")
+  )
+  
+  
+  return(
+    list(
+      diagnosticAggregate = c(
+        probastId, 
+        diagSim
+      ),
+      diagnosticFull = diag
+    )
+  )
+}
+  
diff --git a/R/Diagnostics.R b/R/Diagnostics.R
deleted file mode 100644
index 5be89f0f5..000000000
--- a/R/Diagnostics.R
+++ /dev/null
@@ -1,501 +0,0 @@
-# @file Diagnostics.R
-#
-# Copyright 2021 Observational Health Data Sciences and Informatics
-#
-# This file is part of PatientLevelPrediction
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#' diagnostic - Investigates the prediction problem settings - use before training a model
-#'
-#' @description
-#' This function runs a set of prediction diagnoses to help pick a suitable T, O, TAR and determine 
-#' whether the prediction problem is worth executing.
-#' 
-#' @details
-#' Users can define set of Ts, Os, databases and population settings.  A list of data.frames containing details such as
-#' follow-up time distribution, time-to-event information, characteriszation details, time from last prior event, 
-#' observation time distribution. 
-#'
-#' @param plpData                      The data object to do the diagnostic on - if NULL you need to specify the connection settings below
-#' @param cdmDatabaseName             The name of the database being diagnosed
-#' @param cohortName                  Name of the target cohort
-#' @param outcomeNames                Vector of outcome names
-#' @param databaseDetails            (only used is plpData is NULL) The database details created using \code{createDatabaseDetails}
-#' @param restrictPlpDataSettings    (only used is plpData is NULL) The restrictPlpDataSettings created using \code{createRestrictPlpDataSettings}
-#' @param populationSettings         The population setting details created using \code{createPopulationSettings}
-#' @param outputFolder           Location to save results for shiny app
-#' @param minCellCount           The minimum count that will be displayed
-#' 
-#' @return
-#' An object containing the model or location where the model is save, the data selection settings, the preprocessing
-#' and training settings as well as various performance measures obtained by the model.
-#'
-#' \item{distribution}{list for each O of a data.frame containing: i) Time to observation end distribution, ii) Time from observation start distribution, iii) Time to event distribution and iv) Time from last prior event to index distribution (only for patients in T who have O before index) }
-#' \item{incident}{list for each O of incidence of O in T during TAR}
-#' \item{characterization}{list for each O of Characterization of T, TnO, Tn~O}
-#'
-#'
-#' @export
-#' @examples
-#' \dontrun{
-#' #******** EXAMPLE 1 ********* 
-#' } 
-diagnostic <- function(
-  plpData = NULL,
-  cdmDatabaseName = 'none',
-  cohortName,
-  outcomeNames,
-  databaseDetails,
-  restrictPlpDataSettings,
-  populationSettings,
-  outputFolder = NULL,
-  minCellCount = 5
-){
-  
-  if(is.null(plpData)){
-    checkIsClass(databaseDetails, 'databaseDetails')
-    cdmDatabaseName <- attr(databaseDetails, 'cdmDatabaseName')
-    checkIsClass(restrictPlpDataSettings, 'restrictPlpDataSettings')
-  }
-  
-  if(class(populationSettings) != 'list'){
-    populationSettings <- list(populationSettings)
-  }
-  lapply(populationSettings, function(x) checkIsClass(x, 'populationSettings'))
-  
-  if(!is.null(outputFolder)){
-    if(!dir.exists(file.path(outputFolder))){
-      dir.create(file.path(outputFolder), recursive = T)
-    }
-  }
-  
-  if(!is.null(plpData)){
-    cohortId <- unique(plpData$cohorts$cohortId)
-    outcomeIds <- unique(plpData$outcomes$outcomeId)
-  } else{
-    cohortId <- databaseDetails$cohortId
-    outcomeIds <- databaseDetails$outcomeIds
-  }
-  
-  #create cohort names csv:
-  if(file.exists(file.path(outputFolder,'namesdetails.csv'))){
-    cohortNames <- utils::read.csv(file.path(outputFolder,'namesdetails.csv'))
-    
-    newNames <- data.frame(ids = c(cohortId,outcomeIds), 
-                           names = c(cohortName,outcomeNames))
-    
-    newNames<- newNames[!newNames$ids%in%cohortNames$ids,]
-    if(length(newNames$ids)>0){
-      cohortNames <- rbind(cohortNames, newNames)
-    }
-    
-  } else {
-    cohortNames <- data.frame(ids = c(cohortId,outcomeIds), 
-                           names = c(cohortName,outcomeNames))
-  }
-  ParallelLogger::logInfo('Saving cohort names to csv')
-  utils::write.csv(cohortNames, file.path(outputFolder,'namesdetails.csv'), row.names = F)
-  
-  #create settings:
-  if(file.exists(file.path(outputFolder,'settings.csv'))){
-    settings <- utils::read.csv(file.path(outputFolder,'settings.csv'))
-  } else{
-    settings <- c()
-  }
-  maxAnalysis <-  ifelse(is.null(settings$analysisId), 0, max(settings$analysisId))
-  for(i in 1:length(populationSettings)){
-    for( j in 1:length(outcomeIds)){
-      maxAnalysis <- maxAnalysis + 1
-      settingsTemp <- data.frame(analysisId = maxAnalysis,
-                                 cdmDatabaseName = cdmDatabaseName,
-                                 cohortId = cohortId,
-                                 outcomeId = outcomeIds[j],
-                                 riskWindowStart = populationSettings[[i]]$riskWindowStart,
-                                 startAnchor = populationSettings[[i]]$startAnchor,
-                                 riskWindowEnd = populationSettings[[i]]$riskWindowEnd,
-                                 endAnchor = populationSettings[[i]]$endAnchor
-      )
-      settings <- unique(rbind(settings, settingsTemp))
-    }
-  }
-  
-  ParallelLogger::logInfo('Saving settings to csv')
-  utils::write.csv(settings, file.path(outputFolder,'settings.csv'), row.names = F)
-  
-  
-  if(is.null(plpData)){
-    # get outcome and cohort data - dont need covariates
-    
-    ParallelLogger::logInfo('Extracting data')
-    data <- do.call(
-      getPlpData, 
-      list(
-        databaseDetails = databaseDetails, 
-        covariateSettings = FeatureExtraction::createDefaultCovariateSettings(),
-        restrictPlpDataSettings = restrictPlpDataSettings
-      )
-    )
-  } else {
-    data <- plpData
-  } 
-  
-  outcomeIds <- unique(data$outcomes$outcomeId)
-  
-  ParallelLogger::logInfo('Calculating distributions')
-  distribution <- getDistribution(cohort = data$cohorts,
-                                  outcomes = data$outcomes,
-                                  outputFolder = outputFolder, 
-                                  databaseName = cdmDatabaseName)
-  
-  
-  # get survival data:
-  ParallelLogger::logInfo('Calculating survival data')
-  if(file.exists(file.path(outputFolder, 'survival.csv'))){
-    surv <- utils::read.csv(file.path(outputFolder, 'survival.csv')) 
-  } else {
-    surv <- c()
-  }
-  survTemp <- lapply(outcomeIds, function(oi) getSurvival(plpData = data, 
-                                                          outcomeId = oi,
-                                                          cohortId = cohortId, 
-                                                          cdmDatabaseName  = cdmDatabaseName ))
-  surv <- unique(rbind(surv, do.call('rbind', survTemp)))
-  if(!is.null(outputFolder)){
-    utils::write.csv(surv, file.path(outputFolder, 'survival.csv'), row.names = F)
-  }
-  
-  # do characterisation - needs TAR
-  ParallelLogger::logInfo('Calculating proportion and characterizations')
-  
-  if(file.exists(file.path(outputFolder, 'proportion.csv'))){
-    proportion <- utils::read.csv(file.path(outputFolder, 'proportion.csv')) 
-  } else {
-    proportion <- c()
-  }
-  
-  if(file.exists(file.path(outputFolder, 'characterization.csv'))){
-    characterization <- utils::read.csv(file.path(outputFolder, 'characterization.csv')) 
-  } else {
-    characterization <- c()
-  }
-  for(i in 1:length(outcomeIds)){
-    oi <- outcomeIds[i]
-    for(j in 1:length(populationSettings)){
-      
-      population <- createStudyPopulation(
-        plpData = data, 
-        outcomeId = oi, 
-        populationSettings = populationSettings[[j]]
-        )
-
-      analysisId <- getAnalysisId(
-        settings = settings, 
-        cohortId = cohortId,
-        outcomeId = oi,
-        riskWindowStart = populationSettings[[j]]$riskWindowStart, 
-        startAnchor = populationSettings[[j]]$startAnchor, 
-        riskWindowEnd = populationSettings[[j]]$riskWindowEnd, 
-        endAnchor = populationSettings[[j]]$endAnchor
-      )
-      
-      proportionTemp <- getProportions(
-        population,
-        analysisId = analysisId,
-        cdmDatabaseName = cdmDatabaseName,
-        cohortId = cohortId,
-        outcomeId = oi,
-        minCellCount = minCellCount
-      )
-     
-      proportion <- unique(rbind(proportion, proportionTemp))
-
-      characterizationTemp <- covariateSummary(
-        covariateData = plpData$covariateData, 
-        cohort = population %>% dplyr::select(.data$rowId),
-        labels = population %>% dplyr::select(.data$rowId, .data$outcomeCount)
-        )
-      
-      
-      characterizationTemp <- characterizationTemp[,c('covariateId',
-                                                      'covariateName',
-                                                      'CovariateCount',
-                                                      'WithOutcome_CovariateCount',
-                                                      'WithNoOutcome_CovariateCount',
-                                                      'WithOutcome_CovariateMean',
-                                                      'WithNoOutcome_CovariateMean')]
-      
-      characterizationTemp[is.na(characterizationTemp)] <- 0
-      
-      ind <- (characterizationTemp$CovariateCount < minCellCount)  
-      ind2 <- (characterizationTemp$WithOutcome_CovariateCount < minCellCount) | (characterizationTemp$WithNoOutcome_CovariateCount < minCellCount)
-      
-      characterizationTemp[ind,'CovariateCount'] <- -1
-      characterizationTemp[ind,'WithOutcome_CovariateCount'] <- -1
-      characterizationTemp[ind,'WithNoOutcome_CovariateCount'] <- -1
-      characterizationTemp[ind,'WithOutcome_CovariateMean'] <- -1
-      characterizationTemp[ind,'WithNoOutcome_CovariateMean'] <- -1
-      
-      characterizationTemp[ind2,'WithOutcome_CovariateCount'] <- -1
-      characterizationTemp[ind2,'WithNoOutcome_CovariateCount'] <- -1
-      characterizationTemp[ind2,'WithOutcome_CovariateMean'] <- -1
-      characterizationTemp[ind2,'WithNoOutcome_CovariateMean'] <- -1
-      
-      # add analysisId
-      characterizationTemp$analysisId <- analysisId
-      characterization <- rbind(characterization, characterizationTemp)
-    }
-    
-  }
-  
-  if(!is.null(outputFolder)){
-    utils::write.csv(proportion, file.path(outputFolder, 'proportion.csv'), row.names = F)
-    utils::write.csv(characterization, file.path(outputFolder, 'characterization.csv'), row.names = F)
-  }
-  
-  # Add all to zip file -------------------------------------------------------------------------------
-  ParallelLogger::logInfo("Adding results to zip file")
-  zipName <- file.path(outputFolder, paste0("Results_", cdmDatabaseName, ".zip"))
-  files <- list.files(outputFolder, pattern = ".*\\.csv$")
-  oldWd <- setwd(outputFolder)
-  on.exit(setwd(oldWd), add = TRUE)
-  DatabaseConnector::createZipFile(zipFile = zipName, files = files)
-  ParallelLogger::logInfo("Results are ready for sharing at: ", zipName)
-  
-  result <- list(distribution = distribution,
-                 proportion = proportion,
-                 characterization = characterization,
-                 survival = surv)
-  
-  return(result)
-}
-
-
-getSurvival <- function(plpData, outcomeId, cohortId, cdmDatabaseName ){
-
-  object <- plpData$outcomes %>% 
-    dplyr::filter(.data$outcomeId == !!outcomeId) %>%
-    dplyr::right_join(plpData$cohorts, by ='rowId') %>%
-    dplyr::group_by(.data$rowId) %>%
-    dplyr::summarise(daysToObsEnd = min(.data$daysToObsEnd),
-                     daysToEvent = min(.data$daysToEvent))
-    
-    
-  object$censoredTime <- apply(object[,-1], 1, function(x) min(x, na.rm = T))
-  object$event <- 0
-  object$event[!is.na(object$daysToEvent)] <- ifelse(object$event[!is.na(object$daysToEvent)] <= object$censoredTime[!is.na(object$daysToEvent)], 1,0)
-  
-  
-  result <- object %>% dplyr::group_by(.data$censoredTime) %>%
-    dplyr::summarise(events = sum(.data$event),
-                     censored = length(.data$event)-sum(.data$event))
-  
-  totalCensored <- lapply(unique(object$censoredTime), function(i) sum(result %>% dplyr::filter(.data$censoredTime <= i) %>% dplyr::select(.data$censored)))
-
-  totalCensored <- data.frame(censoredTime = unique(object$censoredTime),
-                              totalCensored = unlist(totalCensored))
-  
-  totalLost <- lapply(unique(object$censoredTime), function(i) sum(result %>% dplyr::filter(.data$censoredTime <= i) %>% dplyr::mutate(lost = .data$censored + .data$events) %>% dplyr::select(.data$lost)))
-  totalLost <- data.frame(censoredTime = unique(object$censoredTime),
-                          nAtRisk = nrow(plpData$cohorts) - unlist(totalLost))
-  
-  result <- result %>% 
-    dplyr::left_join(totalCensored, by ='censoredTime') %>% 
-    dplyr::left_join(totalLost, by ='censoredTime')
-  
-  result$outcomeId <- outcomeId
-  result$cohortId <- cohortId
-  result$cdmDatabaseName <- cdmDatabaseName 
-  return(result)
-}
-
-
-getDistribution <- function(cohort,
-                            outcomes,
-                            outputFolder = NULL, 
-                            databaseName){
-  
-  cohortId <- unique(cohort$cohortId)
-  outcomesIds <- unique(outcomes$outcomeId)
-  
-  if(file.exists(file.path(outputFolder, 'distribution.csv'))){
-    result <- utils::read.csv(file.path(outputFolder, 'distribution.csv'))
-  } else{
-    result <- c()
-  }
-  
-  for(i in  1:length(outcomesIds)){
-    oi <- outcomesIds[i]
-    ind <- outcomes$outcomeId==oi & outcomes$daysToEvent >= 0
-    if(sum(ind)>0){
-      afterC <- stats::aggregate(x = outcomes$daysToEvent[ind], 
-                          by = list(outcomes$rowId[ind]),
-                          FUN = min)
-      colnames(afterC) <- c('rowId','daysToOutcomeAfterMin')
-      } else {
-        afterC <- data.frame(rowId = -1, daysToOutcomeAfterMin = 0)
-      }
-  
-    
-    ind <- outcomes$outcomeId==oi & outcomes$daysToEvent < 0
-    if(sum(ind)>0){
-      beforeC <- stats::aggregate(x = abs(outcomes$daysToEvent[ind]), 
-                           by = list(outcomes$rowId[ind]),
-                           FUN = min)
-      colnames(beforeC) <- c('rowId','daysToOutcomeBeforeMin')
-    } else {
-      beforeC <- data.frame(rowId = -1, daysToOutcomeBeforeMin = 0)
-    }
-    
-    tempResult <- merge(cohort, afterC, by='rowId', all.x = T)
-    tempResult <- merge(tempResult, beforeC, by='rowId', all.x = T)
-
-    tempResult <- processDistribution(tempResult)
-    
-    tempResult$databaseName <- databaseName 
-    tempResult$outcomeId <- oi
-    tempResult$targetId <- cohortId
-    
-    result <- unique(rbind(result, tempResult))
- 
-  }
-  
-  if(!is.null(outputFolder)){
-    utils::write.csv(result, file.path(outputFolder, 'distribution.csv'), row.names = F)
-  }
-  
-  return(result)
-}
-
-
-processDistribution <- function(distribution){
-  
-  distribution$year <- format(as.Date(as.character(distribution$cohortStartDate), format="%Y-%m-%d"),"%Y")
-  distribution <- distribution[, c('year','daysFromObsStart','daysToObsEnd','daysToOutcomeAfterMin','daysToOutcomeBeforeMin')]
-  results <- do.call(rbind, lapply(c('all',unique(distribution$year)), function(x) getQuantiles(distribution, x) ))
-  return(results) 
-}
-
-getQuantiles <- function(distribution, year= 'all'){
-  
-  if(year != 'all'){
-    distribution <- distribution[distribution$year==year,]
-  } 
-  quants <- data.frame(
-    year = year,
-    daysFromObsStart = stats::quantile(distribution$daysFromObsStart, seq(0,1,0.01)),
-    daysToObsEnd = stats::quantile(distribution$daysToObsEnd, seq(0,1,0.01)),
-    daysToOutcomeAfterMin = stats::quantile(distribution$daysToOutcomeAfterMin[!is.na(distribution$daysToOutcomeAfterMin)], seq(0,1,0.01)),
-    daysToOutcomeBeforeMin = stats::quantile(distribution$daysToOutcomeBeforeMin[!is.na(distribution$daysToOutcomeBeforeMin)], seq(0,1,0.01))
-  )
-  heading <- data.frame(
-    year = year,
-    daysFromObsStart =length(distribution$daysFromObsStart),
-    daysToObsEnd = length(distribution$daysToObsEnd),
-    daysToOutcomeAfterMin = sum(!is.na(distribution$daysToOutcomeAfterMin)),
-    daysToOutcomeBeforeMin = sum(!is.na(distribution$daysToOutcomeBeforeMin))
-  )
-  results <- rbind(N = heading, quants)
-  results$type = rownames(results)
-  rownames(results) <- NULL
-  return(results)
-}
-
-
-
-getAnalysisId <- function(settings, 
-                          cohortId,
-                          outcomeId,
-                          riskWindowStart, 
-                          startAnchor, 
-                          riskWindowEnd, 
-                          endAnchor){
-  
-  ind <- (settings$cohortId == cohortId) & (settings$outcomeId == outcomeId) & 
-    (settings$riskWindowStart == riskWindowStart) & (settings$riskWindowEnd == riskWindowEnd) &
-      (settings$startAnchor == startAnchor) &  (settings$endAnchor == endAnchor)
-      if(sum(ind)==0){
-        writeLines(paste('cohortId:',cohortId, '-outcomeId:',outcomeId, 
-                         '-riskWindowStart:', riskWindowStart, '-riskWindowEnd:', riskWindowEnd,
-                         '-startAnchor:', startAnchor, '-endAnchor:',endAnchor))
-        print(settings)
-        stop('No analysis id found for the settings')
-      } else {
-        return(settings$analysisId[ind][1])
-      }
-}
-
-
-
-getProportions <- function(population, 
-                           analysisId,
-                           cdmDatabaseName,
-                           cohortId,
-                           outcomeId,
-                           minCellCount = NULL){
-  
-  details <- attr(population, 'metaData')$populationSettings
-  
-  TAR <- paste0(details$startAnchor, ' + ', details$riskWindowStart, ' days - ',
-                details$endAnchor, ' + ', details$riskWindowEnd, ' days')
-  
-  
-  result <- population %>% dplyr::mutate(ageGroup = paste0(floor(.data$ageYear/5)*5 ,' - ', (floor(.data$ageYear/5)+1)*5-1 ),
-                               year = substring(.data$cohortStartDate,1,4)) %>%
-    dplyr::group_by(.data$year, .data$ageGroup, .data$gender) %>%
-    dplyr::summarize(N = length(.data$rowId), 
-                     O = sum(.data$outcomeCount>0)
-                     ) %>% 
-    dplyr::select(.data$year, .data$ageGroup, .data$gender, .data$N, .data$O) 
-  
-  # add all years:
-  allYears <- result %>% dplyr::group_by(.data$ageGroup, .data$gender) %>%
-    dplyr::summarize(N = sum(.data$N), 
-                     O = sum(.data$O),
-                     year = 'all'
-    ) %>% dplyr::select(.data$year, .data$ageGroup, .data$gender, .data$N, .data$O) 
-  # add all gender:
-  allGender <- result %>% dplyr::group_by(.data$year, .data$ageGroup) %>%
-    dplyr::summarize(N = sum(.data$N), 
-                     O = sum(.data$O),
-                     gender = -1
-    ) %>% dplyr::select(.data$year, .data$ageGroup, .data$gender, .data$N, .data$O) 
-  
-  # add all gender:
-  allAge <- result %>% dplyr::group_by(.data$year, .data$gender) %>%
-    dplyr::summarize(N = sum(.data$N), 
-                     O = sum(.data$O),
-                     ageGroup = 'all'
-    ) %>% dplyr::select(.data$year, .data$ageGroup, .data$gender, .data$N, .data$O) 
-  
-  result <- rbind(result, allYears, allGender, allAge)
-  
-  result$opercent <- result$O/result$N*100
-  
-  # censor
-  if(!is.null(minCellCount)){
-    result$opercent[result$O < minCellCount] <- -1
-    result$N[result$N<minCellCount] <- paste0('<', minCellCount)
-    result$O[result$O<minCellCount] <- paste0('<', minCellCount)
-    
-  }
-  
-  result$TAR <- TAR 
-  result$analysisId <- analysisId
-  result$cdmDatabaseName <- cdmDatabaseName
-  result$cohortId <- cohortId
-  result$outcomeId <- outcomeId
-  
-  return(result)
-}
-  
diff --git a/R/EvaluationSummary.R b/R/EvaluationSummary.R
index 179c75717..a318cb923 100644
--- a/R/EvaluationSummary.R
+++ b/R/EvaluationSummary.R
@@ -35,7 +35,7 @@ getEvaluationStatistics_binary <- function(prediction, evalColumn, ...){
     
     # auc
     ParallelLogger::logInfo(paste0('Calculating Performance for ', evalType))
-    ParallelLogger::logInfo('=============')
+   ParallelLogger::logInfo('=============')
     
     ParallelLogger::logTrace('Calculating AUC')
     auc <- computeAuc(predictionOfInterest, confidenceInterval = T)
@@ -71,13 +71,22 @@ getEvaluationStatistics_binary <- function(prediction, evalColumn, ...){
       c(evalType, 'brier score scaled', brier$brierScaled)
     )
     ParallelLogger::logInfo(sprintf('%-20s%.2f', 'Brier: ', brier$brier))
+
     
     # using rms::val.prob
     indValProb <- predictionOfInterest$value>0 & predictionOfInterest$value < 1
-    valProb <- tryCatch(rms::val.prob(predictionOfInterest$value[indValProb], predictionOfInterest$outcomeCount[indValProb]), 
-      error = function(e){ParallelLogger::logInfo(e); return(c(Eavg = 0, 
-        E90 = 0, 
-        Emax = 0))})
+    valProb <- tryCatch(
+      calculateEStatisticsBinary(prediction = predictionOfInterest[indValProb, ]),
+      error = function(e) {
+        ParallelLogger::logInfo(e); return(
+          c(
+            Eavg = 0, 
+            E90 = 0, 
+            Emax = 0
+          )
+        )
+      }
+    )
     result <- rbind(
       result, 
       c(evalType, 'Eavg', valProb['Eavg']),
@@ -85,6 +94,8 @@ getEvaluationStatistics_binary <- function(prediction, evalColumn, ...){
       c(evalType, 'Emax', valProb['Emax'])
     )
     ParallelLogger::logInfo(sprintf('%-20s%.2f', 'Eavg: ', round(valProb['Eavg'], digits = 4)))
+
+
     
     
     # Removing for now as too slow...
@@ -213,28 +224,48 @@ getEvaluationStatistics_survival <- function(prediction, evalColumn, timepoint,
       c(evalType, timepoint, 'C-statistic upper 95% CI', cStatistic_u95CI)
     )
     ParallelLogger::logInfo(paste0('C-statistic: ',cStatistic, ' (',cStatistic_l95CI ,'-', cStatistic_u95CI ,')'))
-    
-    
+
     # add e-stat
-    w <- tryCatch(
-      {
-        rms::val.surv(
-          est.surv = 1-p, 
-          S = S,
-          u = timepoint, 
-          fun = function(pr)log(-log(pr))
+
+    .validateSurvival <- function(p, S, timepoint) {
+      estimatedSurvival <- 1 - p
+      notMissing <- !is.na(estimatedSurvival + S[, 1] + S[, 2])
+      estimatedSurvival   <- estimatedSurvival[notMissing]
+      S <- S[notMissing, ]
+      .curtail <- function(x) pmin(.9999, pmax(x, .0001))
+      f <- polspline::hare(
+        S[, 1],
+        S[, 2],
+        log(-log((.curtail(estimatedSurvival)))),
+        maxdim = 5
+      )
+      actual <- 1 - polspline::phare(timepoint, log(-log(estimatedSurvival)), f)
+
+      return(
+        list(
+          actual = actual,
+          estimatedSurvival = estimatedSurvival
         )
-      },
-      error = function(e){ParallelLogger::logError(e); return(NULL)}
+      )
+    }
+
+    w <- tryCatch(
+    {
+      .validateSurvival(
+        p = p,
+        S = S,
+        timepoint = timepoint
+      )
+    },
+    error = function(e){ParallelLogger::logError(e); return(NULL)}
     )
-    
-    eStatistic <- -1
-    eStatistic90 <- -1
-    if(!is.null(w)){
-      eStatistic<-mean(abs(w$actual - w$p))
-      eStatistic90<-stats::quantile((abs(w$actual - w$p)),0.9)
-      
+
+    eStatistic <- eStatistic90 <- -1
+    if (!is.null(w)) {
+      eStatistic <- mean(abs(w$actual - w$estimatedSurvival)) 
+      eStatistic90 <- stats::quantile(abs(w$actual - w$estimatedSurvival), probs = .9)
     }
+
     result <- rbind(
       result, 
       c(evalType, timepoint, 'E-statistic', eStatistic),
@@ -251,6 +282,28 @@ getEvaluationStatistics_survival <- function(prediction, evalColumn, timepoint,
 }
 
 
+calculateEStatisticsBinary <- function(prediction) {
+  risk <- prediction$value
+  outcome <- prediction$outcomeCount
+  notna <- ! is.na(risk + outcome)
+  risk <- risk[notna]
+  outcome <- outcome[notna]
+  smoothFit <- stats::lowess(risk, outcome, iter = 0)
+  smoothCalibration <- stats::approx(smoothFit, xout = risk, ties = mean)$y
+  distance <- abs(risk - smoothCalibration)
+  eavg <- mean(abs(risk - smoothCalibration))
+  emax <- max(distance)
+  e90 <- stats::quantile(distance, probs = .9)
+  names(e90) <- NULL
+  return(
+    c(
+      Eavg = eavg,
+      E90 = e90,
+      Emax = emax
+    )
+  )
+}
+
 
 #==================================
 # Fucntions for the summary
@@ -280,13 +333,13 @@ computeAuc <- function(prediction,
 }
 
 aucWithCi <- function(prediction, truth){
-  auc <- pROC::auc(as.factor(truth), prediction, direction="<")
+  auc <- pROC::auc(as.factor(truth), prediction, direction="<", quiet=TRUE)
   aucci <-pROC::ci(auc)
   return(data.frame(auc = aucci[2], auc_lb95ci = aucci[1], auc_ub95ci = aucci[3]))
 }
 
 aucWithoutCi <- function(prediction, truth){
-  auc <- pROC::auc(as.factor(truth), prediction, direction="<")
+  auc <- pROC::auc(as.factor(truth), prediction, direction="<", quiet=TRUE)
   return(as.double(auc))
 }
 
diff --git a/R/ExternalValidatePlp.R b/R/ExternalValidatePlp.R
index 84ca205ce..8e3e7a64a 100644
--- a/R/ExternalValidatePlp.R
+++ b/R/ExternalValidatePlp.R
@@ -17,65 +17,68 @@
 # limitations under the License.
 
 
-externalValidatePlp <- function(
-  plpModel,
-  plpData,
-  databaseName = 'database 1',
-  population,
-  settings = createValidationSettings(  # add covariateSummary option?
-    recalibrate = 'weakRecalibration'
-  )
-){
-  
-  # Apply model 
+externalValidatePlp <- function(plpModel,
+                                plpData,
+                                population,
+                                settings = list(# add covariateSummary option?
+                                  recalibrate = 'weakRecalibration')) {
+  # Apply model
   #=======
   prediction <- tryCatch({
-    predictPlp(
-      plpModel = plpModel, 
-      plpData = plpData, 
-      population = population
-    )},
-    error = function(e){ParallelLogger::logError(e)}
-  )
+    predictPlp(plpModel = plpModel,
+               plpData = plpData,
+               population = population)
+  },
+  error = function(e) {
+    ParallelLogger::logError(e)
+  })
   
   prediction$evaluationType <- 'Validation'
   
   # Recalibrate
   #=======
-  if(!is.null(settings$recalibrate)){
-    for(recalMethod in settings$recalibrate){
-      prediction <- tryCatch({recalibratePlp(prediction = prediction, method = recalMethod)
-      },
-        error = function(e){ParallelLogger::logError(e)}
-      )
+  if (!is.null(settings$recalibrate)) {
+    for (recalMethod in settings$recalibrate) {
+      prediction <-
+        tryCatch({
+          recalibratePlp(prediction = prediction, method = recalMethod)
+        },
+        error = function(e) {
+          ParallelLogger::logError(e)
+        })
     }
   }
   
-  # Evaluate 
+  # Evaluate
   #=======
   performance <- tryCatch({
     evaluatePlp(prediction = prediction, typeColumn = 'evaluationType')
   },
-    error = function(e){ParallelLogger::logError(e)}
-  )
+  error = function(e) {
+    ParallelLogger::logError(e)
+  })
   
   # step 6: covariate summary
   labels <- tryCatch({
-    population %>% dplyr::select(.data$rowId, .data$outcomeCount)
+    population %>% dplyr::select("rowId", "outcomeCount")
   },
-    error = function(e){ return(NULL) }
-  )
+  error = function(e) {
+    return(NULL)
+  })
   
-  if(settings$runCovariateSummary){
-  covariateSum <- tryCatch({
-    covariateSummary(
-      covariateData = plpData$covariateData, 
-      cohort = population[,colnames(population) != 'outcomeCount'], 
-      labels = labels, 
-      variableImportance = plpModel$covariateImportance %>% dplyr::select(.data$covariateId, .data$covariateValue)
-    )},
-    error = function(e){ParallelLogger::logInfo(e); return(NULL) }
-  )
+  if (settings$runCovariateSummary) {
+    covariateSum <- tryCatch({
+      covariateSummary(
+        covariateData = plpData$covariateData,
+        cohort = population[, colnames(population) != 'outcomeCount'],
+        labels = labels,
+        variableImportance = plpModel$covariateImportance %>% dplyr::select("covariateId", "covariateValue")
+      )
+    },
+    error = function(e) {
+      ParallelLogger::logInfo(e)
+      return(NULL)
+    })
   } else{
     covariateSum <- NULL
   }
@@ -83,10 +86,10 @@ externalValidatePlp <- function(
   executionSummary <- list(
     ExecutionDateTime = Sys.Date(),
     PackageVersion = list(
-      rVersion= R.Version()$version.string,
+      rVersion = R.Version()$version.string,
       packageVersion = utils::packageVersion("PatientLevelPrediction")
     ),
-    PlatformDetails= list(
+    PlatformDetails = list(
       platform = R.Version()$platform,
       cores = Sys.getenv('NUMBER_OF_PROCESSORS'),
       RAM = memuse::Sys.meminfo()[1]
@@ -95,16 +98,21 @@ externalValidatePlp <- function(
   
   model = list(
     model = 'external validation of model',
-    settings = plpModel$settings,
+    modelDesign = plpModel$modelDesign,
+    # was settings
     validationDetails   = list(
-      analysisId = '', #TODO add from model
-      analysisSource = '', #TODO add from model
-      developmentDatabase = plpModel$trainDetails$cdmDatabaseSchema,
-      cdmDatabaseSchema = databaseName,
+      analysisId = '',
+      #TODO add from model
+      analysisSource = '',
+      #TODO add from model
+      developmentDatabase = plpModel$trainDetails$developmentDatabase,
+      developmentDatabaseId = plpModel$trainDetails$developmentDatabaseId,
+      validationDatabase = plpData$metaData$databaseDetails$cdmDatabaseSchema,
+      validationDatabaseId = plpData$metaData$databaseDetails$cdmDatabaseId,
       populationSettings = attr(population, 'metaData')$populationSettings,
-      plpDataSettings = attr(plpData, 'metaData')$restrictPlpDataSettings,
+      restrictPlpDataSettings = plpData$metaData$restrictPlpDataSettings,
       outcomeId = attr(population, 'metaData')$outcomeId,
-      cohortId = attr(plpData, 'metaData')$databaseDetails$cohortId,
+      targetId = plpData$metaData$databaseDetails$targetId,
       attrition = attr(population, 'metaData')$attrition,
       validationDate = Sys.Date() # is this needed?
     )
@@ -132,61 +140,62 @@ externalValidatePlp <- function(
 #' @description
 #' This function extracts data using a user specified connection and cdm_schema, applied the model and then calcualtes the performance
 #' @details
-#' Users need to input a trained model (the output of runPlp()) and new database connections. The function will return a list of length equal to the 
+#' Users need to input a trained model (the output of runPlp()) and new database connections. The function will return a list of length equal to the
 #' number of cdm_schemas input with the performance on the new data
-#' 
+#'
 #' @param plpModel                    The model object returned by runPlp() containing the trained model
 #' @param validationDatabaseDetails   A list of objects of class \code{databaseDetails} created using \code{createDatabaseDetails}
 #' @param validationRestrictPlpDataSettings   A list of population restriction settings created by \code{createRestrictPlpDataSettings()}
 #' @param settings                    A settings object of class \code{validationSettings} created using \code{createValidationSettings}
-#' @param logSettings                 An object of \code{logSettings} created using \code{createLogSettings} 
-#'                                    specifying how the logging is done 
+#' @param logSettings                 An object of \code{logSettings} created using \code{createLogSettings}
+#'                                    specifying how the logging is done
 #' @param outputFolder                The directory to save the validation results to (subfolders are created per database in validationDatabaseDetails)
-#' 
+#'
 #' @return
-#' A list containing the performance for each validation_schema 
+#' A list containing the performance for each validation_schema
 #'
 #'
 #' @export
-externalValidateDbPlp <- function(
-  plpModel,
-  validationDatabaseDetails = createDatabaseDetails(),
-  validationRestrictPlpDataSettings = createRestrictPlpDataSettings(),
-  settings = createValidationSettings(
-    recalibrate = 'weakRecalibration'
-    ),
-  logSettings = createLogSettings(verbosity = 'INFO', logName = 'validatePLP'),
-  outputFolder = getwd()
-){
-  
-  # Input checks 
+externalValidateDbPlp <- function(plpModel,
+                                  validationDatabaseDetails = createDatabaseDetails(),
+                                  validationRestrictPlpDataSettings = createRestrictPlpDataSettings(),
+                                  settings = createValidationSettings(recalibrate = 'weakRecalibration'),
+                                  logSettings = createLogSettings(verbosity = 'INFO', logName = 'validatePLP'),
+                                  outputFolder = getwd()) {
+  # Input checks
   #=======
   
   checkIsClass(plpModel, 'plpModel')
   
   # check the class and make a list if a single database setting
-  if(class(validationDatabaseDetails) == 'list'){
-    lapply(validationDatabaseDetails, function(x) checkIsClass(x, 'databaseDetails'))
+  if (inherits(validationDatabaseDetails, 'list')) {
+    lapply(validationDatabaseDetails, function(x)
+      checkIsClass(x, 'databaseDetails'))
   } else{
     checkIsClass(validationDatabaseDetails, 'databaseDetails')
     validationDatabaseDetails <- list(validationDatabaseDetails)
   }
-  checkIsClass(validationRestrictPlpDataSettings, 'restrictPlpDataSettings')
+  checkIsClass(validationRestrictPlpDataSettings,
+               'restrictPlpDataSettings')
   checkIsClass(settings, 'validationSettings')
-
+  
   # create results list with the names of the databases to validate across
   result <- list()
   length(result) <- length(validationDatabaseDetails)
-  names(result) <- unlist(lapply(validationDatabaseDetails, function(x) attr(x, 'cdmDatabaseName')))
+  names(result) <-
+    unlist(lapply(validationDatabaseDetails, function(x)
+      attr(x, 'cdmDatabaseName')))
   
-  for(databaseDetails in validationDatabaseDetails){
-    
+  for (databaseDetails in validationDatabaseDetails) {
     databaseName <- attr(databaseDetails, 'cdmDatabaseName')
     
     # initiate log
-    logSettings$saveDirectory <- file.path(outputFolder, databaseName, plpModel$trainDetails$analysisId)
+    logSettings$saveDirectory <-
+      file.path(outputFolder,
+                databaseName,
+                plpModel$trainDetails$analysisId)
     logSettings$logFileName <- 'validationLog'
-    logger <- do.call(createLog,logSettings)
+    logger <- do.call(createLog, logSettings)
     ParallelLogger::registerLogger(logger)
     
     ParallelLogger::logInfo(paste('Validating model on', databaseName))
@@ -194,87 +203,113 @@ externalValidateDbPlp <- function(
     # Step 1: get data
     #=======
     
-    getPlpDataSettings <- list(
-      databaseDetails = databaseDetails,
-      restrictPlpDataSettings = validationRestrictPlpDataSettings
-      )
-    if(is.null(getPlpDataSettings$databaseDetails$cohortId)){
-      ParallelLogger::logInfo("cohortId not in databaseSettings so using model's")
-      getPlpDataSettings$databaseDetails$cohortId <- plpModel$trainDetails$cohortId
+    getPlpDataSettings <- list(databaseDetails = databaseDetails,
+                               restrictPlpDataSettings = validationRestrictPlpDataSettings)
+    if (is.null(getPlpDataSettings$databaseDetails$targetId)) {
+      ParallelLogger::logInfo("targetId not in databaseSettings so using model's")
+      getPlpDataSettings$databaseDetails$targetId <-
+        plpModel$modelDesign$targetId
     }
-    if(is.null(getPlpDataSettings$databaseDetails$outcomeIds)){
-      ParallelLogger::logInfo("cohortId not in databaseSettings  so using model's")
-      getPlpDataSettings$databaseDetails$outcomeIds <- plpModel$trainDetails$outcomeId
+    if (is.null(getPlpDataSettings$databaseDetails$outcomeIds)) {
+      ParallelLogger::logInfo("outcomeId not in databaseSettings  so using model's")
+      getPlpDataSettings$databaseDetails$outcomeIds <-
+        plpModel$modelDesign$outcomeId
     }
     
-    if(is.null(getPlpDataSettings$restrictPlpDataSettings$firstExposureOnly)){
+    if (is.null(getPlpDataSettings$restrictPlpDataSettings$firstExposureOnly)) {
       ParallelLogger::logInfo("firstExposureOnly not in restrictPlpDataSettings  so using model's")
-      getPlpDataSettings$restrictPlpDataSettings$firstExposureOnly <- plpModel$settings$plpDataSettings$firstExposureOnly
+      getPlpDataSettings$restrictPlpDataSettings$firstExposureOnly <-
+        plpModel$modelDesign$restrictPlpDataSettings$firstExposureOnly
     }
-    if(is.null(getPlpDataSettings$restrictPlpDataSettings$washoutPeriod)){
+    if (is.null(getPlpDataSettings$restrictPlpDataSettings$washoutPeriod)) {
       ParallelLogger::logInfo("washoutPeriod not in restrictPlpDataSettings so using model's")
-      getPlpDataSettings$restrictPlpDataSettings$washoutPeriod <- plpModel$settings$plpDataSettings$washoutPeriod
+      getPlpDataSettings$restrictPlpDataSettings$washoutPeriod <-
+        plpModel$modelDesign$restrictPlpDataSettings$washoutPeriod
     }
     
-    # we need to update this to restrict to model covariates and update custom features
-    getPlpDataSettings$covariateSettings <- plpModel$settings$covariateSettings
+    # TODO: we need to update this to restrict to model covariates and update custom features
+    getPlpDataSettings$covariateSettings <-
+      plpModel$modelDesign$covariateSettings
     
     plpData <- tryCatch({
       do.call(getPlpData, getPlpDataSettings)
     },
-      error = function(e){ParallelLogger::logError(e); return(NULL)}
-    )
+    error = function(e) {
+      ParallelLogger::logError(e)
+      return(NULL)
+    })
     
-    if(is.null(plpData)){
+    if (is.null(plpData)) {
       closeLog(logger)
     }
     
-    # Step 2: create population 
+    # Step 2: create population
     #=======
     
     population <- tryCatch({
       do.call(
-        createStudyPopulation, 
+        createStudyPopulation,
         list(
           plpData = plpData,
           outcomeId = getPlpDataSettings$databaseDetails$outcomeIds,
-          populationSettings = plpModel$settings$populationSettings
+          populationSettings = plpModel$modelDesign$populationSettings
         )
       )
     },
-      error = function(e){ParallelLogger::logError(e); return(NULL)}
-    )
+    error = function(e) {
+      ParallelLogger::logError(e)
+      return(NULL)
+    })
     
-    if(is.null(population)){
+    if (is.null(population)) {
       closeLog(logger)
     }
     
     # Step 3: Apply model to plpData and population
     #=======
-
+    
     result[[databaseName]] <- tryCatch({
-      externalValidatePlp(
-        plpModel,
-        plpData,
-        databaseName = databaseName,
-        population,
-        settings = settings
-      )},
-      error = function(e){ParallelLogger::logInfo(e); return(NULL)}
-    )
+      externalValidatePlp(plpModel,
+                          plpData,
+                          #databaseName = databaseName,
+                          population,
+                          settings = settings)
+    },
+    error = function(e) {
+      ParallelLogger::logInfo(e)
+      return(NULL)
+    })
     
-    if(is.null(result[[databaseName]])){
+    if (is.null(result[[databaseName]])) {
       closeLog(logger)
     } else{
+      if (!dir.exists(file.path(
+        outputFolder,
+        databaseName,
+        plpModel$trainDetails$analysisId
+      ))) {
+        dir.create(
+          file.path(
+            outputFolder,
+            databaseName,
+            plpModel$trainDetails$analysisId
+          ),
+          recursive = T
+        )
+      }
       
-      if(!dir.exists(file.path(outputFolder, databaseName, plpModel$trainDetails$analysisId))){
-        dir.create(file.path(outputFolder, databaseName, plpModel$trainDetails$analysisId), recursive = T)
-      }  
-      
-      savePlpResult(result[[databaseName]], dirPath = file.path(outputFolder, databaseName, plpModel$trainDetails$analysisId, 'validationResult'))
+      savePlpResult(
+        result[[databaseName]],
+        dirPath = file.path(
+          outputFolder,
+          databaseName,
+          plpModel$trainDetails$analysisId,
+          'validationResult'
+        )
+      )
     }
   }
- 
+  
   # Now return results
   return(invisible(result))
 }
@@ -286,29 +321,247 @@ externalValidateDbPlp <- function(
 #' This function creates the settings required by externalValidatePlp
 #' @details
 #' Users need to specify whether they want to sample or recalibate when performing external validation
-#' 
-#' @param recalibrate                      A vector of characters specifying the recalibration method to apply                 
+#'
+#' @param recalibrate                      A vector of characters specifying the recalibration method to apply
 #' @param runCovariateSummary              Whether to run the covariate summary for the validation data
 #' @return
 #' A setting object of class \code{validationSettings} containing a list of settings for externalValidatePlp
 #'
 #' @export
-createValidationSettings <- function(
-  recalibrate = NULL,
-  runCovariateSummary = T
-){
-  
-  checkIsClass(recalibrate, c('character','NULL'))
-  if(!is.null(recalibrate)){
-    if(sum(recalibrate %in% c('recalibrationInTheLarge', 'weakRecalibration'))!=length(recalibrate)){
-      ParallelLogger::logError('Incorrect recalibrate options used.  Must be recalibrationInTheLarge or weakRecalibration')
+createValidationSettings <- function(recalibrate = NULL,
+                                     runCovariateSummary = T) {
+  checkIsClass(recalibrate, c('character', 'NULL'))
+  if (!is.null(recalibrate)) {
+    if (sum(recalibrate %in% c('recalibrationInTheLarge', 'weakRecalibration')) !=
+        length(recalibrate)) {
+      ParallelLogger::logError(
+        'Incorrect recalibrate options used.  Must be recalibrationInTheLarge or weakRecalibration'
+      )
     }
   }
-
-  result = list(
-    recalibrate = recalibrate,
-    runCovariateSummary = runCovariateSummary
-  )
+  
+  result <- list(recalibrate = recalibrate,
+                 runCovariateSummary = runCovariateSummary)
   class(result) <- 'validationSettings'
   return(result)
 }
+
+#' createValidationDesign - Define the validation design for external validation
+#'
+#' @param targetId The targetId of the target cohort to validate on
+#' @param outcomeId The outcomeId of the outcome cohort to validate on
+#' @param populationSettings A list of population restriction settings created by \code{createPopulationSettings}
+#' @param restrictPlpDataSettings A list of plpData restriction settings created by \code{createRestrictPlpDataSettings}
+#' @param plpModelList A list of plpModels objects created by \code{runPlp} or a path to such objects
+#' @param recalibrate A vector of characters specifying the recalibration method to apply,
+#' @param runCovariateSummary whether to run the covariate summary for the validation data
+#' @export
+createValidationDesign <-
+  function(targetId,
+           outcomeId,
+           populationSettings,
+           restrictPlpDataSettings,
+           plpModelList,
+           recalibrate = NULL,
+           runCovariateSummary = TRUE) {
+    checkIsClass(targetId, c("numeric", "integer"))
+    checkIsClass(outcomeId, c("numeric", "integer"))
+    checkIsClass(populationSettings, c("populationSettings"))
+    checkIsClass(restrictPlpDataSettings, "restrictPlpDataSettings")
+    checkIsClass(plpModelList, "list")
+    lapply(plpModelList, function(x)
+      checkIsClass(x, c("plpModel", "character")))
+    checkIsClass(recalibrate, c("character", "NULL"))
+    checkIsClass(runCovariateSummary, "logical")
+    
+    design <- list(
+      targetId = targetId,
+      outcomeId = outcomeId,
+      populationSettings = populationSettings,
+      plpModelList = plpModelList,
+      restrictPlpDataSettings = restrictPlpDataSettings,
+      recalibrate = recalibrate,
+      runCovariateSummary = runCovariateSummary
+    )
+    class(design) <- "validationDesign"
+    return(design)
+  }
+
+
+#' externalValidatePlp - Validate model performance on new data
+#'
+#' @param validationDesignList A list of objects created with \code{createValidationDesign}
+#' @param databaseDetails A list of objects of class
+#' \code{databaseDetails} created using \code{createDatabaseDetails}
+#' @param logSettings               An object of \code{logSettings} created
+#' using \code{createLogSettings}
+#' @param outputFolder        The directory to save the validation results to
+#' (subfolders are created per database in validationDatabaseDetails)
+#' @export
+validateExternal <- function(validationDesignList,
+                             databaseDetails,
+                             logSettings,
+                             outputFolder) {
+  # Input checks
+  #=======
+  if (inherits(validationDesignList, 'list')) {
+    lapply(validationDesignList, function(x)
+      checkIsClass(x, 'validationDesign'))
+  } else {
+    checkIsClass(validationDesignList, 'validationDesign')
+    validationDesignList <- list(validationDesignList)
+  }
+ 
+  # check the class and make a list if a single database setting
+  if (inherits(databaseDetails, 'list')) {
+    lapply(databaseDetails, function(x)
+      checkIsClass(x, 'databaseDetails'))
+  } else {
+    checkIsClass(databaseDetails, 'databaseDetails')
+    databaseDetails <- list(databaseDetails)
+  }
+  
+  # create results list with the names of the databases to validate across
+  result <- list()
+  length(result) <- length(databaseDetails)
+  names(result) <-
+    unlist(lapply(databaseDetails, function(x)
+      attr(x, 'cdmDatabaseName')))
+  
+  # Need to keep track of incremental analysisId's for each database
+  databaseNames <- unlist(lapply(databaseDetails, function(x)
+    x$cdmDatabaseName))
+  analysisInfo <- list()
+  for (name in databaseNames) {
+    analysisInfo[name] <- 1
+  }
+  for (design in validationDesignList) {
+    for (database in databaseDetails) {
+      databaseName <- database$cdmDatabaseName
+      # initiate log
+      logSettings$saveDirectory <- file.path(outputFolder,
+                                             database$cdmDatabaseName)
+      logSettings$logFileName <- 'validationLog'
+      logger <- do.call(createLog, logSettings)
+      ParallelLogger::registerLogger(logger)
+      on.exit(logger$close())
+      
+      ParallelLogger::logInfo(paste('Validating model on', database$cdmDatabaseName))
+      
+      database$targetId <- design$targetId
+      
+      database$outcomeIds <- design$outcomeId
+      
+      allCovSettings <-
+        lapply(design$plpModelList, function(plpModel)
+          plpModel$modelDesign$covariateSettings)
+      # compare all to first covSettings, if not the same stop
+      if (!Reduce(function(x, y)
+        x &&
+        identical(y, allCovSettings[[1]]),
+        allCovSettings[-1],
+        init = TRUE)) {
+        stop("covariateSettings are not the same across models which is not supported yet")
+      }
+      plpData <- tryCatch({
+        do.call(
+          getPlpData,
+          list(
+            databaseDetails = database,
+            restrictPlpDataSettings = design$restrictPlpDataSettings,
+            covariateSettings = design$plpModelList[[1]]$modelDesign$covariateSettings
+          )
+        )
+      },
+      error = function(e) {
+        ParallelLogger::logError(e)
+        return(NULL)
+      })     
+      plpDataName <-
+        paste0("targetId_", design$targetId, "_L", "1") # Is the 1 for how many targetIds in file ?
+      plpDataLocation <-
+        file.path(outputFolder, databaseName, plpDataName)
+      if (!dir.exists(file.path(outputFolder, databaseName))) {
+        dir.create(file.path(outputFolder, databaseName), recursive = TRUE)
+      }
+      savePlpData(plpData, file = plpDataLocation)
+      
+      # create study population
+      population <- tryCatch({
+        do.call(
+          createStudyPopulation,
+          list(
+            plpData = plpData,
+            outcomeId = design$outcomeId,
+            populationSettings = design$populationSettings
+          )
+        )
+      },
+      error = function(e) {
+        ParallelLogger::logError(e)
+        return(NULL)
+      })
+      
+      results <- lapply(design$plpModelList, function(model) {
+        analysisName <- paste0("Analysis_", analysisInfo[databaseName])
+        validateModel(
+          plpModel = model,
+          plpData = plpData,
+          population = population,
+          recalibrate = design$recalibrate,
+          runCovariateSummary = design$runCovariateSummary,
+          outputFolder = outputFolder,
+          databaseName = databaseName,
+          analysisName = analysisName
+        )
+        analysisInfo[[databaseName]] <<- analysisInfo[[databaseName]] + 1
+      })
+    }
+  }
+  for (database in databaseDetails) {
+    databaseName <- database$cdmDatabaseName
+    sqliteLocation <-
+      file.path(outputFolder, 'sqlite')
+    tryCatch({
+      insertResultsToSqlite(
+        resultLocation = file.path(outputFolder, databaseName),
+        cohortDefinitions = NULL,
+        databaseList = createDatabaseList(
+          cdmDatabaseSchemas = database$cdmDatabaseSchema,
+          cdmDatabaseNames = database$cdmDatabaseName,
+          databaseRefIds = database$cdmDatabaseId
+        ),
+        sqliteLocation = sqliteLocation
+      )
+    },
+    error = function(e) ParallelLogger::logInfo(e)
+    )
+  }
+  return(invisible(results))
+}
+
+validateModel <-
+  function(plpModel,
+           plpData,
+           population,
+           recalibrate,
+           runCovariateSummary,
+           outputFolder,
+           databaseName,
+           analysisName) {
+    result <- externalValidatePlp(
+      plpModel = plpModel,
+      plpData = plpData,
+      population = population,
+      settings = list(recalibrate = recalibrate,
+                      runCovariateSummary = runCovariateSummary)
+    )
+    savePlpResult(result,
+                  dirPath = file.path(
+                    outputFolder,
+                    databaseName,
+                    analysisName,
+                    'validationResult'
+                  ))
+    return(result)
+}
diff --git a/R/ExtractData.R b/R/ExtractData.R
index e7a0d8d8c..e754eb802 100644
--- a/R/ExtractData.R
+++ b/R/ExtractData.R
@@ -85,7 +85,8 @@ createRestrictPlpDataSettings <- function(
 #'                                       instance. Requires read permissions to this database. On SQL
 #'                                       Server, this should specifiy both the database and the schema,
 #'                                       so for example 'cdm_instance.dbo'.
-#' @param cdmDatabaseName                A string with a shareable name of the database (this will be shown to OHDSI researchers if the results get transported)
+#' @param cdmDatabaseName                A string with the name of the database - this is used in the shiny app and when externally validating models to name the result list and to specify the folder name when saving validation results (defaults to cdmDatabaseSchema if not specified)
+#' @param cdmDatabaseId                  A string with a unique identifier for the database and version - this is stored in the plp object for future reference and used by the shiny app (defaults to cdmDatabaseSchema if not specified)
 #' @param tempEmulationSchema            For dmbs like Oracle only: the name of the database schema where you
 #'                                       want all temporary tables to be managed. Requires
 #'                                       create/insert permissions to this database.
@@ -101,10 +102,11 @@ createRestrictPlpDataSettings <- function(
 #' @param outcomeTable                   The tablename that contains the outcome cohorts.  Expectation is
 #'                                       outcomeTable has format of COHORT table: COHORT_DEFINITION_ID,
 #'                                       SUBJECT_ID, COHORT_START_DATE, COHORT_END_DATE.
-#' @param cohortId                       An integer specifying the cohort id for the target cohort
+#' @param targetId                       An integer specifying the cohort id for the target cohort
 #' @param outcomeIds                      A single integer or vector of integers specifying the cohort ids for the outcome cohorts
-#' @param cdmVersion                     Define the OMOP CDM version used: currently support "4" and
-#'                                       "5".
+#' @param cdmVersion                     Define the OMOP CDM version used: currently support "4" and "5".
+#' @param cohortId                       (depreciated: use targetId) old input for the target cohort id
+#' 
 #' @return
 #' A list with the the database specific settings (this is used by the runMultiplePlp function and the skeleton packages)
 #'
@@ -113,25 +115,55 @@ createDatabaseDetails <- function(
   connectionDetails,
   cdmDatabaseSchema,
   cdmDatabaseName,
+  cdmDatabaseId, # added for strategus
   tempEmulationSchema = cdmDatabaseSchema,
   cohortDatabaseSchema = cdmDatabaseSchema,
   cohortTable = "cohort",
   outcomeDatabaseSchema = cdmDatabaseSchema,
   outcomeTable = "cohort",
-  cohortId = NULL,
+  targetId = NULL,
   outcomeIds = NULL,
-  cdmVersion = 5
+  cdmVersion = 5,
+  cohortId = NULL
 ){
+  
+  if(is.null(targetId)){
+    if(!is.null(cohortId)){
+      ParallelLogger::logWarn('cohortId has been depreciated.  Please use targetId.')
+      targetId <- cohortId
+    }
+  }
+  
+  if(missing(cdmDatabaseName)){
+    ParallelLogger::logInfo('No cdm database name entered so using cdmDatabaseSchema')
+    cdmDatabaseName <- removeInvalidString(cdmDatabaseSchema)
+  }
+  if(missing(cdmDatabaseId)){
+    ParallelLogger::logInfo('No cdm database id entered so using cdmDatabaseSchema - if cdmDatabaseSchema is the same for multiple different databases, please use cdmDatabaseId to specify a unique identifier for the database and version')
+    cdmDatabaseId <- removeInvalidString(cdmDatabaseSchema)
+  }
+  
+  if(length(cdmDatabaseId) == 0 ){
+    stop('cdmDatabaseId must be a string with length > 0')
+  }
+  
+  # check to make sure cdmDatabaseId is not an int as that will cause issues
+  if(!inherits(cdmDatabaseId, 'character')){
+    ParallelLogger::logInfo('cdmDatabaseId is not a string - this will cause issues when inserting into a result database so casting it')
+    cdmDatabaseId <- as.character(cdmDatabaseId)
+  }
+  
   result <- list(
     connectionDetails = connectionDetails,
     cdmDatabaseSchema = cdmDatabaseSchema,
     cdmDatabaseName = cdmDatabaseName,
+    cdmDatabaseId = cdmDatabaseId,
     tempEmulationSchema = tempEmulationSchema,
     cohortDatabaseSchema = cohortDatabaseSchema,
     cohortTable = cohortTable,
     outcomeDatabaseSchema = outcomeDatabaseSchema,
     outcomeTable = outcomeTable,
-    cohortId = cohortId ,
+    targetId = targetId,
     outcomeIds = outcomeIds,
     cdmVersion = cdmVersion
   )
@@ -188,10 +220,10 @@ getPlpData <- function(
   checkIsClass(databaseDetails, 'databaseDetails')
   checkIsClass(restrictPlpDataSettings, 'restrictPlpDataSettings')
 
-  if(is.null(databaseDetails$cohortId))
-    stop('User must input cohortId')
-  if(length(databaseDetails$cohortId)>1)
-    stop('Currently only supports one cohortId at a time')
+  if(is.null(databaseDetails$targetId))
+    stop('User must input targetId')
+  if(length(databaseDetails$targetId)>1)
+    stop('Currently only supports one targetId at a time')
   if(is.null(databaseDetails$outcomeIds))
     stop('User must input outcomeIds')
   #ToDo: add other checks the inputs are valid
@@ -202,16 +234,22 @@ getPlpData <- function(
   
   ParallelLogger::logTrace("\nConstructing the at risk cohort")
   if(!is.null(restrictPlpDataSettings$sampleSize))  writeLines(paste("\n Sampling ",restrictPlpDataSettings$sampleSize, " people"))
-  renderedSql <- SqlRender::loadRenderTranslateSql(
-    "CreateCohorts.sql",
-    packageName = "PatientLevelPrediction",
-    dbms = dbms, 
-    tempEmulationSchema = databaseDetails$tempEmulationSchema,
+
+  pathToSql <- system.file(
+    paste("sql/", "sql_server", 
+          sep = ""),
+    'CreateCohorts.sql', 
+    package = "PatientLevelPrediction"
+  )
+  
+  renderedSql <- readChar(pathToSql, file.info(pathToSql)$size) 
+  renderedSql <- SqlRender::render(
+    sql = renderedSql,
     cdm_database_schema = databaseDetails$cdmDatabaseSchema,
     cohort_database_schema = databaseDetails$cohortDatabaseSchema,
     cohort_table = databaseDetails$cohortTable,
     cdm_version = databaseDetails$cdmVersion,
-    cohort_id = databaseDetails$cohortId,
+    target_id = databaseDetails$targetId,
     study_start_date = restrictPlpDataSettings$studyStartDate,
     study_end_date = restrictPlpDataSettings$studyEndDate,
     first_only = restrictPlpDataSettings$firstExposureOnly,
@@ -219,36 +257,57 @@ getPlpData <- function(
     use_sample = !is.null(restrictPlpDataSettings$sampleSize),
     sample_number = restrictPlpDataSettings$sampleSize
   )
+  renderedSql <- SqlRender::translate(
+    sql = renderedSql, 
+    targetDialect = dbms, 
+    tempEmulationSchema = databaseDetails$tempEmulationSchema
+  )
+    
   DatabaseConnector::executeSql(connection, renderedSql)
   
   ParallelLogger::logTrace("Fetching cohorts from server")
   start <- Sys.time()
-  cohortSql <- SqlRender::loadRenderTranslateSql(
-    "GetCohorts.sql",
-    packageName = "PatientLevelPrediction",
-    dbms = dbms, 
-    tempEmulationSchema = databaseDetails$tempEmulationSchema,
+  
+  pathToSql <- system.file(
+    paste("sql/", "sql_server", 
+          sep = ""),
+    "GetCohorts.sql", 
+    package = "PatientLevelPrediction"
+  )
+  
+  cohortSql <- readChar(pathToSql, file.info(pathToSql)$size) 
+  
+  cohortSql <- SqlRender::render(
+    sql = cohortSql,
     cdm_version = databaseDetails$cdmVersion
   )
+  
+  cohortSql <- SqlRender::translate(
+    sql = cohortSql, 
+    targetDialect = dbms, 
+    tempEmulationSchema = databaseDetails$tempEmulationSchema
+  )
   cohorts <- DatabaseConnector::querySql(connection, cohortSql)
   colnames(cohorts) <- SqlRender::snakeCaseToCamelCase(colnames(cohorts))
-  metaData.cohort <- list(cohortId = databaseDetails$cohortId)
+  metaData.cohort <- list(targetId = databaseDetails$targetId)
   
-  if(nrow(cohorts)==0)
+  if(nrow(cohorts)==0){
     stop('Target population is empty')
+  }
   
   delta <- Sys.time() - start
   ParallelLogger::logTrace(paste("Loading cohorts took", signif(delta, 3), attr(delta, "units")))
-  
-  #covariateSettings$useCovariateCohortIdIs1 <- TRUE
-  covariateData <- FeatureExtraction::getDbCovariateData(connection = connection, 
-                                                         oracleTempSchema = databaseDetails$tempEmulationSchema,
-                                                         cdmDatabaseSchema = databaseDetails$cdmDatabaseSchema,
-                                                         cdmVersion = databaseDetails$cdmVersion,
-                                                         cohortTable = "#cohort_person",
-                                                         cohortTableIsTemp = TRUE,
-                                                         rowIdField = "row_id",
-                                                         covariateSettings = covariateSettings)
+
+  covariateData <- FeatureExtraction::getDbCovariateData(
+    connection = connection, 
+    oracleTempSchema = databaseDetails$tempEmulationSchema,
+    cdmDatabaseSchema = databaseDetails$cdmDatabaseSchema,
+    cdmVersion = databaseDetails$cdmVersion,
+    cohortTable = "#cohort_person",
+    cohortTableIsTemp = TRUE,
+    rowIdField = "row_id",
+    covariateSettings = covariateSettings
+  )
   # add indexes for tidyCov + covariate summary
   Andromeda::createIndex(covariateData$covariates, c('rowId'),
                          indexName = 'covariates_rowId')
@@ -260,23 +319,38 @@ getPlpData <- function(
   if(max(databaseDetails$outcomeIds)!=-999){
     ParallelLogger::logTrace("Fetching outcomes from server")
     start <- Sys.time()
-    outcomeSql <- SqlRender::loadRenderTranslateSql(
-      "GetOutcomes.sql",
-      packageName = "PatientLevelPrediction",
-      dbms = dbms,
-      tempEmulationSchema = databaseDetails$tempEmulationSchema,
+    
+    pathToSql <- system.file(
+      paste("sql/", "sql_server", 
+            sep = ""),
+      "GetOutcomes.sql", 
+      package = "PatientLevelPrediction"
+    )
+    
+    outcomeSql <- readChar(pathToSql, file.info(pathToSql)$size) 
+    
+    outcomeSql <- SqlRender::render(
+      sql = outcomeSql,
       cdm_database_schema = databaseDetails$cdmDatabaseSchema,
       outcome_database_schema = databaseDetails$outcomeDatabaseSchema,
       outcome_table = databaseDetails$outcomeTable,
       outcome_ids = databaseDetails$outcomeIds,
       cdm_version = databaseDetails$cdmVersion
     )
+    
+    outcomeSql <- SqlRender::translate(
+      sql = outcomeSql, 
+      targetDialect = dbms, 
+      tempEmulationSchema = databaseDetails$tempEmulationSchema
+    )
+    
     outcomes <- DatabaseConnector::querySql(connection, outcomeSql)
     colnames(outcomes) <- SqlRender::snakeCaseToCamelCase(colnames(outcomes))
     metaData.outcome <- data.frame(outcomeIds = databaseDetails$outcomeIds)
     attr(outcomes, "metaData") <- metaData.outcome
-    if(nrow(outcomes)==0)
+    if(nrow(outcomes)==0){
       stop('No Outcomes')
+    }
     
     metaData.cohort$attrition <- getCounts2(cohorts,outcomes, "Original cohorts")
     attr(cohorts, "metaData") <- metaData.cohort
@@ -288,17 +362,22 @@ getPlpData <- function(
   }
   
   
-  
-  
   # Remove temp tables:
-  renderedSql <- SqlRender::loadRenderTranslateSql(
-    "RemoveCohortTempTables.sql",
-    packageName = "PatientLevelPrediction",
-    dbms = dbms,
+  pathToSql <- system.file(
+    paste("sql/", "sql_server", 
+          sep = ""),
+    "RemoveCohortTempTables.sql", 
+    package = "PatientLevelPrediction"
+  )
+  
+  removeSql <- readChar(pathToSql, file.info(pathToSql)$size) 
+  removeSql <- SqlRender::translate(
+    sql = removeSql, 
+    targetDialect = dbms, 
     tempEmulationSchema = databaseDetails$tempEmulationSchema
   )
   
-  DatabaseConnector::executeSql(connection, renderedSql, progressBar = FALSE, reportOverallTime = FALSE)
+  DatabaseConnector::executeSql(connection, removeSql, progressBar = FALSE, reportOverallTime = FALSE)
   #DatabaseConnector::disconnect(connection)
   
   metaData <- covariateData$metaData
@@ -336,7 +415,7 @@ getPlpData <- function(
 print.plpData <- function(x, ...) {
   writeLines("plpData object")
   writeLines("")
-  writeLines(paste("At risk concept ID:", attr(x$cohorts, "metaData")$cohortId))
+  writeLines(paste("At risk concept ID:", attr(x$cohorts, "metaData")$targetId))
   writeLines(paste("Outcome concept ID(s):", paste(attr(x$outcomes, "metaData")$outcomeIds, collapse = ",")))
 }
 
@@ -366,7 +445,7 @@ summary.plpData <- function(object,...){
 print.summary.plpData <- function(x, ...) {
   writeLines("plpData object summary")
   writeLines("")
-  writeLines(paste("At risk cohort concept ID:", x$metaData$cohortId))
+  writeLines(paste("At risk cohort concept ID:", x$metaData$targetId))
   writeLines(paste("Outcome concept ID(s):", x$metaData$outcomeIds, collapse = ","))
   writeLines("")
   writeLines(paste("People:", paste(x$people)))
diff --git a/R/FeatureEngineering.R b/R/FeatureEngineering.R
index 24b2d28b3..a5f4bc1ca 100644
--- a/R/FeatureEngineering.R
+++ b/R/FeatureEngineering.R
@@ -16,6 +16,30 @@
 # limitations under the License.
 
 
+featureEngineer <- function(data, featureEngineeringSettings){
+  
+  ParallelLogger::logInfo('Starting Feature Engineering')
+  
+  # if a single setting, make it a list
+  if(inherits(featureEngineeringSettings, 'featureEngineeringSettings')){
+    featureEngineeringSettings <- list(featureEngineeringSettings)
+  }
+  
+  for(featureEngineeringSetting in featureEngineeringSettings){
+    fun <- attr(featureEngineeringSetting, "fun")
+    args <- list(trainData = data,
+                 featureEngineeringSettings = featureEngineeringSetting)
+    ParallelLogger::logInfo(paste0('Applying ',fun))
+    data <- do.call(eval(parse(text = fun)), args)
+  }
+  
+  attr(data, 'metaData')$featureEngineeringSettings <- featureEngineeringSettings
+  
+  ParallelLogger::logInfo('Done Feature Engineering')
+  
+  return(data)
+  
+}
 
 #' Create the settings for defining any feature engineering that will be done
 #'
@@ -23,7 +47,7 @@
 #' Returns an object of class \code{featureEngineeringSettings} that specifies the sampling function that will be called and the settings
 #'
 #' @param type              (character) Choice of:  \itemize{
-#'                                         \item{'none'}{ No feature engineering - this is the default }
+#'                                         \item'none' No feature engineering - this is the default 
 #'                                         } 
 #'
 #' @return
@@ -54,7 +78,11 @@ createFeatureEngineeringSettings <- function(type = 'none'){
 #' @export
 createUnivariateFeatureSelection <- function(k = 100){
   
-  checkIsClass(k, c('numeric','integer'))
+  if (inherits(k, 'numeric')) {
+    k <- as.integer(k)
+  }
+  
+  checkIsClass(k, 'integer')
   checkHigherEqual(k, 0)
   
   featureEngineeringSettings <- list(k = k) 
@@ -95,6 +123,296 @@ createRandomForestFeatureSelection <- function(ntrees = 2000, maxDepth = 17){
   return(featureEngineeringSettings)
 }
 
+#' Create the settings for adding a spline for continuous variables
+#'
+#' @details
+#' Returns an object of class \code{featureEngineeringSettings} that specifies the sampling function that will be called and the settings
+#'
+#' @param continousCovariateId     The covariateId to apply splines to
+#' @param knots            Either number of knots of vector of split values
+#' @param analysisId       The analysisId to use for the spline covariates  
+#'
+#' @return
+#' An object of class \code{featureEngineeringSettings}
+#' @export
+createSplineSettings <- function(
+    continousCovariateId,
+    knots,
+    analysisId = 683
+    ){
+  
+  checkIsClass(continousCovariateId, c('numeric','integer'))
+  checkIsClass(knots, c('numeric','integer'))
+  
+  featureEngineeringSettings <- list(
+    continousCovariateId = continousCovariateId,
+    knots = knots,
+    analysisId = analysisId
+  )
+  
+  attr(featureEngineeringSettings, "fun") <- "splineCovariates"
+  class(featureEngineeringSettings) <- "featureEngineeringSettings"
+  
+  return(featureEngineeringSettings)
+}
+  
+splineCovariates <- function(
+    trainData, 
+    featureEngineeringSettings,
+    knots = NULL
+    ){
+  
+  ParallelLogger::logInfo('Starting splineCovariates')
+  
+  if(is.null(knots)){
+    
+    if (length(featureEngineeringSettings$knots) == 1) {
+      measurements <- trainData$covariateData$covariates %>%
+        dplyr::filter(.data$covariateId == !!featureEngineeringSettings$continousCovariateId) %>%
+        as.data.frame()
+      knots <- measurements$covariateValue %>%
+        stats::quantile(seq(0.01, 0.99, length.out = featureEngineeringSettings$knots))
+    } else {
+      knots <- featureEngineeringSettings$knots
+    }
+  
+  }
+  
+  # apply the spline mapping  
+  trainData <- splineMap(
+    data = trainData,
+    covariateId = featureEngineeringSettings$continousCovariateId,
+    analysisId = featureEngineeringSettings$analysisId,
+    knots = knots
+    )
+
+  featureEngineering <- list(
+    funct = 'splineCovariates',
+    settings = list(
+      featureEngineeringSettings = featureEngineeringSettings,
+      knots = knots
+    )
+  )
+  
+  # add the feature engineering in
+  attr(trainData, 'metaData')$featureEngineering = listAppend(
+    attr(trainData, 'metaData')$featureEngineering,
+    featureEngineering
+  )
+  ParallelLogger::logInfo('Finished splineCovariates')
+  
+  return(trainData)
+}
+
+# create the spline map to add spline columns
+splineMap <- function(
+    data, 
+    covariateId,
+    analysisId,
+    knots
+){
+  
+  ParallelLogger::logInfo('Starting splineMap')
+  measurements <- data$covariateData$covariates %>%
+    dplyr::filter(.data$covariateId == !!covariateId) %>%
+    as.data.frame()
+  
+  designMatrix <- splines::bs(
+    x = measurements$covariateValue,#knots[1]:knots[length(knots)],
+    knots = knots[2:(length(knots) - 1)],
+    Boundary.knots = knots[c(1, length(knots))]
+  )
+  
+  data$covariateData$covariates <- data$covariateData$covariates %>%
+    dplyr::filter(.data$covariateId != !!covariateId)
+  
+  # get the covariate name
+  details <- data$covariateData$covariateRef %>%
+    dplyr::filter(.data$covariateId == !!covariateId) %>%
+    as.data.frame()
+  covariateName <- details$covariateName
+  
+  data$covariateData$covariateRef <- data$covariateData$covariateRef %>%
+    dplyr::filter(.data$covariateId != !!covariateId)
+  
+  # remove last 3 numbers as this was old analysis id
+  covariateId <- floor(covariateId/1000)
+  
+  # add the spline columns
+  for(i in 1:ncol(designMatrix)){
+    Andromeda::appendToTable(
+      tbl = data$covariateData$covariates, 
+      data = data.frame(
+        rowId = measurements$rowId,
+        covariateId = covariateId*10000+i*1000+analysisId,
+        covariateValue = designMatrix[,i]
+      )
+    )
+  }
+  
+  # add the covariates to the ref table
+  Andromeda::appendToTable(
+    tbl = data$covariateData$covariateRef,
+    data = data.frame(
+      covariateId = covariateId*10000+(1:(ncol(designMatrix)))*1000+analysisId,
+      covariateName = paste(
+        paste0(covariateName," spline component "),
+        1:ncol(designMatrix)
+      ),
+      conceptId = 0,
+      analysisId = analysisId
+    )
+  )
+  
+  # add analysisRef for the first time a spline is added
+  analysisRef <- data$covariateData$analysisRef %>% as.data.frame()
+  if(!analysisId  %in% analysisRef$analysisId){
+    Andromeda::appendToTable(
+      tbl = data$covariateData$analysisRef,
+      data = data.frame(
+        analysisId = analysisId,
+        analysisName = 'splines',
+        domainId = 'feature engineering',
+        startDay = 0,
+        endDay = 0,
+        isBinary = 'N',
+        missingMeansZero = 'N'
+      )
+    )
+  }
+  ParallelLogger::logInfo('Finished splineMap')
+  return(data)
+}
+
+
+
+#' Create the settings for adding a spline for continuous variables
+#'
+#' @details
+#' Returns an object of class \code{featureEngineeringSettings} that specifies how to do stratified imputation
+#'
+#' @param covariateId     The covariateId that needs imputed values
+#' @param ageSplits       A vector of age splits in years to create age groups
+#'
+#' @return
+#' An object of class \code{featureEngineeringSettings}
+#' @export
+createStratifiedImputationSettings <- function(
+    covariateId,
+    ageSplits = NULL
+){
+  
+  checkIsClass(covariateId, c('numeric','integer'))
+  checkIsClass(ageSplits, c('numeric','integer'))
+  
+  featureEngineeringSettings <- list(
+    covariateId = covariateId,
+    ageSplits = ageSplits
+  )
+  
+  attr(featureEngineeringSettings, "fun") <- "stratifiedImputeCovariates"
+  class(featureEngineeringSettings) <- "featureEngineeringSettings"
+  
+  return(featureEngineeringSettings)
+}
+
+stratifiedImputeCovariates <- function(
+    trainData, 
+    featureEngineeringSettings,
+    stratifiedMeans = NULL
+){
+  
+  if(is.null(stratifiedMeans)){
+    
+    stratifiedMeans <- calculateStratifiedMeans(
+      trainData = trainData,
+      featureEngineeringSettings = featureEngineeringSettings
+    )
+    
+  }
+  
+  trainData <- imputeMissingMeans(
+    trainData = trainData, 
+    covariateId = featureEngineeringSettings$covariateId,
+    ageSplits = featureEngineeringSettings$ageSplits,
+    stratifiedMeans = stratifiedMeans
+    )
+  
+  return(trainData)
+}
+
+calculateStratifiedMeans <- function(
+    trainData,
+    featureEngineeringSettings
+){
+  if(is.null(featureEngineeringSettings$ageSplits)){
+    trainData$cohorts$ageGroup <- floor(trainData$cohorts$ageYear/5)
+  } else{
+    trainData$cohorts$ageGroup <- rep(0, length(trainData$cohorts$ageYear))
+    for(i in 1:length(featureEngineeringSettings$ageSplits)){
+      trainData$cohorts$ageGroup[trainData$cohorts$ageYear > featureEngineeringSettings$ageSplits[i]] <- i
+    }
+  }
+  
+  trainData$covariateData$cohorts <- trainData$cohorts[,c('rowId', 'ageGroup', 'gender')]
+  
+  stratifiedMeans <- trainData$covariateData$covariates %>%
+    dplyr::filter(.data$covariateId == !!featureEngineeringSettings$covariateId) %>%
+    dplyr::inner_join(
+      y = trainData$covariateData$cohorts, 
+      by = c('rowId')
+    ) %>%
+    dplyr::group_by(.data$ageGroup, .data$gender) %>%
+    dplyr::summarise(covariateValue = mean(.data$covariateValue, na.rm = TRUE)) %>%
+    as.data.frame()
+  
+  return(stratifiedMeans)
+}
+
+imputeMissingMeans <- function(
+  trainData, 
+  covariateId,
+  ageSplits,
+  stratifiedMeans
+){
+  
+  if(is.null(ageSplits)){
+    trainData$cohorts$ageGroup <- floor(trainData$cohorts$ageYear/5)
+  } else{
+    trainData$cohorts$ageGroup <- rep(0, length(trainData$cohorts$ageYear))
+    for(i in 1:length(ageSplits)){
+      trainData$cohorts$ageGroup[trainData$cohorts$ageYear > ageSplits[i]] <- i
+    }
+  }
+  
+  rowIdsWithValues <- trainData$covariateData$covariates %>%
+    dplyr::filter(.data$covariateId == !! covariateId) %>%
+    dplyr::select('rowId') %>%
+    dplyr::pull()
+  rowIdsWithMissingValues <- trainData$cohorts$rowId[!trainData$cohorts$rowId %in% rowIdsWithValues]
+  
+
+  imputedData <- trainData$cohorts %>% 
+    dplyr::filter(.data$rowId %in% rowIdsWithMissingValues) %>%
+    dplyr::select('rowId', 'ageGroup', 'gender') %>%
+    dplyr::left_join(
+      y = stratifiedMeans, 
+      by = c('ageGroup', 'gender')
+      ) %>%
+    dplyr::mutate(
+      covariateId = !!covariateId,
+      covariateValue = .data$covariateValue
+      ) %>%
+    dplyr::select('rowId', 'covariateId', 'covariateValue')
+    
+  Andromeda::appendToTable(
+    tbl = trainData$covariateData$covariates, 
+    data = imputedData
+  )
+  
+  return(trainData)
+}
+
 univariateFeatureSelection <- function(
   trainData, 
   featureEngineeringSettings,
@@ -122,13 +440,15 @@ univariateFeatureSelection <- function(
     SelectKBest <- sklearn$feature_selection$SelectKBest
     chi2 <- sklearn$feature_selection$chi2
     
-    kbest <- SelectKBest(chi2, k = featureEngineeringSettings$k)$fit(X, y)
+    kbest <- SelectKBest(chi2, k = featureEngineeringSettings$k)$fit(X, y$outcomeCount)
     kbest$scores_ <- np$nan_to_num(kbest$scores_)
-    threshold <- -np$sort(-kbest$scores_)[(featureEngineeringSettings$k-1)]
-    
-    inc <- kbest$scores_ >= threshold
+
+    # taken from sklearn code, matches the application during transform call
+    k <- featureEngineeringSettings$k
+    mask <- np$zeros(length(kbest$scores_), dtype='bool')
+    mask[np$argsort(kbest$scores_, kind="mergesort")+1][(length(kbest$scores_)-k+1):length(kbest$scores_)] <- TRUE
     
-    covariateIdsInclude <- covariateMap[inc,]$covariateId
+    covariateIdsInclude <- covariateMap[mask,]$covariateId
   }
   
   trainData$covariateData$covariates <- trainData$covariateData$covariates %>% 
@@ -137,7 +457,7 @@ univariateFeatureSelection <- function(
   trainData$covariateData$covariateRef <- trainData$covariateData$covariateRef %>% 
     dplyr::filter(.data$covariateId %in% covariateIdsInclude)
   
-  featureEngeering <- list(
+  featureEngineering <- list(
     funct = 'univariateFeatureSelection',
     settings = list(
       featureEngineeringSettings = featureEngineeringSettings,
@@ -147,7 +467,7 @@ univariateFeatureSelection <- function(
   
   attr(trainData, 'metaData')$featureEngineering = listAppend(
     attr(trainData, 'metaData')$featureEngineering,
-    featureEngeering
+    featureEngineering
   )
   
   return(trainData)
@@ -184,7 +504,7 @@ randomForestFeatureSelection <- function(
     max_depth = featureEngineeringSettings$max_depth #17
     
     rf = sklearn$ensemble$RandomForestClassifier(
-      max_features = 'auto', 
+      max_features = 'sqrt', 
       n_estimators = as.integer(ntrees),
       max_depth = as.integer(max_depth),
       min_samples_split = as.integer(2), 
@@ -226,25 +546,4 @@ randomForestFeatureSelection <- function(
 
 
 
-featureEngineer <- function(data, featureEngineeringSettings){
-  
-  ParallelLogger::logInfo('Starting Feature Engineering')
-  
-  # if a single setting, make it a list
-  if(class(featureEngineeringSettings) == 'featureEngineeringSettings'){
-    featureEngineeringSettings <- list(featureEngineeringSettings)
-  }
-  
-  for(featureEngineeringSetting in featureEngineeringSettings){
-    fun <- attr(featureEngineeringSetting, "fun")
-    args <- list(trainData = data,
-                 featureEngineeringSettings = featureEngineeringSetting)
-    ParallelLogger::logInfo(paste0('Applying ',fun))
-    data <- do.call(eval(parse(text = fun)), args)
-  }
-  
-  ParallelLogger::logInfo('Done Feature Engineering')
-  
-  return(data)
-  
-}
+
diff --git a/R/FeatureImportance.R b/R/FeatureImportance.R
index 7d2b934a6..6c73b1223 100644
--- a/R/FeatureImportance.R
+++ b/R/FeatureImportance.R
@@ -49,26 +49,37 @@ pfi <- function(plpResult, population, plpData, repeats = 1,
   if(is.null(covariates)){
     covariates <- plpResult$model$covariateImportance %>% 
       dplyr::filter(.data$covariateValue != 0) %>% 
-      dplyr::select(.data$covariateId) %>% 
+      dplyr::select("covariateId") %>% 
       dplyr::pull()
   }
   
   # add code to format covariateData based on plpModel
-  # do feature engineering/selection
-  plpData$covariateData <- do.call(
-    applyFeatureengineering, 
-    list(covariateData = plpData$covariateData,
-      settings = plpResult$model$settings$featureEngineering
+  
+  if(!is.null(plpResult$model$preprocessing$featureEngineering)){
+    # do feature engineering/selection
+    ParallelLogger::logInfo('Running FE in model')
+    plpData <- do.call(
+      applyFeatureengineering, 
+      list(plpData = plpData,
+           settings = plpResult$model$preprocessing$featureEngineering
+      )
     )
-  )
+  } else{
+    ParallelLogger::logInfo('No FE in model')
+  }
   
-  # do preprocessing
-  plpData$covariateData <- do.call(
-    applyTidyCovariateData, 
-    list(covariateData = plpData$covariateData,
-      preprocessSettings = plpResult$model$settings$tidyCovariates
+  if(!is.null(plpResult$model$preprocessing$tidyCovariates)){
+    # do preprocessing
+    ParallelLogger::logInfo('Applying data tidying done in model')
+    plpData$covariateData <- do.call(
+      applyTidyCovariateData, 
+      list(covariateData = plpData$covariateData,
+           preprocessSettings = plpResult$model$preprocessing$tidyCovariates
+      )
     )
-  )
+  } else{
+    ParallelLogger::logInfo('No data tidying done in model')
+  }
   
   # apply prediction function
   pred <- do.call(
@@ -94,23 +105,20 @@ pfi <- function(plpResult, population, plpData, repeats = 1,
     ParallelLogger::logInfo(paste0('Using all ', cores))
     ParallelLogger::logInfo(paste0('Set cores input to use fewer...'))
   }
-  
+  getVpiSettings <- function(i) {
+    result <- list(plpModel = plpResult$model, 
+                   population = population, 
+                   plpDataLocation = plpDataLocation,
+                   covariateId = covariates[i],
+                   repeats = repeats)
+    return(result)
+  }
+  if (cores > 1) {
   cluster <- ParallelLogger::makeCluster(numberOfThreads = cores)
   ParallelLogger::clusterRequire(cluster, c("PatientLevelPrediction", "Andromeda"))
   
-  
-  getVpiSettings <- function(i){
-    result <-list(plpModel = plpResult$model, 
-                  population = population, 
-                  plpDataLocation = plpDataLocation,
-                  covariateId = covariates[i],
-                  repeats = repeats)
-    return(result)
-  }
   vpiSettings <- lapply(1:length(covariates), getVpiSettings)
 
-  
-  #lapply(vpiSettings, function(x) do.call(permutePerf, x))
   aucP <- ParallelLogger::clusterApply(cluster = cluster, 
                                                 x = vpiSettings, 
                                                 fun = permutePerf, 
@@ -118,13 +126,15 @@ pfi <- function(plpResult, population, plpData, repeats = 1,
                                                 progressBar = TRUE)
   ParallelLogger::stopCluster(cluster)
   
+  } else {
+    ParallelLogger::logInfo("Running in serial")
+    aucP <- lapply(1:length(covariates), function(i) {
+      permutePerf(getVpiSettings(i))
+    })
+  }
   aucP <- do.call(c, aucP)
-  
-  # do this in parellel
-
   varImp <- data.frame(covariateId = covariates,
-                       pfi = auc-aucP)
-  
+                       pfi = auc - aucP)
   return(varImp)
   
 }
@@ -178,7 +188,7 @@ permute <- function(plpDataLocation,cId,population){
   #get analysisId
   aId <- plpData$covariateData$covariateRef %>% 
     dplyr::filter(.data$covariateId == !!cId) %>%
-    dplyr::select(.data$analysisId) %>% dplyr::collect()
+    dplyr::select("analysisId") %>% dplyr::collect()
   
   # if analysis id is not 3 (age group), 4 (race) or 5 (ethnicity)
   if(!aId$analysisId %in% c(3,4,5)){
@@ -189,7 +199,7 @@ permute <- function(plpDataLocation,cId,population){
   
     # find a new random selection of people and give them the covariate and value
     newPlp <- sample(population$rowId,nSamp)
-    newData <- tibble::as_tibble(cbind(rowId = newPlp,coi[,-1]))
+    newData <- dplyr::as_tibble(cbind(rowId = newPlp,coi[,-1]))
     
     # swap old covariate data with new
     plpData$covariateData$covariates <- plpData$covariateData$covariates %>% dplyr::filter(.data$covariateId != !!cId) %>% dplyr::collect()
@@ -204,20 +214,20 @@ permute <- function(plpDataLocation,cId,population){
     
     # sample the pop to replace 
     swapPlp <- sample(population$rowId,nSamp)
-    haveCidDataSwapped <- tibble::as_tibble(cbind(rowId = swapPlp,haveCidData[,-1]))
+    haveCidDataSwapped <- dplyr::as_tibble(cbind(rowId = swapPlp,haveCidData[,-1]))
     
     # find the swapped people to switch 
     connectedCovs <- plpData$covariateData$covariateRef %>% 
       dplyr::filter(.data$analysisId == !!aId$analysisId) %>% 
       dplyr::group_by(.data$covariateId) %>% 
-      dplyr::select(.data$covariateId) %>% 
+      dplyr::select("covariateId") %>% 
       dplyr::collect()
     plpToSwap <- plpData$covariateData$covariates %>% 
       dplyr::filter(.data$covariateId %in% !!connectedCovs$covariateId) %>% 
       dplyr::filter(.data$rowId %in% swapPlp) %>% 
       dplyr::collect()
     
-    swappedForCid <- tibble::as_tibble(cbind(rowId = haveCidData$rowId[1:nrow(plpToSwap)],plpToSwap[,-1]))
+    swappedForCid <- dplyr::as_tibble(cbind(rowId = haveCidData$rowId[1:nrow(plpToSwap)],plpToSwap[,-1]))
     
 
     # swap old covariate data with new
diff --git a/R/Fit.R b/R/Fit.R
index a0a788f3b..30ef99710 100644
--- a/R/Fit.R
+++ b/R/Fit.R
@@ -29,24 +29,23 @@
 #'                                         data extracted from the CDM.
 #' @param modelSettings                    An object of class \code{modelSettings} created using one of the function:
 #'                                         \itemize{
-#'                                         \item{logisticRegressionModel()}{ A lasso logistic regression model}
-#'                                         \item{GBMclassifier()}{ A gradient boosting machine}
-#'                                         \item{RFclassifier()}{ A random forest model}
-#'                                         \item{GLMclassifier ()}{ A generalised linear model}
-#'                                         \item{KNNclassifier()}{ A KNN model}
+#'                                         \item setLassoLogisticRegression() A lasso logistic regression model
+#'                                         \item setGradientBoostingMachine() A gradient boosting machine
+#'                                         \item setRandomForest() A random forest model
+#'                                         \item setKNN() A KNN model
 #'                                         }
 #' @param search                           The search strategy for the hyper-parameter selection (currently not used)                                        
 #' @param analysisId                       The id of the analysis
+#' @param analysisPath                     The path of the analysis
 #' @return
 #' An object of class \code{plpModel} containing:
 #' 
 #' \item{model}{The trained prediction model}
-#' \item{modelLoc}{The path to where the model is saved (if saved)}
-#' \item{trainAuc}{The AUC obtained on the training set}
-#' \item{trainCalibration}{The calibration obtained on the training set}
-#' \item{modelSettings}{A list specifiying the model, preprocessing, outcomeId and cohortId}
-#' \item{metaData}{The model meta data}
-#' \item{trainingTime}{The time taken to train the classifier}
+#' \item{preprocessing}{The preprocessing required when applying the model}
+#' \item{prediction}{The cohort data.frame with the predicted risk column added}
+#' \item{modelDesign}{A list specifiying the modelDesign settings used to fit the model}
+#' \item{trainDetails}{The model meta data}
+#' \item{covariateImportance}{The covariate importance for the model}
 #'
 #'
 #' @export
@@ -54,7 +53,8 @@ fitPlp <- function(
   trainData,   
   modelSettings,
   search = "grid",
-  analysisId
+  analysisId,
+  analysisPath
   )
   {
   
@@ -75,13 +75,18 @@ fitPlp <- function(
   fun <- eval(parse(text = modelSettings$fitFunction))
   args <- list(
     trainData = trainData,
-    param = modelSettings$param,
+    modelSettings, # old: param = modelSettings$param, # make this model settings?
     search = search,
-    analysisId = analysisId
+    analysisId = analysisId,
+    analysisPath = analysisPath
     )
   plpModel <- do.call(fun, args)
   ParallelLogger::logTrace('Returned from classifier function')
   
+  # adding trainDetails databaseId to all classifiers
+  # TODO - move other details into fit
+  plpModel$trainDetails$developmentDatabaseId = attr(trainData, "metaData")$cdmDatabaseId
+  
   class(plpModel) <- 'plpModel'
   
   return(plpModel)
diff --git a/R/Formatting.R b/R/Formatting.R
index 1aa584741..1e545b818 100644
--- a/R/Formatting.R
+++ b/R/Formatting.R
@@ -86,12 +86,12 @@ toSparseM <- function(plpData, cohort = NULL, map=NULL){
 
   ParallelLogger::logInfo(paste0('toSparseM non temporal used'))
     
-  checkRam(newcovariateData, 0.9)  # estimates size of RAM required and makes sure it is less that 90%
+  checkRam(newcovariateData)  # estimates size of RAM required and prints it
     
   data <- Matrix::sparseMatrix(
-    i = newcovariateData$covariates %>% dplyr::select(.data$rowId) %>% dplyr::pull(),
-    j = newcovariateData$covariates %>% dplyr::select(.data$columnId) %>% dplyr::pull(),
-    x = newcovariateData$covariates %>% dplyr::select(.data$covariateValue) %>% dplyr::pull(),
+    i = newcovariateData$covariates %>% dplyr::select("rowId") %>% dplyr::pull(),
+    j = newcovariateData$covariates %>% dplyr::select("columnId") %>% dplyr::pull(),
+    x = newcovariateData$covariates %>% dplyr::select("covariateValue") %>% dplyr::pull(),
     dims=c(maxX,maxY)
   )
     
@@ -108,9 +108,13 @@ toSparseM <- function(plpData, cohort = NULL, map=NULL){
   return(result)
 }
 
-# this functions takes covariate data and a cohort/population and remaps 
-# the covaiate and row ids 
-# restricts to pop and saves/creates mapping
+#' Map covariate and row Ids so they start from 1
+#' @description this functions takes covariate data and a cohort/population and remaps 
+#' the covariate and row ids, restricts to pop and saves/creates mapping
+#' @param covariateData a covariateData object
+#' @param cohort        if specified rowIds restricted to the ones in cohort
+#' @param mapping       A pre defined mapping to use 
+#' @export
 MapIds <- function(
   covariateData,
   cohort = NULL,
@@ -123,7 +127,7 @@ MapIds <- function(
   # change the rowIds in cohort (if exists)
   if(!is.null(cohort)){
     rowMap <- data.frame(
-      rowId = cohort %>% dplyr::select(.data$rowId)
+      rowId = cohort %>% dplyr::select("rowId")
     )
     rowMap$xId <- 1:nrow(rowMap)
   } else{
@@ -166,17 +170,16 @@ MapIds <- function(
     newCovariateData$rowMap <- rowMap
     newCovariateData$covariates <- newCovariateData$covariates %>%
       dplyr::inner_join(newCovariateData$rowMap, by = 'rowId') %>% 
-      dplyr::select(- .data$rowId) %>%
-      dplyr::rename(rowId = .data$xId)
+      dplyr::select(- "rowId") %>%
+      dplyr::rename(rowId = "xId")
     
     if(!is.null(cohort)){
       # change the rowId in labels
       newCovariateData$cohort <- cohort %>%
         dplyr::inner_join(rowMap, by = 'rowId') %>% 
-        #dplyr::select(- .data$rowId) %>%
         dplyr::rename(
-          originalRowId = .data$rowId,
-          rowId = .data$xId
+          originalRowId = "rowId",
+          rowId = "xId"
           ) %>%
         dplyr::arrange(.data$rowId)  # make sure it is ordered lowest to highest
     }
@@ -188,22 +191,12 @@ MapIds <- function(
   return(newCovariateData)
 }
 
-checkRam <- function(covariateData, maxPercent){
-  
-  ensure_installed('memuse')
+checkRam <- function(covariateData){
   
   nrowV <- covariateData$covariates %>% dplyr::summarise(size = dplyr::n()) %>% dplyr::collect()
   estRamB <- (nrowV$size/1000000*24000984)
   
-  ramFree <- memuse::Sys.meminfo()
-  ramFree <- as.double(ramFree$freeram)
-  
-  if(0.9*ramFree<estRamB){
-    ParallelLogger::logWarn('plpData size bigger than current available RAM')
-    stop('Insufficient RAM')
-  } else{
-    ParallelLogger::logInfo(paste0('plpData size estimated to use ',round(estRamB/ramFree*100,2),'% of available RAM (', round(estRamB/1000000000,1), 'GBs)'))
-  }
+  ParallelLogger::logInfo(paste0('plpData size estimated to use ', round(estRamB/1000000000,1), 'GBs of RAM'))
   
   return(invisible(TRUE))
 }
diff --git a/R/GradientBoostingMachine.R b/R/GradientBoostingMachine.R
index 0e5885d9d..7035ee44f 100644
--- a/R/GradientBoostingMachine.R
+++ b/R/GradientBoostingMachine.R
@@ -43,61 +43,76 @@ setGradientBoostingMachine <- function(ntrees=c(100, 300), nthread=20, earlyStop
   
   checkIsClass(seed, c('numeric', 'integer'))
   
-  if(length(nthread)>1)
+  if(length(nthread)>1){
     stop(paste('nthreads must be length 1'))
-  if(!class(seed)%in%c('numeric','NULL', 'integer'))
+  }
+  if(!inherits(x = seed, what = c('numeric','NULL', 'integer'))){
     stop('Invalid seed')
-  if(!class(ntrees) %in% c("numeric", "integer"))
-    stop('ntrees must be a numeric value >0 ')
-  if(sum(ntrees < 1)>0)
-    stop('ntrees must be greater that 0 or -1')
-  if(!class(maxDepth) %in% c("numeric", "integer"))
-    stop('maxDepth must be a numeric value >0')
-  if(sum(maxDepth < 1)>0)
+  }
+  if(!inherits(x = ntrees, what =  c("numeric", "integer"))){
+    stop('ntrees must be a numeric value ')
+  }
+  if(sum(ntrees < 1)  > 0 ){
+    stop('ntrees must be greater than 0 or -1')
+  }
+  if(!inherits(x = maxDepth, what =  c("numeric", "integer"))){
+    stop('maxDepth must be a numeric value')
+    }
+  if(sum(maxDepth < 1) > 0){
     stop('maxDepth must be greater that 0')
-  if(!class(minChildWeight) %in% c("numeric", "integer"))
-    stop('minChildWeight must be a numeric value >0')
-  if(sum(minChildWeight < 0)>0)
+  }
+  if(!inherits(x = minChildWeight, what =  c("numeric", "integer"))){
+    stop('minChildWeight must be a numeric value')
+  }
+  if(sum(minChildWeight < 0) > 0){
     stop('minChildWeight must be greater that 0')
-  if(class(learnRate)!='numeric')
-    stop('learnRate must be a numeric value >0 and <= 1')
-  if(sum(learnRate <= 0)>0)
+  }
+  if(!inherits(x = learnRate, what = 'numeric')){
+    stop('learnRate must be a numeric value')
+  }
+  if(sum(learnRate <= 0) > 0){
     stop('learnRate must be greater that 0')
-  if(sum(learnRate > 1)>0)
+  }
+  if(sum(learnRate > 1) > 0){
     stop('learnRate must be less that or equal to 1')
-  if(!class(earlyStopRound) %in% c("numeric", "integer", "NULL"))
+  }
+  if(!inherits(x = earlyStopRound, what =  c("numeric", "integer", "NULL"))){
     stop('incorrect class for earlyStopRound')
-  if (sum(lambda < 0)>0)
+  }
+  if (!inherits(x = lambda, what =  c("numeric", "integer"))){
+    stop('lambda must be a numeric value')
+  }
+  if (sum(lambda < 0) > 0){
     stop('lambda must be 0 or greater')
-  if (!class(lambda) %in% c("numeric", "integer"))
-    stop('lambda must be a numeric value >= 0')
-  if (sum(alpha < 0)>0)
+  }
+  if(!inherits(x = alpha, what =  c("numeric", "integer"))){
+    stop('alpha must be a numeric value')
+  }
+  if (sum(alpha < 0) > 0){
     stop('alpha must be 0 or greater')
-  if (!class(alpha) %in% c("numeric", "integer"))
-    stop('alpha must be a numeric value >= 0')
-  if (sum(scalePosWeight < 0)>0)
-    stop('scalePosWeight must be 0 or greater')
-  if (!class(scalePosWeight) %in% c("numeric", "integer"))
+  }
+  if (!inherits(x = scalePosWeight, what =  c("numeric", "integer"))){
     stop('scalePosWeight must be a numeric value >= 0')
+  }
+  if (sum(scalePosWeight < 0) > 0){
+    stop('scalePosWeight must be 0 or greater')
+  }
+
+  paramGrid <- list(
+      ntrees = ntrees, 
+      earlyStopRound = earlyStopRound,
+      maxDepth = maxDepth, 
+      minChildWeight = minChildWeight, 
+      learnRate = learnRate,
+      lambda = lambda,
+      alpha = alpha,
+      scalePosWeight = scalePosWeight
+      )
   
-  
-  param <- split(
-    expand.grid(
-      ntrees=ntrees, 
-      earlyStopRound=earlyStopRound,
-      maxDepth=maxDepth, 
-      minChildWeight=minChildWeight, 
-      learnRate=learnRate,
-      lambda=lambda,
-      alpha=alpha,
-      scalePosWeight=scalePosWeight
-      ),
-    1:(length(ntrees)*length(maxDepth)*length(minChildWeight)*length(learnRate)*
-         length(earlyStopRound)*length(lambda)*length(alpha)*length(scalePosWeight))
-  )
+  param <- listCartesian(paramGrid)
   
   attr(param, 'settings') <- list(
-    modeType = 'Xgboost',
+    modelType = 'Xgboost',
     seed = seed[[1]],
     modelName = "Gradient Boosting Machine",
     threads = nthread[1],
@@ -131,8 +146,8 @@ varImpXgboost <- function(
   varImp <- merge(covariateMap, varImp, by.x='columnId', by.y='Feature')
   varImp <- varImp %>% 
     dplyr::mutate(included = 1) %>%
-    dplyr::rename(covariateValue = .data$Gain) %>% 
-    dplyr::select(.data$covariateId, .data$covariateValue, .data$included)
+    dplyr::rename(covariateValue = "Gain") %>% 
+    dplyr::select("covariateId", "covariateValue", "included")
   
   return(varImp)
   
@@ -144,13 +159,13 @@ predictXgboost <- function(
   cohort
   ){
   
-  if(class(data) == 'plpData'){
+  if(inherits(data , 'plpData')){
     # convert
     matrixObjects <- toSparseM(
       plpData = data, 
       cohort = cohort,
       map = plpModel$covariateImportance %>% 
-        dplyr::select(.data$columnId, .data$covariateId)
+        dplyr::select("columnId", "covariateId")
     )
     
     # use the include??
@@ -162,7 +177,7 @@ predictXgboost <- function(
     newData <- data
   }
   
-  if(class(plpModel) == 'plpModel'){
+  if(inherits(plpModel, 'plpModel')){
     model <- plpModel$model
   } else{
     model <- plpModel
@@ -175,8 +190,8 @@ predictXgboost <- function(
   # fix the rowIds to be the old ones?
   # now use the originalRowId and remove the matrix rowId
   prediction <- prediction %>% 
-    dplyr::select(-.data$rowId) %>%
-    dplyr::rename(rowId = .data$originalRowId)
+    dplyr::select(-"rowId") %>%
+    dplyr::rename(rowId = "originalRowId")
 
   attr(prediction, "metaData") <- list(modelType = attr(plpModel, "modelType"))
   
@@ -213,7 +228,7 @@ fitXgboost <- function(
   outcomes <- sum(labels$outcomeCount>0)
   N <- nrow(labels)
   outcomeProportion <- outcomes/N
-  
+  set.seed(settings$seed)
   model <- xgboost::xgb.train(
     data = train, 
     params = list(
diff --git a/R/HelperFunctions.R b/R/HelperFunctions.R
index c18e8e153..d73a4e9f0 100644
--- a/R/HelperFunctions.R
+++ b/R/HelperFunctions.R
@@ -1,14 +1,25 @@
 # fix issue with nrow - temp fix for me locally
 nrow <- function(x){UseMethod("nrow",x)}
+#' @exportS3Method NULL
 nrow.default <- base::nrow
+#' @exportS3Method NULL
 nrow.tbl <- function(x){x %>% dplyr::tally() %>% dplyr::pull()}
 
 
+removeInvalidString <- function(string){
+  modString <- gsub('_', ' ', string)
+  modString <- gsub('\\.', ' ', modString)
+  modString <- gsub("[[:punct:]]", "", modString)
+  modString <- gsub(' ', '_', modString)
+  return(modString)
+}
+
+
 # Borrowed from devtools: https://github.com/hadley/devtools/blob/ba7a5a4abd8258c52cb156e7b26bb4bf47a79f0b/R/utils.r#L44
-is_installed <- function (pkg, version = 0) {
+is_installed <- function (pkg) {
   installed_version <- tryCatch(utils::packageVersion(pkg), 
                                 error = function(e) NA)
-  !is.na(installed_version) && installed_version >= version
+  !is.na(installed_version)
 }
 
 # Borrowed and adapted from devtools: https://github.com/hadley/devtools/blob/ba7a5a4abd8258c52cb156e7b26bb4bf47a79f0b/R/utils.r#L74
@@ -18,7 +29,7 @@ ensure_installed <- function(pkg) {
     if (interactive()) {
       message(msg, "\nWould you like to install it?")
       if (utils::menu(c("Yes", "No")) == 1) {
-        if(pkg%in%c('BigKnn', "IterativeHardThresholding")){
+        if(pkg%in%c('BigKnn', "IterativeHardThresholding", "ShinyAppBuilder", "ResultModelManager")){
           
           # add code to check for devtools...
           dvtCheck <- tryCatch(utils::packageVersion('devtools'), 
@@ -40,10 +51,13 @@ ensure_installed <- function(pkg) {
   }
 }
 
+#' Create a temporary model location
+#' 
+#' @export
 createTempModelLoc <- function(){
   repeat{
-    loc <- paste(tempdir(), paste0('python_models_',sample(10002323,1)), sep = '\\')
-    #loc <- file.path(tempdir(), paste0('python_models_',sample(10002323,1)))
+    ##loc <- paste(tempdir(), paste0('python_models_',sample(10002323,1)), sep = '\\')
+    loc <- file.path(tempdir(), paste0('python_models_',sample(10002323,1)))
     if(!dir.exists(loc)){
       return(loc)
     }
@@ -82,45 +96,48 @@ listAppend <- function(a, b){
 #'
 #' @param envname   A string for the name of the virtual environment (default is 'PLP') 
 #' @param envtype   An option for specifying the environment as'conda' or 'python'.  If NULL then the default is 'conda' for windows users and 'python' for non-windows users 
+#' @param condaPythonVersion String, Python version to use when creating a conda environment
 #'
 #' @export
-configurePython <- function(envname='PLP', envtype=NULL){
+configurePython <- function(envname='PLP', envtype=NULL, condaPythonVersion="3.11"){
   
   if(is.null(envtype)){
     if(getOs()=='windows'){
-      envtype=='conda'
+      envtype <- "conda"
     } else {
-      envtype=='python'
+      envtype <- "python"
     }
   }
   
   if(envtype=='conda'){
     pEnvironments <- reticulate::conda_list()
     if(length(pEnvironments) > 0 && envname %in% pEnvironments$name){
-      warning(paste0('Conda environment ', envname,' exists.  You can use removePython() to remove if you want to fresh config'))
+      location <- ''
+      warning(paste0('Conda environment ', envname,' exists.  You can use reticulate::conda_remove() to remove if you want to fresh config'))
     } else {
       ParallelLogger::logInfo(paste0('Creating virtual conda environment called ', envname))
-      location <- reticulate::conda_create(envname=envname, packages = "python", conda = "auto")
+      location <- reticulate::conda_create(envname=envname, packages = paste0("python==", condaPythonVersion), conda = "auto")
     }
-    packages <- c('numpy','scipy','scikit-learn', 'pandas','pydotplus','joblib', 'sklearn-json')
+    packages <- c('numpy','scipy','scikit-learn', 'pandas','pydotplus','joblib')
     ParallelLogger::logInfo(paste0('Adding python dependancies to ', envname))
     reticulate::conda_install(envname=envname, packages = packages, forge = TRUE, pip = FALSE,
                               pip_ignore_installed = TRUE, conda = "auto")
   } else {
     pEnvironments <- reticulate::virtualenv_list()
     if(length(pEnvironments) > 0 && envname %in% pEnvironments){
-      warning(paste0('Python environment ', envname,' exists.  You can use removePython() to remove if you want to fresh config'))
+      location <- ''
+      warning(paste0('Python environment ', envname,' exists.'))
     } else {
       ParallelLogger::logInfo(paste0('Creating virtual python environment called ', envname))
       location <- reticulate::virtualenv_create(envname=envname)
     }
-    packages <- c('numpy', 'scikit-learn','scipy', 'pandas','pydotplus','sklearn-json')
+    packages <- c('numpy', 'scikit-learn','scipy', 'pandas','pydotplus')
     ParallelLogger::logInfo(paste0('Adding python dependancies to ', envname))
     reticulate::virtualenv_install(envname=envname, packages = packages, 
                                    ignore_installed = TRUE)
   }
   
-  return(location)
+  return(invisible(location))
 }
 
 #' Use the virtual environment created using configurePython()
@@ -176,3 +193,100 @@ getOs <- function(){
   tolower(os)
 }
 
+
+# Borrowed and adapted from Hmisc: https://github.com/harrelfe/Hmisc/blob/39011dae3af3c943e67401ed6000644014707e8b/R/cut2.s
+cut2 <- function(x, g, m = 150, digits = 3) {
+
+  method <- 1 ## 20may02
+  x.unique <- sort(unique(c(x[!is.na(x)])))
+  min.dif <- min(diff(x.unique))/2
+  min.dif.factor <- 1
+
+  oldopt <- options('digits')
+  options(digits=digits)
+  on.exit(options(oldopt))
+
+  xlab <- attr(x, 'label')
+
+  nnm <- sum(!is.na(x))
+  if(missing(g)) g <- max(1,floor(nnm/m))
+  if(g < 1)
+    stop('g must be >=1, m must be positive')
+
+  options(digits=15)
+  n <- table(x)
+  xx <- as.double(names(n))
+  options(digits = digits)
+  cum <- cumsum(n)
+  m <- length(xx)
+
+  y <- as.integer(ifelse(is.na(x),NA,1))
+  labs <- character(g)
+  cuts <- stats::approx(cum, xx, xout=(1:g)*nnm/g,
+    method='constant', rule=2, f=1)$y
+  cuts[length(cuts)] <- max(xx)
+  lower <- xx[1]
+  upper <- 1e45
+  up <- low <- double(g)
+  i <- 0
+  for(j in 1:g) {
+    cj <- if(method==1 || j==1) cuts[j] else {
+      if(i==0)
+        stop('program logic error')
+      # Not used unique values found in table(x)
+      s <- if(is.na(lower)) FALSE else xx >= lower
+      cum.used <- if(all(s)) 0 else max(cum[!s])
+      if(j==m) max(xx) else if(sum(s)<2) max(xx) else
+                                                   stats::approx(cum[s]-cum.used, xx[s], xout=(nnm-cum.used)/(g-j+1),
+                                                     method='constant', rule=2, f=1)$y
+    }
+    
+    if(cj==upper) next
+    
+    i <- i + 1
+    upper <- cj
+    # assign elements to group i
+    # y contains the group number in the end
+    y[x >= (lower-min.dif.factor*min.dif)]  <- i
+    low[i] <- lower
+    lower <- if(j==g) upper else min(xx[xx > upper])
+    
+    if(is.na(lower)) lower <- upper
+    
+    up[i]  <- lower
+  }
+  
+  low  <- low[1:i]
+  up   <- up[1:i]
+  # Are the bounds different?
+  variation <- logical(i)
+  for(ii in 1:i) {
+    r <- range(x[y==ii], na.rm=TRUE)
+    variation[ii] <- diff(r) > 0
+  }
+  flow <- do.call(format,c(list(low), digits = 3))
+  fup  <- do.call(format,c(list(up),  digits = 3))
+  bb   <- c(rep(')',i-1),']')
+  labs <- ifelse(low==up | (!variation), flow,
+    paste('[',flow,',',fup,bb,sep=''))
+  ss <- y==0 & !is.na(y)
+  if(any(ss))
+    stop(paste('categorization error in cut2.  Values of x not appearing in any interval:\n',
+      paste(format(x[ss],digits=12),collapse=' '),
+      '\nLower endpoints:',
+      paste(format(low,digits=12), collapse=' '),
+      '\nUpper endpoints:',
+      paste(format(up,digits=12),collapse=' ')))
+
+  y <- structure(y, class='factor', levels=labs)
+
+  attr(y,'class') <- "factor"
+  if(length(xlab)){
+    #label(y) <- xlab  # what is label?
+    # think the below does the same as the line above
+    class(y) <- 'labelled'
+    attr(y, 'label') <- xlab
+  }
+
+  return(y)
+}
diff --git a/R/ImportFromCsv.R b/R/ImportFromCsv.R
new file mode 100644
index 000000000..61536acd3
--- /dev/null
+++ b/R/ImportFromCsv.R
@@ -0,0 +1,584 @@
+#' Function to insert results into a database from csvs
+#' @description
+#' This function converts a folder with csv results into plp objects and loads them into a plp result database
+#'
+#' @details
+#' The user needs to have plp csv results in a single folder and an existing plp result database  
+#' 
+#' @param csvFolder        The location to the csv folder with the plp results
+#' @param connectionDetails             A connection details for the plp results database that the csv results will be inserted into
+#' @param databaseSchemaSettings       A object created by \code{createDatabaseSchemaSettings} with all the settings specifying the result tables to insert the csv results into  
+#' @param modelSaveLocation        The location to save any models from the csv folder - this should be the same location you picked when inserting other models into the database
+#' @param csvTableAppend       A string that appends the csv file names                       
+#' 
+#' @return
+#' Returns a data.frame indicating whether the results were inported into the database
+#' 
+#' @export
+insertCsvToDatabase <- function(
+  csvFolder,
+  connectionDetails,
+  databaseSchemaSettings,
+  modelSaveLocation,
+  csvTableAppend = ''
+){
+  
+  ensure_installed('readr')
+  
+  ParallelLogger::logInfo('Starting input checks')
+  
+  csvFileNames <- tryCatch({
+    dir(csvFolder, pattern = 'csv')
+  }, 
+  error = function(e){ParallelLogger::logInfo(e); return(NULL)}
+  )
+  if(is.null(csvFileNames)){
+    return(invisible(NULL))
+  }
+  
+  if(!missing(csvTableAppend)){ 
+    csvFileNamesNoAppend <- sub(csvTableAppend, '', csvFileNames)
+  } else{
+    csvFileNamesNoAppend <- csvFileNames
+    }
+  
+  # check all tables are in folder
+  # settings/resultsDataModelSpecification.csv table_name
+  resultNames <- paste0(unique(
+    readr::read_csv(
+      system.file(
+        'settings',
+        'resultsDataModelSpecification.csv',
+        package = "PatientLevelPrediction"
+      )
+    )$table_name
+  ), '.csv')
+  if(sum(csvFileNamesNoAppend %in% resultNames) != length(resultNames)){
+    missingTables <- paste(resultNames[!resultNames %in% csvFileNamesNoAppend], collapse = ',')
+    ParallelLogger::logInfo(paste0('CSV folder missing these tables: ', missingTables))
+    return(invisible(NULL))
+  }
+ 
+  alltables <- getTableNamesPlp(
+    connectionDetails = connectionDetails,
+    databaseSchema = databaseSchemaSettings$resultSchema
+  )
+  
+  if(!tolower(paste0(databaseSchemaSettings$tablePrefix,'PERFORMANCES')) %in% alltables){
+    ParallelLogger::logInfo(
+      paste0(
+        'performance table: ',paste0(toupper(databaseSchemaSettings$tablePrefix),'PERFORMANCES'),' not found, result database only contains ', 
+        paste(alltables, collapse =  ',')
+        )
+      )
+    return(invisible(NULL))
+  }
+  
+  ParallelLogger::logInfo('Input checks passed')
+  ParallelLogger::logInfo('Extracting cohort definitions')
+  # create cohortDefinitions:
+  cohortDefinitions <- extractCohortDefinitionsCSV(
+    csvFolder = csvFolder
+  )
+    
+  ParallelLogger::logInfo('Extracting database details')
+  # create databaseList
+  databaseList <- extractDatabaseListCSV(
+    csvFolder = csvFolder
+  )
+  
+  ParallelLogger::logInfo('Extracting performance ids')
+  performanceIds <- readr::read_csv(file.path(csvFolder, csvFileNames[grep('performances', csvFileNames)]))$performance_id
+    
+  if(length(performanceIds) > 0 ){
+    for(performanceId in performanceIds){  
+      ParallelLogger::logInfo(
+        paste0(
+          'Converting and inserting performance id', 
+          performanceId
+        )
+      )
+      # convert to runPlp
+      runPlp <- extractObjectFromCsv(
+        performanceId = performanceId, 
+        csvFolder = csvFolder
+      )
+      
+      # load into database
+      addRunPlpToDatabase(
+        runPlp = runPlp,
+        connectionDetails = connectionDetails,
+        databaseSchemaSettings = databaseSchemaSettings,
+        cohortDefinitions = cohortDefinitions,
+        modelSaveLocation = modelSaveLocation,
+        databaseList = databaseList
+      )
+    }
+  }
+  
+  diagnosticIds <- readr::read_csv(file.path(csvFolder, csvFileNames[grep('diagnostics', csvFileNames)]))$diagnostic_id
+
+  if(length(diagnosticIds) > 0){
+    for(diagnosticId in diagnosticIds){  
+      ParallelLogger::logInfo(
+        paste0(
+          'Converting and inserting diagnostic id', 
+          diagnosticId
+        )
+      )
+      diagnosePlp <- extractDiagnosticFromCsv(
+        diagnosticId = diagnosticId, 
+        csvFolder = csvFolder
+      )
+      if(!is.null(diagnosePlp)){
+        tryCatch(
+          {
+            addDiagnosePlpToDatabase(
+              diagnosePlp = diagnosePlp,
+              connectionDetails = connectionDetails,
+              databaseSchemaSettings = databaseSchemaSettings,
+              cohortDefinitions = cohortDefinitions,
+              databaseList = databaseList
+            )
+          }, error = function(e){ParallelLogger::logError(e)}
+        )
+      }
+      
+    }
+  }
+  
+  
+  return(TRUE)
+  
+}
+
+
+
+
+
+extractCohortDefinitionsCSV <- function(
+  csvFolder
+){
+  
+  # cohorts: cohort_id, cohort_definition_id, cohort_name
+  # cohort_definition: cohort_definition_id	cohort_name	description	json	sql_command 
+  
+  cohortDefinitionName <- dir(csvFolder, pattern = 'cohort_definition.csv')
+  cohort_definition <- readr::read_csv(file.path(csvFolder, cohortDefinitionName))
+  
+  result <- data.frame(
+    cohortId = cohort_definition$cohort_definition_id,
+    cohortName = cohort_definition$cohort_name,
+    json = cohort_definition$json,
+    sql = cohort_definition$sql_command 
+      )
+  
+  return(result)
+}
+
+extractDatabaseListCSV <- function(
+  csvFolder
+){
+  # database_meta_data: database_id	cdm_source_name	cdm_source_abbreviation
+  # database_details: database_id	database_meta_data_id
+  databaseMetaDataName <- dir(csvFolder, pattern = 'database_meta_data.csv')
+  databaseMetaData  <- readr::read_csv(file.path(csvFolder, databaseMetaDataName))
+
+  databaseList <- createDatabaseList(
+    cdmDatabaseSchemas = databaseMetaData$cdm_source_name,
+    cdmDatabaseNames = databaseMetaData$cdm_source_abbreviation,
+    databaseRefIds = databaseMetaData$database_id
+  )
+  
+  return(databaseList)
+}
+
+
+getModelDesignSettingTable <- function(modeldesignsRow){
+  result <- data.frame(
+    tableName = c('cohorts', 'cohorts',
+                  'population_settings', 'plp_data_settings',
+                  'model_settings', 'covariate_settings', 'sample_settings', 
+                  'split_settings', 'feature_engineering_settings', 
+                  'tidy_covariates_settings'),
+    idColumn = c('cohort_id', 'cohort_id',
+                 'population_setting_id', 'plp_data_setting_id',
+                 'model_setting_id', 'covariate_setting_id', 'sample_setting_id', 
+                 'split_setting_id', 'feature_engineering_setting_id', 
+                 'tidy_covariates_setting_id'),
+    jsonColumn = c('cohort_definition_id', 'cohort_definition_id',
+                   'population_settings_json', 'plp_data_settings_json',
+                   'model_settings_json', 'covariate_settings_json', 'sample_settings_json', 
+                   'split_settings_json', 'feature_engineering_settings_json', 
+                   'tidy_covariates_settings_json'),
+    convertJson = c(rep(F,2), rep(T, 8)),
+    value = c(modeldesignsRow$target_id, modeldesignsRow$outcome_id,
+              modeldesignsRow$population_setting_id, modeldesignsRow$plp_data_setting_id,
+              modeldesignsRow$model_setting_id, modeldesignsRow$covariate_setting_id, modeldesignsRow$sample_setting_id,
+              modeldesignsRow$split_setting_id, modeldesignsRow$feature_engineering_setting_id	, 
+              modeldesignsRow$tidy_covariates_setting_id),
+    modelDesignInput = c('targetId', 'outcomeId', 
+                         'populationSettings', 'restrictPlpDataSettings', 
+                         'modelSettings', 'covariateSettings', 'sampleSettings', 
+                         'splitSettings', 'featureEngineeringSettings',
+                         'preprocessSettings')
+  )
+  return(result)
+}
+
+getModelDesignCsv <- function(
+  modelDesignSettingTable, 
+  csvFolder = csvFolder
+) {
+  
+  csvFileNames <- dir(csvFolder, pattern = '.csv')
+  
+  result <- list()
+  for(i in 1:nrow(modelDesignSettingTable)){
+    table <- readr::read_csv(file.path(csvFolder, csvFileNames[grep(modelDesignSettingTable$tableName[i], csvFileNames)]))
+    ind <- table[modelDesignSettingTable$idColumn[i]] == modelDesignSettingTable$value[i]
+    result[[i]] <- table[ind,][modelDesignSettingTable$jsonColumn[i]]
+    if(modelDesignSettingTable$convertJson[i]){
+      result[[i]] <- ParallelLogger::convertJsonToSettings(as.character(result[[i]]))
+    } else{
+      # ids need to be integer
+      result[[i]] <- as.double(result[[i]])
+    }
+  }
+  names(result) <- modelDesignSettingTable$modelDesignInput
+
+  modelDesign <- do.call(what = PatientLevelPrediction::createModelDesign, args = result)
+  
+  return(modelDesign)
+}
+
+getPerformanceEvaluationCsv <- function(
+  performanceId, 
+  csvFolder
+){
+  
+  csvFileNames <- dir(csvFolder, pattern = '.csv')
+  
+  result <- list(
+    
+    evaluationStatistics = tryCatch(
+      {
+      res <- readr::read_csv(file.path(csvFolder, csvFileNames[grep('evaluation_statistics', csvFileNames)])) %>%
+        dplyr::filter(.data$performance_id == !!performanceId) %>%
+        dplyr::select(-"performance_id");
+      colnames(res) <- SqlRender::snakeCaseToCamelCase( colnames(res));
+      res
+    }, 
+    error = function(e){ParallelLogger::logInfo(e); return(NULL)}
+    ),
+    
+    thresholdSummary = tryCatch({
+      res <- readr::read_csv(file.path(csvFolder, csvFileNames[grep('threshold_summary', csvFileNames)])) %>%
+        dplyr::filter(.data$performance_id == !!performanceId) %>%
+        dplyr::select(-"performance_id");
+      colnames(res) <- SqlRender::snakeCaseToCamelCase( colnames(res));
+      res
+    }, 
+    error = function(e){ParallelLogger::logInfo(e); return(NULL)}
+    ),
+    
+    calibrationSummary = tryCatch({
+      res <- readr::read_csv(file.path(csvFolder, csvFileNames[grep('calibration_summary', csvFileNames)])) %>%
+        dplyr::filter(.data$performance_id == !!performanceId) %>%
+        dplyr::select(-"performance_id");
+      colnames(res) <- SqlRender::snakeCaseToCamelCase( colnames(res));
+      res
+    }, 
+    error = function(e){ParallelLogger::logInfo(e); return(NULL)}
+    ),
+    
+    demographicSummary = tryCatch({
+      res <- readr::read_csv(file.path(csvFolder, csvFileNames[grep('demographic_summary', csvFileNames)])) %>%
+        dplyr::filter(.data$performance_id == !!performanceId) %>%
+        dplyr::select(-"performance_id");
+      colnames(res) <- SqlRender::snakeCaseToCamelCase( colnames(res));
+      res
+    }, 
+    error = function(e){ParallelLogger::logInfo(e); return(NULL)}
+    ),
+    
+    predictionDistribution = tryCatch({
+      res <- readr::read_csv(file.path(csvFolder, csvFileNames[grep('prediction_distribution', csvFileNames)])) %>%
+        dplyr::filter(.data$performance_id == !!performanceId) %>%
+        dplyr::select(-"performance_id");
+      colnames(res) <- SqlRender::snakeCaseToCamelCase( colnames(res));
+      res
+    }, 
+    error = function(e){ParallelLogger::logInfo(e); return(NULL)}
+    )
+  )
+  
+  return(result)
+  
+}
+
+extractObjectFromCsv <- function(
+  performanceId, 
+  csvFolder
+){
+  
+  csvFileNames <- dir(csvFolder, pattern = '.csv')
+  
+  # get the model design
+  # performance_id	model_design_id	development_database_id	validation_database_id	target_id	outcome_id	tar_id	plp_data_setting_id	population_setting_id	model_development	execution_date_time	plp_version
+  performances <- readr::read_csv(file.path(csvFolder, csvFileNames[grep('performances', csvFileNames)]))
+  poi <- performances[performances$performance_id == performanceId,,]
+  
+  modelDesignId <- poi$model_design_id
+  modeldesigns <- readr::read_csv(file.path(csvFolder, csvFileNames[grep('model_designs', csvFileNames)]))
+  # model_design_id	target_id	outcome_id	tar_id	plp_data_setting_id	population_setting_id	model_setting_id	covariate_setting_id	sample_setting_id	split_setting_id	feature_engineering_setting_id	tidy_covariates_setting_id
+  modeldesigns <- modeldesigns[modeldesigns$model_design_id == modelDesignId,,]
+  
+  modelDesignSettingTable <- getModelDesignSettingTable(
+    modeldesignsRow = modeldesigns
+  )
+    
+  modelDesign <- getModelDesignCsv(
+    modelDesignSettingTable = modelDesignSettingTable, 
+    csvFolder = csvFolder
+    )
+  
+  covariateSummary <- readr::read_csv(file.path(csvFolder, csvFileNames[grep('covariate_summary', csvFileNames)])) %>%
+    dplyr::filter(.data$performance_id == !!poi$performance_id) %>%
+    dplyr::select(-"performance_id")
+  colnames(covariateSummary) <- SqlRender::snakeCaseToCamelCase(colnames(covariateSummary))
+  
+  performanceEvaluation <- getPerformanceEvaluationCsv(
+    performanceId = poi$performance_id, 
+    csvFolder = csvFolder
+  )
+  
+  modelMissing <- F
+  if(poi$model_development == 1){
+
+    modelsName <- dir(csvFolder, pattern = 'models.csv')
+    models  <- readr::read_csv(file.path(csvFolder, modelsName))
+    models <- models %>% 
+      dplyr::filter(.data$model_design_id == !!poi$model_design_id ) %>%
+      dplyr::filter(.data$database_id == !!poi$development_database_id)
+    
+    modelLoc <- strsplit(x = models$plp_model_file, split = '/')[[1]][length(strsplit(x = models$plp_model_file, split = '/')[[1]])]
+    plpModel <- tryCatch({
+      PatientLevelPrediction::loadPlpModel(file.path(csvFolder, 'models', modelLoc))
+    }, 
+      error = function(e){ParallelLogger::logInfo(e); return(NULL)}
+    )
+    
+    resultClass <- 'runPlp'
+  
+    if(is.null(modelLoc)){
+      ParallelLogger::logInfo('Models missing from csv folder - just adding performance')
+      modelMissing <- T
+    }
+    
+  }
+  
+  if(poi$model_development == 0 | modelMissing){
+    
+    # database_details: database_id	database_meta_data_id
+    databaseMetaDataName <- dir(csvFolder, pattern = 'database_meta_data.csv')
+    databaseMetaData  <- readr::read_csv(file.path(csvFolder, databaseMetaDataName))
+    databaseDetailsName <- dir(csvFolder, pattern = 'database_details.csv')
+    databaseDetails  <- readr::read_csv(file.path(csvFolder, databaseDetailsName))
+    databases <- merge(databaseDetails, databaseMetaData, by.x = 'database_meta_data_id', by.y = 'database_id')
+    
+    dev <- databases[databases$database_id == poi$development_database_id,,]
+    val <- databases[databases$database_id == poi$validation_database_id,,]
+    
+    developmentDatabase <- dev$cdm_source_name
+    developmentDatabaseId <- dev$database_meta_data_id
+    validationDatabase <- val$cdm_source_name
+    validationDatabaseId <- val$database_meta_data_id
+    
+      attritionName <- dir(csvFolder, pattern = 'attrition.csv')
+      attrition  <- readr::read_csv(file.path(csvFolder, attritionName)) %>%
+        dplyr::filter(.data$performance_id == !!poi$performance_id) %>%
+        dplyr::select(-"performance_id")
+      colnames(attrition) <- SqlRender::snakeCaseToCamelCase(colnames(attrition))
+      
+      cohortsName <- dir(csvFolder, pattern = 'cohorts.csv')
+      cohorts  <- readr::read_csv(file.path(csvFolder, cohortsName))
+      plpDataSetName <- dir(csvFolder, pattern = 'plp_data_settings.csv')
+      plpDataSet  <- readr::read_csv(file.path(csvFolder, plpDataSetName))
+      popSetName <- dir(csvFolder, pattern = 'population_settings.csv')
+      popSet  <- readr::read_csv(file.path(csvFolder, popSetName))
+      
+      # get the model
+      plpModel <- list(
+        model = 'external validation of model',
+        modelDesign = modelDesign,
+        validationDetails = list(
+          analysisId = '', 
+          analysisSource = '', 
+          developmentDatabase = developmentDatabase,
+          developmentDatabaseId = developmentDatabaseId,
+          validationDatabase = validationDatabase,
+          validationDatabaseId = validationDatabaseId,
+          
+          populationSettings = ParallelLogger::convertJsonToSettings(
+            as.character(
+              popSet %>% 
+                dplyr::filter(.data$population_setting_id == !!poi$population_setting_id) %>%
+                dplyr::select("population_settings_json")
+            )
+          ),
+          restrictPlpDataSettings = ParallelLogger::convertJsonToSettings(
+            as.character(
+              plpDataSet %>% 
+              dplyr::filter(.data$plp_data_setting_id == !!poi$plp_data_setting_id) %>%
+              dplyr::select("plp_data_settings_json")
+            )
+          ),
+          
+          outcomeId = as.double(
+            cohorts %>% 
+              dplyr::filter(.data$cohort_id == !!poi$outcome_id) %>%
+              dplyr::select("cohort_definition_id")
+          ),
+          targetId = as.double(
+            cohorts %>% 
+              dplyr::filter(.data$cohort_id == !!poi$target_id) %>%
+              dplyr::select("cohort_definition_id")
+          ),
+          
+          attrition = attrition
+        )
+      )
+      attr(plpModel, "predictionFunction") <- 'none'
+      attr(plpModel, "saveType") <- 'RtoJson'
+      class(plpModel) <- 'plpModel'
+    
+    resultClass <- 'externalValidatePlp'
+  }
+  
+
+  result <- list(
+    executionSummary = list(
+      PackageVersion = list(
+        packageVersion = poi$plp_version
+      ),
+      #TotalExecutionElapsedTime = ,
+      ExecutionDateTime = poi$execution_date_time	
+    ),
+    model = plpModel,
+    performanceEvaluation = performanceEvaluation,
+    covariateSummary = covariateSummary,
+    analysisRef = list(
+      analysisId = ''
+      )
+  )
+  class(result) <- resultClass
+  
+  # return the object
+  return(result)
+  
+}
+
+extractDiagnosticFromCsv <- function(
+  diagnosticId, 
+  csvFolder
+){
+  
+  # diagnostic_id	model_design_id	database_id	execution_date_time
+  csvFileNames <- dir(csvFolder, pattern = '.csv')
+  
+  # get the model design
+  # performance_id	model_design_id	development_database_id	validation_database_id	target_id	outcome_id	tar_id	plp_data_setting_id	population_setting_id	model_development	execution_date_time	plp_version
+  diagnostics <- readr::read_csv(file.path(csvFolder, csvFileNames[grep('diagnostics', csvFileNames)]))
+  if(length(diagnostics) == 0){
+    ParallelLogger::logInfo('No diagnostics in csv results')
+    return(NULL)
+  }
+  doi <- diagnostics[diagnostics$diagnostic_id == diagnosticId,,]
+  if(nrow(doi) == 0){
+    ParallelLogger::logInfo('No diagnostics in csv results with specified diagnosticId')
+    return(NULL)
+  }
+  
+  modelDesignId <- doi$model_design_id
+  modeldesigns <- readr::read_csv(file.path(csvFolder, csvFileNames[grep('model_designs', csvFileNames)]))
+  # model_design_id	target_id	outcome_id	tar_id	plp_data_setting_id	population_setting_id	model_setting_id	covariate_setting_id	sample_setting_id	split_setting_id	feature_engineering_setting_id	tidy_covariates_setting_id
+  modeldesigns <- modeldesigns[modeldesigns$model_design_id == modelDesignId,,]
+  
+  modelDesignSettingTable <- getModelDesignSettingTable(
+    modeldesignsRow = modeldesigns
+  )
+  
+  modelDesign <- getModelDesignCsv(
+    modelDesignSettingTable = modelDesignSettingTable, 
+    csvFolder = csvFolder
+  )
+  
+  databaseMetaDataName <- dir(csvFolder, pattern = 'database_meta_data.csv')
+  databaseMetaData  <- readr::read_csv(file.path(csvFolder, databaseMetaDataName))
+  databaseDetailsName <- dir(csvFolder, pattern = 'database_details.csv')
+  databaseDetails  <- readr::read_csv(file.path(csvFolder, databaseDetailsName))
+  databases <- merge(databaseDetails, databaseMetaData, by.x = 'database_meta_data_id', by.y = 'database_id')
+  
+  db <- databases[databases$database_id == doi$database_id]
+  
+  databaseSchema <- db$cdm_source_name
+  databaseId <- db$database_meta_data_id
+  
+  outcomesName <- dir(csvFolder, pattern = 'diagnostic_outcomes.csv')
+  outcomes  <- readr::read_csv(file.path(csvFolder, outcomesName)) %>%
+    dplyr::filter(.data$diagnostic_id == !! diagnosticId) %>%
+    dplyr::select(-"diagnostic_id")
+  colnames(outcomes)  <- SqlRender::snakeCaseToCamelCase(colnames(outcomes))
+  
+  predictorsName <- dir(csvFolder, pattern = 'diagnostic_predictors.csv')
+  predictors   <- readr::read_csv(file.path(csvFolder, predictorsName)) %>%
+    dplyr::filter(.data$diagnostic_id == !! diagnosticId) %>%
+    dplyr::select(-"diagnostic_id")
+  colnames(predictors)  <- SqlRender::snakeCaseToCamelCase(colnames(predictors))
+  
+  participantsName <- dir(csvFolder, pattern = 'diagnostic_participants.csv')
+  participants  <- readr::read_csv(file.path(csvFolder, participantsName)) %>%
+    dplyr::filter(.data$diagnostic_id == !! diagnosticId) %>%
+    dplyr::select(-"diagnostic_id")
+  colnames(participants)  <- SqlRender::snakeCaseToCamelCase(colnames(participants))
+  
+  summaryName <- dir(csvFolder, pattern = 'diagnostic_summary.csv')
+  summary  <- readr::read_csv(file.path(csvFolder, summaryName)) %>%
+    dplyr::filter(.data$diagnostic_id == !! diagnosticId) %>%
+    dplyr::select(-"diagnostic_id")
+  colnames(summary)  <- SqlRender::snakeCaseToCamelCase(colnames(summary))
+  
+  result <- list(
+    summary = summary,
+    participants = participants,
+    predictors = predictors,
+    outcomes = outcomes,
+    designs = NULL,
+    modelDesign = modelDesign,
+    databaseSchema = databaseSchema,
+    databaseId = databaseId
+  )
+  
+  class(result) <- 'diagnosePlp'
+  
+  return(result)
+}
+
+
+getTableNamesPlp <- function(
+    connectionDetails,
+    databaseSchema
+){
+  
+  # check some plp tables exists in databaseSchemaSettings 
+  conn <- DatabaseConnector::connect(connectionDetails) 
+  on.exit(DatabaseConnector::disconnect(conn))
+  
+  result <- DatabaseConnector::getTableNames(
+    connection = conn, 
+    databaseSchema = databaseSchema
+  )
+  
+  return(tolower(result))
+}
diff --git a/R/JsonHelpers.R b/R/JsonHelpers.R
deleted file mode 100644
index 5ae3339f3..000000000
--- a/R/JsonHelpers.R
+++ /dev/null
@@ -1,78 +0,0 @@
-# code to parse the modelDesign list into json
-
-addAttributes <- function(x){
-  attributeValues <- attributes(x)
-  if('names' %in% names(attributeValues)){
-    attributeValues$names <- NULL
-  }
-  
-  if(length(attributeValues)>0){
-    names(attributeValues) <- paste0('attr_', names(attributeValues))
-    x <- c(x, attributeValues)
-  }
-  return(x)
-}
-
-prepareToJson <- function(md){
-  md$restrictPlpDataSettings <- addAttributes(md$restrictPlpDataSettings)
-  md$populationSettings <- addAttributes(md$populationSettings)
-  md$preprocessSettings <- addAttributes(md$preprocessSettings)
-  md$executeSettings <- addAttributes(md$executeSettings)
-  
-  md$modelSettings$param <- addAttributes(md$modelSettings$param)
-  md$modelSettings <- addAttributes(md$modelSettings)
-  
-  if(class(md$covariateSettings) == 'covariateSettings'){
-    md$covariateSettings <- list(md$covariateSettings)
-  }
-  md$covariateSettings <- lapply(md$covariateSettings, function(x) addAttributes(x)) 
-  
-  if(class(md$sampleSettings) == 'sampleSettings'){
-    md$sampleSettings <- list(md$sampleSettings)
-  }
-  md$sampleSettings <- lapply(md$sampleSettings, function(x) addAttributes(x)) 
-  
-  if(class(md$featureEngineeringSettings) == 'featureEngineeringSettings'){
-    md$featureEngineeringSettings<- list(md$featureEngineeringSettings)
-  }
-  md$featureEngineeringSettings <- lapply(md$featureEngineeringSettings, function(x) addAttributes(x)) 
-  
-  md <- addAttributes(md)
-  return(md)
-}
-
-
-
-extractAttributes <- function(x){
-  
-  ind <- grep('attr_', names(x))
-  
-  if(length(ind)>0){
-    attributeValues <- x[ind]
-    x <- x[-ind]
-    names(attributeValues) <- gsub(pattern = 'attr_',replacement = '',x = names(attributeValues))
-    attributeValues$names <- names(x)
-    attributes(x) <- attributeValues
-  }
-  
-  return(x)
-}
-
-prepareToRlist <- function(md){
-  md$restrictPlpDataSettings <- extractAttributes(md$restrictPlpDataSettings)
-  md$populationSettings <- extractAttributes(md$populationSettings)
-  md$preprocessSettings <- extractAttributes(md$preprocessSettings)
-  md$executeSettings <- extractAttributes(md$executeSettings)
-  
-  md$modelSettings$param <- extractAttributes(md$modelSettings$param)
-  md$modelSettings <- extractAttributes(md$modelSettings)
-  
-  md$covariateSettings <- lapply(md$covariateSettings, function(x) extractAttributes(x)) 
-  
-  md$sampleSettings <- lapply(md$sampleSettings, function(x) extractAttributes(x)) 
-  
-  md$featureEngineeringSettings <- lapply(md$featureEngineeringSettings, function(x) extractAttributes(x)) 
-  
-  md <- extractAttributes(md)
-  return(md)
-}
\ No newline at end of file
diff --git a/R/KNN.R b/R/KNN.R
index 84a74db34..834b26eaa 100644
--- a/R/KNN.R
+++ b/R/KNN.R
@@ -48,6 +48,7 @@ setKNN <- function(k=1000, indexFolder=file.path(getwd(),'knn'), threads = 1  ){
     )
   
   attr(param, 'settings') <- list(
+    modelType = 'knn',
     modelName = 'K Nearest Neighbors'
   )
   
@@ -63,7 +64,9 @@ setKNN <- function(k=1000, indexFolder=file.path(getwd(),'knn'), threads = 1  ){
   return(result)
 }
 
-fitKNN <- function(trainData, param, search = 'none', analysisId ){
+fitKNN <- function(trainData, modelSettings, search = 'none', analysisId, ...){
+  
+  param <- modelSettings$param
 
   if (!FeatureExtraction::isCovariateData(trainData$covariateData)){
     stop("Needs correct covariateData")
@@ -104,52 +107,55 @@ fitKNN <- function(trainData, param, search = 'none', analysisId ){
     cohort = trainData$labels, 
     plpModel = list(
       model = indexFolder,
-      settings = list(
-      modelSettings = list(
-        model = 'knn',
-        param = list(
-          k = k,
-          indexFolder = indexFolder,
-          threads = param$threads
+      trainDetails = list(
+        finalModelParameters = list(
+            k = k,
+            threads = param$threads
           )
-        )
-      )
       )
     )
+  )
+      
   
   prediction$evaluationType <- 'Train'
   
   result <- list(
     model = indexFolder,
     
+    preprocessing = list(
+      featureEngineering = attr(trainData, "metaData")$featureEngineering,#learned mapping
+      tidyCovariates = attr(trainData$covariateData, "metaData")$tidyCovariateDataSettings,  #learned mapping
+      requireDenseMatrix = F
+    ),
+    
     prediction = prediction,
     
-    settings = list(
-      plpDataSettings = attr(trainData, "metaData")$plpDataSettings,
+    modelDesign = PatientLevelPrediction::createModelDesign(
+      targetId = attr(trainData, "metaData")$targetId,
+      outcomeId = attr(trainData, "metaData")$outcomeId,
+      restrictPlpDataSettings = attr(trainData, "metaData")$restrictPlpDataSettings,
       covariateSettings = attr(trainData, "metaData")$covariateSettings,
-      featureEngineering = attr(trainData$covariateData, "metaData")$featureEngineering,
-      tidyCovariates = attr(trainData$covariateData, "metaData")$tidyCovariateDataSettings, 
-      requireDenseMatrix = F,
       populationSettings = attr(trainData, "metaData")$populationSettings,
-      modelSettings = list(
-        model = 'KNN', 
-        param = param,
-        finalModelParameters = list(),
-        extraSettings = attr(param, 'settings')
-      ),
+      featureEngineeringSettings = attr(trainData$covariateData, "metaData")$featureEngineeringSettings,
+      preprocessSettings = attr(trainData$covariateData, "metaData")$preprocessSettings, 
+      modelSetting = modelSettings,
       splitSettings = attr(trainData, "metaData")$splitSettings,
       sampleSettings = attr(trainData, "metaData")$sampleSettings
     ),
     
     trainDetails = list(
       analysisId = analysisId,
-      cdmDatabaseSchema = attr(trainData, "metaData")$cdmDatabaseSchema,
-      outcomeId = attr(trainData, "metaData")$outcomeId,
-      cohortId = attr(trainData, "metaData")$cohortId,
+      developmentDatabase = attr(trainData, "metaData")$cdmDatabaseName,
+      developmentDatabaseSchema = attr(trainData, "metaData")$cdmDatabaseSchema, 
       attrition = attr(trainData, "metaData")$attrition, 
-      trainingTime = comp,
+      trainingTime = paste(as.character(abs(comp)), attr(comp,'units')),
       trainingDate = Sys.Date(),
-      hyperParamSearch =c()
+      modelName = 'KNN',
+      hyperParamSearch = data.frame(),
+      finalModelParameters = list(
+        k = k,
+        threads = param$threads
+          )
     ),
     
     covariateImportance = variableImportance
@@ -174,9 +180,9 @@ predictKnn <- function(
     covariates = data$covariateData$covariates,
     cohort = cohort[,!colnames(cohort)%in%'cohortStartDate'],
     indexFolder = plpModel$model,
-    k = plpModel$settings$modelSettings$param$k,
+    k = plpModel$trainDetails$finalModelParameters$k,
     weighted = TRUE,
-    threads = plpModel$settings$modelSettings$param$threads
+    threads = plpModel$trainDetails$finalModelParameters$threads
   )
   
   # can add: threads = 1 in the future
@@ -187,7 +193,7 @@ predictKnn <- function(
     all.x=T, fill=0)
   prediction$value[is.na(prediction$value)] <- 0
   
-  attr(prediction, "metaData") <- 'binary'
+  attr(prediction, "metaData")$modelType <- 'binary'
   
   return(prediction)
   
diff --git a/R/LearningCurve.R b/R/LearningCurve.R
index 0ccf78aeb..863d7e778 100644
--- a/R/LearningCurve.R
+++ b/R/LearningCurve.R
@@ -40,7 +40,7 @@
 #'   \code{trainFractions}. Note, providing \code{trainEvents} will override
 #'   your input to \code{trainFractions}. The format should be as follows:
 #'   \itemize{
-#'     \item{ \code{c(500, 1000, 1500) } - a list of training events}
+#'     \item \code{c(500, 1000, 1500) } - a list of training events
 #'   }
 #' @param featureEngineeringSettings An object of \code{featureEngineeringSettings} specifying any feature engineering to be learned (using the train data)                                                        
 #' @param preprocessSettings         An object of \code{preprocessSettings}. This setting specifies the minimum fraction of 
@@ -48,17 +48,12 @@
 #'                                   and whether to normalise the covariates before training  
 #' @param modelSettings              An object of class \code{modelSettings} created using one of the function:
 #'                                         \itemize{
-#'                                         \item{setLassoLogisticRegression()}{ A lasso logistic regression model}
-#'                                         \item{setGradientBoostingMachine()}{ A gradient boosting machine}
-#'                                         \item{setAdaBoost()}{ An ada boost model}
-#'                                         \item{setRandomForest()}{ A random forest model}
-#'                                         \item{setDecisionTree()}{ A decision tree model}
-#'                                         \item{setCovNN())}{ A convolutional neural network model}
-#'                                         \item{setCIReNN()}{ A recurrent neural network model}
-#'                                         \item{setMLP()}{ A neural network model}
-#'                                         \item{setDeepNN()}{ A deep neural network model}
-#'                                         \item{setKNN()}{ A KNN model}
-#'                                         
+#'                                         \item \code{setLassoLogisticRegression()} A lasso logistic regression model
+#'                                         \item \code{setGradientBoostingMachine()} A gradient boosting machine
+#'                                         \item \code{setAdaBoost()} An ada boost model
+#'                                         \item \code{setRandomForest()} A random forest model
+#'                                         \item \code{setDecisionTree()} A decision tree model
+#'                                         \item \code{setKNN()} A KNN model
 #'                                         } 
 #' @param logSettings                An object of \code{logSettings} created using \code{createLogSettings} 
 #'                                   specifying how the logging is done                                                                            
@@ -98,7 +93,7 @@ createLearningCurve <- function(
   populationSettings = createStudyPopulationSettings(),
   splitSettings = createDefaultSplitSetting(),
   trainFractions = c(0.25, 0.50, 0.75),
-  trainEvents = c(500, 1000, 1500),
+  trainEvents = NULL,
   sampleSettings = createSampleSettings(),
   featureEngineeringSettings = createFeatureEngineeringSettings(),
   preprocessSettings = createPreprocessSettings(
@@ -188,7 +183,7 @@ createLearningCurve <- function(
     nRuns <- length(trainFractions)
     
     settings = list(
-      plpData = plpData,
+      plpData = quote(plpData),
       outcomeId = outcomeId,
       analysisId = analysisId,
       populationSettings = populationSettings,
@@ -238,7 +233,7 @@ createLearningCurve <- function(
 
 lcWrapper <- function(settings){
   plpData <- PatientLevelPrediction::loadPlpData(settings$plpData)
-  settings$plpData <- plpData
+  settings$plpData <- quote(plpData)
   result <- tryCatch({do.call(runPlp, settings)},
                      warning = function(war) {
                        ParallelLogger::logInfo(paste0('a warning: ', war))
@@ -282,7 +277,7 @@ getTrainFractions <- function(
   trainFractionsTemp <- samplesRequired/nrow(population)
   
   # filter out no. of events that would exceed the available training set size
-  binaryMask <- trainFractionsTemp <= (1.0 - splitSettings$testFraction)
+  binaryMask <- trainFractionsTemp <= (1.0 - splitSettings$test)
   
   # override any input to trainFractions with event-based training fractions
   trainFractions <- trainFractionsTemp[binaryMask]
@@ -307,7 +302,7 @@ learningCurveHelper <- function(result, trainFractions){
   
   result$name <- paste(result$evaluation, result$metric, sep='_')
   
-  result <- result %>% dplyr::select(.data$name, .data$value)
+  result <- result %>% dplyr::select("name", "value")
   
   result <- rbind(
     c('executionTime', executeTime), 
@@ -381,9 +376,9 @@ plotLearningCurve <- function(learningCurve,
     # create a data.frame with evalautionType, AUROC
     tidyLearningCurve <- learningCurve %>% 
       dplyr::rename(
-        Occurrences = .data$Train_outcomeCount, 
-        Observations = .data$Train_populationSize ) %>%
-      dplyr::select(.data$trainFraction, .data$Occurrences, .data$Observations, .data$Test_AUROC, .data$Train_AUROC)
+        Occurrences = "Train_outcomeCount", 
+        Observations = "Train_populationSize" ) %>%
+      dplyr::select("trainFraction", "Occurrences", "Observations", "Test_AUROC", "Train_AUROC")
     
     for(i in 1:ncol(tidyLearningCurve)){
       tidyLearningCurve[,i] <- as.double(as.character(tidyLearningCurve[,i]))
@@ -407,9 +402,9 @@ plotLearningCurve <- function(learningCurve,
     # tidy up dataframe
     tidyLearningCurve <- learningCurve %>% 
       dplyr::rename(
-        Occurrences = .data$Train_outcomeCount, 
-        Observations = .data$Train_populationSize ) %>%
-      dplyr::select(.data$trainFraction, .data$Occurrences, .data$Observations, .data$Test_AUPRC, .data$Train_AUPRC)
+        Occurrences = "Train_outcomeCount", 
+        Observations = "Train_populationSize" ) %>%
+      dplyr::select("trainFraction", "Occurrences", "Observations", "Test_AUPRC", "Train_AUPRC")
     
     for(i in 1:ncol(tidyLearningCurve)){
       tidyLearningCurve[,i] <- as.double(as.character(tidyLearningCurve[,i]))
@@ -432,9 +427,9 @@ plotLearningCurve <- function(learningCurve,
     # tidy up dataframe
     tidyLearningCurve <- learningCurve %>% 
       dplyr::rename(
-        Occurrences = .data$Train_outcomeCount, 
-        Observations = .data$Train_populationSize ) %>%
-      dplyr::select(.data$trainFraction, .data$Occurrences, .data$Observations, .data$`Test_brier score scaled`, .data$`Train_brier score scaled`)
+        Occurrences = "Train_outcomeCount", 
+        Observations = "Train_populationSize" ) %>%
+      dplyr::select("trainFraction", "Occurrences", "Observations", "Test_brier score scaled", "Train_brier score scaled")
     
     for(i in 1:ncol(tidyLearningCurve)){
       tidyLearningCurve[,i] <- as.double(as.character(tidyLearningCurve[,i]))
@@ -470,8 +465,8 @@ plotLearningCurve <- function(learningCurve,
   
   # create plot object
   plot <- tidyLearningCurve %>%
-    ggplot2::ggplot(ggplot2::aes_string(x = abscissa, y= 'value',
-      col = "Dataset")) +
+    ggplot2::ggplot(ggplot2::aes(x = .data[[abscissa]], y = .data[['value']],
+      col = .data[["Dataset"]])) +
     ggplot2::geom_line() +
     ggplot2::coord_cartesian(ylim = yAxisRange, expand = FALSE) +
     ggplot2::labs(title = plotTitle, subtitle = plotSubtitle, 
diff --git a/R/LightGBM.R b/R/LightGBM.R
new file mode 100644
index 000000000..69f1b6c6e
--- /dev/null
+++ b/R/LightGBM.R
@@ -0,0 +1,236 @@
+# @file LightGBM.R
+# Copyright 2023 Observational Health Data Sciences and Informatics
+#
+# This file is part of PatientLevelPrediction
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#' Create setting for gradient boosting machine model using lightGBM (https://github.com/microsoft/LightGBM/tree/master/R-package).
+#'
+#' @param nthread           The number of computer threads to use (how many cores do you have?) 
+#' @param earlyStopRound    If the performance does not increase over earlyStopRound number of trees then training stops (this prevents overfitting)
+#' @param numIterations     Number of boosting iterations. 
+#' @param numLeaves         This hyperparameter sets the maximum number of leaves. Increasing this parameter can lead to higher model complexity and potential overfitting.
+#' @param maxDepth          This hyperparameter sets the maximum depth . Increasing this parameter can also lead to higher model complexity and potential overfitting.
+#' @param minDataInLeaf     This hyperparameter sets the minimum number of data points that must be present in a leaf node. Increasing this parameter can help to reduce overfitting
+#' @param learningRate      This hyperparameter controls the step size at each iteration of the gradient descent algorithm. Lower values can lead to slower convergence but may result in better performance.
+#' @param lambdaL1          This hyperparameter controls L1 regularization, which can help to reduce overfitting by encouraging sparse models.
+#' @param lambdaL2          This hyperparameter controls L2 regularization, which can also help to reduce overfitting by discouraging large weights in the model.
+#' @param scalePosWeight    Controls weight of positive class in loss - useful for imbalanced classes
+#' @param isUnbalance       This parameter cannot be used at the same time with scalePosWeight, choose only one of them. While enabling this should increase the overall performance metric of your model, it will also result in poor estimates of the individual class probabilities. 
+#' @param seed              An option to add a seed when training the final model
+#'
+#' @examples
+#' model.lightgbm <- setLightGBM(
+#'     numLeaves = c(20, 31, 50), maxDepth = c(-1, 5, 10),
+#'     minDataInLeaf = c(10, 20, 30), learningRate = c(0.05, 0.1, 0.3)
+#' )
+#'
+#' @export
+setLightGBM <- function(nthread = 20, 
+                        earlyStopRound = 25, 
+                        numIterations = c(100),
+                        numLeaves = c(31),
+                        maxDepth = c(5, 10),
+                        minDataInLeaf = c(20),
+                        learningRate = c(0.05, 0.1, 0.3),
+                        lambdaL1 = c(0),
+                        lambdaL2 = c(0),
+                        scalePosWeight = 1,
+                        isUnbalance = FALSE,
+                        seed = sample(10000000, 1)) {
+  ensure_installed("lightgbm")
+  checkIsClass(seed, c("numeric", "integer"))
+  
+  if (length(nthread) > 1) {
+    stop("nthread must be length 1")
+  }
+  if (!inherits(x = seed, what = c("numeric", "integer"))) {
+    stop("Invalid seed")
+  }
+  if(sum(numIterations < 1) > 0){
+    stop('numIterations must be greater that 0')
+  }
+  if(sum(numLeaves < 2) > 0){
+    stop('numLeaves must be greater that 1')
+  }
+  if(sum(numLeaves > 131072) > 0){
+    stop('numLeaves must be less that or equal 131072')
+  }
+  if(sum(learningRate <= 0) > 0){
+    stop('learningRate must be greater that 0')
+  }
+  if (sum(lambdaL1 < 0) > 0){
+    stop('lambdaL1 must be 0 or greater')
+  }
+  if (sum(lambdaL2 < 0) > 0){
+    stop('lambdaL2 must be 0 or greater')
+  }
+  if (sum(scalePosWeight < 0) > 0){
+    stop('scalePosWeight must be 0 or greater')
+  }
+  if (isUnbalance == TRUE & sum(scalePosWeight != 1) > 0){
+    stop('isUnbalance cannot be used at the same time with scale_pos_weight != 1, choose only one of them')
+  }
+  
+  paramGrid <- list(
+    earlyStopRound = earlyStopRound,
+    numIterations = numIterations,
+    numLeaves = numLeaves,
+    maxDepth = maxDepth,
+    minDataInLeaf = minDataInLeaf,
+    learningRate = learningRate,
+    lambdaL1 = lambdaL1,
+    lambdaL2 = lambdaL2,
+    isUnbalance = isUnbalance,
+    scalePosWeight = scalePosWeight
+  )
+  
+  param <- listCartesian(paramGrid)
+  
+  attr(param, "settings") <- list(
+    modelType = "LightGBM",
+    seed = seed[[1]],
+    modelName = "LightGBM",
+    threads = nthread[1],
+    varImpRFunction = "varImpLightGBM",
+    trainRFunction = "fitLightGBM",
+    predictRFunction = "predictLightGBM"
+  )
+  
+  attr(param, "saveType") <- "lightgbm"
+  
+  result <- list(
+    fitFunction = "fitRclassifier",
+    param = param
+  )
+  
+  class(result) <- "modelSettings"
+  
+  return(result)
+}
+
+
+
+varImpLightGBM <- function(model,
+                           covariateMap) {
+  varImp <- lightgbm::lgb.importance(model, percentage = T) %>% dplyr::select("Feature", "Gain")
+  
+  varImp <- data.frame(
+    covariateId = gsub(".*_","",varImp$Feature),
+    covariateValue = varImp$Gain,
+    included = 1
+  )
+  
+  varImp <- merge(covariateMap, varImp, by.x = "columnId", by.y = "covariateId")
+  varImp <- varImp %>%
+    dplyr::select("covariateId", "covariateValue", "included")
+  
+  return(varImp)
+}
+
+predictLightGBM <- function(plpModel,
+                            data,
+                            cohort) {
+  if (inherits(data, "plpData")) {
+    # convert
+    matrixObjects <- toSparseM(
+      plpData = data,
+      cohort = cohort,
+      map = plpModel$covariateImportance %>%
+        dplyr::select("columnId", "covariateId")
+    )
+    
+    # use the include??
+    
+    newData <- matrixObjects$dataMatrix
+    cohort <- matrixObjects$labels
+  } else {
+    newData <- data
+  }
+  
+  if (inherits(plpModel, "plpModel")) {
+    model <- plpModel$model
+  } else {
+    model <- plpModel
+  }
+  
+  pred <- data.frame(value = stats::predict(model, newData))
+  prediction <- cohort
+  prediction$value <- pred$value
+  
+  prediction <- prediction %>%
+    dplyr::select(-"rowId") %>%
+    dplyr::rename(rowId = "originalRowId")
+  
+  attr(prediction, "metaData") <- list(modelType = attr(plpModel, "modelType"))
+  
+  return(prediction)
+}
+
+fitLightGBM <- function(dataMatrix,
+                        labels,
+                        hyperParameters,
+                        settings) {
+  if (!is.null(hyperParameters$earlyStopRound)) {
+    trainInd <- sample(nrow(dataMatrix), nrow(dataMatrix) * 0.9)
+    train <- lightgbm::lgb.Dataset(
+      data = dataMatrix[trainInd, , drop = F],
+      label = labels$outcomeCount[trainInd]
+    )
+    test <- lightgbm::lgb.Dataset(
+      data = dataMatrix[-trainInd, , drop = F],
+      label = labels$outcomeCount[-trainInd]
+    )
+    watchlist <- list(train = train, test = test)
+  } else {
+    train <- lightgbm::lgb.Dataset(
+      data = dataMatrix,
+      label = labels$outcomeCount,
+      free_raw_data = FALSE,
+    )
+    watchlist <- list()
+  }
+  
+  outcomes <- sum(labels$outcomeCount > 0)
+  N <- nrow(labels)
+  outcomeProportion <- outcomes / N
+  set.seed(settings$seed)
+  model <- lightgbm::lgb.train(
+    data = train,
+    params = list(
+      objective = "binary",
+      boost = "gbdt",
+      metric = "auc",
+      num_iterations = hyperParameters$numIterations,
+      num_leaves = hyperParameters$numLeaves,
+      max_depth = hyperParameters$maxDepth,
+      learning_rate = hyperParameters$learningRate,
+      feature_pre_filter=FALSE,
+      min_data_in_leaf = hyperParameters$minDataInLeaf,
+      scale_pos_weight = hyperParameters$scalePosWeight,
+      lambda_l1 = hyperParameters$lambdaL1,
+      lambda_l2 = hyperParameters$lambdaL2,
+      seed = settings$seed,
+      is_unbalance = hyperParameters$isUnbalance,
+      max_bin = 255,
+      num_threads = settings$threads
+    ),
+    verbose = 1,
+    early_stopping_rounds = hyperParameters$earlyStopRound,
+    valids = watchlist
+    # categorical_feature = 'auto' # future work
+  )
+  
+  return(model)
+}
diff --git a/R/Logging.R b/R/Logging.R
index 8d1ca16f3..81cd54225 100644
--- a/R/Logging.R
+++ b/R/Logging.R
@@ -22,12 +22,12 @@
 #'
 #' @param verbosity                        Sets the level of the verbosity. If the log level is at or higher in priority than the logger threshold, a message will print. The levels are:
 #'                                         \itemize{
-#'                                         \item{DEBUG}{Highest verbosity showing all debug statements}
-#'                                         \item{TRACE}{Showing information about start and end of steps}
-#'                                         \item{INFO}{Show informative information (Default)}
-#'                                         \item{WARN}{Show warning messages}
-#'                                         \item{ERROR}{Show error messages}
-#'                                         \item{FATAL}{Be silent except for fatal errors}
+#'                                         \item DEBUG Highest verbosity showing all debug statements
+#'                                         \item TRACE Showing information about start and end of steps
+#'                                         \item INFO Show informative information (Default)
+#'                                         \item WARN Show warning messages
+#'                                         \item ERROR Show error messages
+#'                                         \item FATAL Be silent except for fatal errors
 #'                                         }
 #' @param timeStamp                        If TRUE a timestamp will be added to each logging statement. Automatically switched on for TRACE level.
 #' @param logName                          A string reference for the logger
diff --git a/R/ParamChecks.R b/R/ParamChecks.R
index 1fa5772de..542e7e803 100644
--- a/R/ParamChecks.R
+++ b/R/ParamChecks.R
@@ -74,7 +74,7 @@ checkNotNull <- function(parameter) {
 
 checkIsClass<- function(parameter,classes) {
   name = deparse(substitute(parameter))
-  if (!class(parameter)%in%classes) {
+  if (!inherits(x = parameter, what = classes)) {
     ParallelLogger::logError(paste0(name, ' should be of class:', classes))      
     stop(paste0(name, ' is wrong class'))
   }
diff --git a/R/PatientLevelPrediction.R b/R/PatientLevelPrediction.R
index 4f78d4a0f..848c42705 100644
--- a/R/PatientLevelPrediction.R
+++ b/R/PatientLevelPrediction.R
@@ -20,12 +20,11 @@
 #' 
 #' @description A package for running predictions using data in the OMOP CDM
 #'
-#' @docType package
 #' @name PatientLevelPrediction
+#' @keywords internal
 #' @importFrom dplyr %>%
 #' @importFrom rlang .data
-#' @import FeatureExtraction
-NULL
+"_PACKAGE"
 
 #' A simulation profile
 #' @docType data
diff --git a/R/Plotting.R b/R/Plotting.R
index 1fb1dfbf1..f2fe6bf0d 100644
--- a/R/Plotting.R
+++ b/R/Plotting.R
@@ -62,7 +62,7 @@ outcomeSurvivalPlot <- function(
   if(missing(outcomeId)){
     stop('outcomeId missing')
   }
-  if(class(plpData)!='plpData'){
+  if(!inherits(x = plpData, what = 'plpData')){
     stop('Incorrect plpData object')
   }
   if(!outcomeId%in%unique(plpData$outcomes$outcomeId)){
@@ -83,7 +83,7 @@ outcomeSurvivalPlot <- function(
   
   population$daysToEvent[is.na(population$daysToEvent)] <- population$survivalTime[is.na(population$daysToEvent)]
   survivalFit <- survival::survfit(
-    survival::Surv(daysToEvent, outcomeCount)~cohortId,
+    survival::Surv(daysToEvent, outcomeCount)~targetId,
     #riskDecile, 
     population, 
     conf.int = TRUE
@@ -293,11 +293,11 @@ plotSparseRoc <- function(
   plots <- list()
   length(plots) <- length(evalTypes)
   
-  for(i in 1:length(evalTypes)){
+  for (i in 1:length(evalTypes)){
     evalType <- evalTypes[i]
     x <- plpResult$performanceEvaluation$thresholdSummary %>% 
       dplyr::filter(.data[[typeColumn]] == evalType) %>% 
-      dplyr::select(.data$falsePositiveRate, .data$sensitivity)
+      dplyr::select("falsePositiveRate", "sensitivity")
     
     #x <- thresholdSummary[,c('falsePositiveRate','sensitivity')]
     x <- x[order(x$falsePositiveRate, x$sensitivity),]
@@ -316,14 +316,14 @@ plotSparseRoc <- function(
         )
     ) +
       ggplot2::geom_polygon(fill = "blue", alpha = 0.2) +
-      ggplot2::geom_line(size=1) +
+      ggplot2::geom_line(linewidth = 1) +
       ggplot2::geom_abline(intercept = 0, slope = 1,linetype = 2) +
-      ggplot2::scale_x_continuous("1 - specificity", limits=c(0,1)) +
-      ggplot2::scale_y_continuous("Sensitivity", limits=c(0,1)) + 
+      ggplot2::scale_x_continuous("1 - specificity", limits = c(0,1)) +
+      ggplot2::scale_y_continuous("Sensitivity", limits = c(0,1)) + 
       ggplot2::ggtitle(evalType)
   }
   
-  plot <- gridExtra::marrangeGrob(plots, nrow=length(plots), ncol=1)
+  plot <- gridExtra::marrangeGrob(plots, nrow =length(plots), ncol = 1)
   
   if (!is.null(saveLocation)){
     if(!dir.exists(saveLocation)){
@@ -370,11 +370,11 @@ plotPredictedPDF <- function(
     x <- plpResult$performanceEvaluation$thresholdSummary %>% 
       dplyr::filter(.data[[typeColumn]] == evalType) %>% 
       dplyr::select(
-        .data$predictionThreshold,
-        .data$truePositiveCount,
-        .data$trueNegativeCount,
-        .data$falsePositiveCount,
-        .data$falseNegativeCount
+        "predictionThreshold",
+        "truePositiveCount",
+        "trueNegativeCount",
+        "falsePositiveCount",
+        "falseNegativeCount"
       )
   
   x<- x[order(x$predictionThreshold,-x$truePositiveCount, -x$falsePositiveCount),]
@@ -460,11 +460,11 @@ plotPreferencePDF <- function(
     x <- plpResult$performanceEvaluation$thresholdSummary %>% 
       dplyr::filter(.data[[typeColumn]] == evalType) %>% 
       dplyr::select(
-        .data$preferenceThreshold,
-        .data$truePositiveCount,
-        .data$trueNegativeCount,
-        .data$falsePositiveCount,
-        .data$falseNegativeCount
+        "preferenceThreshold",
+        "truePositiveCount",
+        "trueNegativeCount",
+        "falsePositiveCount",
+        "falseNegativeCount"
       )
   
   x<- x[order(x$preferenceThreshold,-x$truePositiveCount, x$trueNegativeCount),]
@@ -551,27 +551,27 @@ plotPrecisionRecall <- function(
     
     N <- max(plpResult$performanceEvaluation$thresholdSummary %>% 
         dplyr::filter(.data[[typeColumn]] == evalType) %>% 
-        dplyr::select(.data$falseCount) %>% 
+        dplyr::select("falseCount") %>% 
         dplyr::pull(), na.rm = T)
 
     
     O <- max(plpResult$performanceEvaluation$thresholdSummary %>% 
         dplyr::filter(.data[[typeColumn]] == evalType) %>% 
-        dplyr::select(.data$trueCount) %>% 
+        dplyr::select("trueCount") %>% 
         dplyr::pull(), na.rm = T)
     
     inc <- O/(O + N)
     
     x <- plpResult$performanceEvaluation$thresholdSummary %>% 
       dplyr::filter(.data[[typeColumn]] == evalType) %>% 
-      dplyr::select(.data$positivePredictiveValue, .data$sensitivity)
+      dplyr::select("positivePredictiveValue", "sensitivity")
     
     plots[[i]] <- ggplot2::ggplot(x, ggplot2::aes(.data$sensitivity, .data$positivePredictiveValue)) +
-      ggplot2::geom_line(size=1) +
+      ggplot2::geom_line(linewidth=1) +
       ggplot2::scale_x_continuous("Recall")+#, limits=c(0,1)) +
       ggplot2::scale_y_continuous("Precision") + #, limits=c(0,1))
       ggplot2::geom_hline(yintercept = inc, linetype="dashed", 
-        color = "red", size=1)  +
+        color = "red", linewidth = 1)  +
       ggplot2::ggtitle(evalType)
   }
   
@@ -622,7 +622,7 @@ plotF1Measure <- function(
     
     x <- plpResult$performanceEvaluation$thresholdSummary %>% 
       dplyr::filter(.data[[typeColumn]] == evalType) %>% 
-      dplyr::select(.data$predictionThreshold, .data$f1Score)
+      dplyr::select("predictionThreshold", "f1Score")
     
   if(sum(is.nan(x$f1Score))>0){
     x <- x[!is.nan(x$f1Score),]
@@ -630,8 +630,8 @@ plotF1Measure <- function(
   }
   
   plots[[i]] <- ggplot2::ggplot(x, ggplot2::aes(.data$predictionThreshold, .data$f1Score)) +
-    ggplot2::geom_line(size=1) +
-    ggplot2::geom_point(size=1) +
+    ggplot2::geom_line(linewidth = 1) +
+    ggplot2::geom_point(size = 1) +
     ggplot2::scale_x_continuous("predictionThreshold")+#, limits=c(0,1)) +
     ggplot2::scale_y_continuous("F1Score") +#, limits=c(0,1))
     ggplot2::ggtitle(evalType)
@@ -688,11 +688,11 @@ plotDemographicSummary <- function(
     x <- plpResult$performanceEvaluation$demographicSummary %>% 
       dplyr::filter(.data[[typeColumn]] == evalType) %>% 
       dplyr::select(
-        .data$ageGroup,
-        .data$genGroup,
-        .data$averagePredictedProbability,
-        .data$PersonCountAtRisk, 
-        .data$PersonCountWithOutcome
+        "ageGroup",
+        "genGroup",
+        "averagePredictedProbability",
+        "PersonCountAtRisk", 
+        "PersonCountWithOutcome"
         )
   
     # remove -1 values:
@@ -729,10 +729,10 @@ plotDemographicSummary <- function(
     ci <- plpResult$performanceEvaluation$demographicSummary %>% 
       dplyr::filter(.data[[typeColumn]] == evalType) %>% 
       dplyr::select(
-        .data$ageGroup,
-        .data$genGroup,
-        .data$averagePredictedProbability,
-        .data$StDevPredictedProbability
+        "ageGroup",
+        "genGroup",
+        "averagePredictedProbability",
+        "StDevPredictedProbability"
       )
     
     ci$StDevPredictedProbability[is.na(ci$StDevPredictedProbability)] <- 1
@@ -769,7 +769,7 @@ plotDemographicSummary <- function(
                                   guide = ggplot2::guide_legend(title = NULL),
                                   labels = c("Expected", "Observed")) +
 
-      ggplot2::guides(linetype=FALSE) +
+      ggplot2::guides( linetype = "none") + # change from FALSE due to warning
       ggplot2::ggtitle(evalType)
   }
   
@@ -821,7 +821,7 @@ plotSparseCalibration <- function(
     evalType <- evalTypes[i]
     x <- plpResult$performanceEvaluation$calibrationSummary %>% 
       dplyr::filter(.data[[typeColumn]] == evalType) %>% 
-      dplyr::select(.data$averagePredictedProbability, .data$observedIncidence)
+      dplyr::select("averagePredictedProbability", "observedIncidence")
 
   maxVal <- max(x$averagePredictedProbability,x$observedIncidence)
   model <- stats::lm(observedIncidence~averagePredictedProbability, data=x)
@@ -844,9 +844,9 @@ plotSparseCalibration <- function(
                                        )) +
     ggplot2::geom_ribbon(ggplot2::aes(ymin=.data$lci,ymax=.data$uci, x=x), 
                          fill="blue", alpha=0.2) +
-    ggplot2::geom_point(size=1, color='darkblue') +
+    ggplot2::geom_point(size = 1, color='darkblue') +
     ggplot2::coord_cartesian(ylim = c(0, maxVal), xlim =c(0,maxVal)) +
-    ggplot2::geom_abline(intercept = 0, slope = 1, linetype = 2, size=1,
+    ggplot2::geom_abline(intercept = 0, slope = 1, linetype = 2, linewidth = 1,
                          show.legend = TRUE) +
     ggplot2::geom_abline(intercept = res['Intercept'], slope = res['Gradient'],
                          linetype = 1,show.legend = TRUE,
@@ -903,7 +903,7 @@ plotSparseCalibration2 <- function(
     evalType <- evalTypes[i]
     x <- plpResult$performanceEvaluation$calibrationSummary %>% 
       dplyr::filter(.data[[typeColumn]] == evalType) %>% 
-      dplyr::select(.data$averagePredictedProbability, .data$observedIncidence, .data$PersonCountAtRisk)
+      dplyr::select("averagePredictedProbability", "observedIncidence", "PersonCountAtRisk")
 
   cis <- apply(x, 1, function(x) stats::binom.test(round(x[2]*x[3]), x[3], alternative = c("two.sided"), conf.level = 0.95)$conf.int)
   x$lci <- cis[1,]  
@@ -919,7 +919,7 @@ plotSparseCalibration2 <- function(
     ggplot2::geom_point(size=2, color='black') +
     ggplot2::geom_errorbar(limits) +
     ggplot2::geom_line(colour='darkgrey') +
-    ggplot2::geom_abline(intercept = 0, slope = 1, linetype = 5, size=0.4,
+    ggplot2::geom_abline(intercept = 0, slope = 1, linetype = 5, linewidth=0.4,
                          show.legend = TRUE) +
     ggplot2::scale_x_continuous("Average Predicted Probability") +
     ggplot2::scale_y_continuous("Observed Fraction With Outcome") +
@@ -993,7 +993,7 @@ plotSmoothCalibration <- function(plpResult,
     if('prediction'%in%names(plpResult)) {
       x <- plpResult$performanceEvaluation$calibrationSummary %>% 
         dplyr::filter(.data[[typeColumn]] == evalType) %>% 
-        dplyr::select(.data$averagePredictedProbability, .data$observedIncidence)
+        dplyr::select("averagePredictedProbability", "observedIncidence")
       
       prediction <-  plpResult$prediction %>% dplyr::filter(.data$evaluationType == evalType)
       
@@ -1042,39 +1042,28 @@ plotSmoothCalibration <- function(plpResult,
         }
         # loess
         smoothData <- data.frame(y, p)
-        fit <- stats::loess(y ~ p, degree = 2)
-        predictedFit <- stats::predict(fit, se = TRUE)
-        smoothData <- smoothData %>%
-          dplyr::mutate(
-            calibration = predictedFit$fit,
-            se = predictedFit$se,
-            lci = .data$calibration - stats::qt(.975, predictedFit$df) * .data$se,
-            uci = .data$calibration + stats::qt(.975, predictedFit$df) * .data$se
-          )
-
-        xlim <- ylim <- c(0, 1)
+        # xlim <- ylim <- c(0, 1)
         smoothPlot <- plotSmoothCalibrationLoess(data = smoothData, span = span) +
           ggplot2::coord_cartesian(
-            xlim = xlim,
-            ylim = ylim
+            xlim = c(0, maxes),
+            ylim = c(0, maxes)
           )
       } else {
         # Restricted cubic splines
 
         smoothData <- data.frame(y, p)
-        smoothPlot <- plotSmoothCalibrationRcs(data = smoothData, nKnots = nKnots)
+        smoothPlot <- plotSmoothCalibrationRcs(data = smoothData, numberOfKnots = nKnots)
         if (is.character(smoothPlot)) {
           plots[[i]] <- smoothPlot
           failedEvalType[evalTypes[i]] <- TRUE
           next
         }
-        xlim <- ylim <- c(0, 1)
+
         smoothPlot <- smoothPlot +
           ggplot2::coord_cartesian(
-            xlim = xlim,
-            ylim = ylim
+            xlim = c(0, maxes),
+            ylim = c(0, maxes)
           )
-
       }
       # construct the plot grid
       if (scatter) {
@@ -1112,7 +1101,7 @@ plotSmoothCalibration <- function(plpResult,
           strip.text = ggplot2::element_blank()
         ) +
         ggplot2::labs(x = "Predicted Probability") +
-        ggplot2::coord_cartesian(xlim = xlim)
+        ggplot2::coord_cartesian(xlim = c(0, maxes))
       
     } else {
       # use calibrationSummary
@@ -1259,15 +1248,14 @@ plotSmoothCalibrationLoess <- function(data, span = 0.75) {
     fill = "blue",
     alpha = 0.2
   ) +
-  ggplot2::geom_segment(
-    ggplot2::aes(
-      x = 0,
-      xend = 1,
-      y = 0,
-      yend = 1,
-      color = "Ideal",
-      linetype = "Ideal"
-    )
+  ggplot2::annotate(
+    geom = "segment",
+    x = 0,
+    xend = 1,
+    y = 0,
+    yend = 1,
+    color = "red",
+    linetype = "dashed"
   ) +
   ggplot2::scale_linetype_manual(
     name = "Models",
@@ -1291,36 +1279,92 @@ plotSmoothCalibrationLoess <- function(data, span = 0.75) {
   return(plot)
 }
 
-plotSmoothCalibrationRcs <- function(data, nKnots) {
+plotSmoothCalibrationRcs <- function(data, numberOfKnots) {
+  data <- data %>%
+    dplyr::filter(!is.na(.data$y) & !is.na(.data$p))
   p <- data$p
-  for (k in nKnots:3) {
-    formSmooth <- paste0('y ~ rms::rcs(p, ', k, ')')
-    smoothFit <- suppressWarnings(rms::lrm(stats::as.formula(formSmooth), data = data, x = TRUE, y = TRUE))
-    smoothFitFail <- smoothFit$fail
-    if (smoothFitFail) {
+
+  .defineKnots <- function(predictedProbabilities, numberOfKnots) {
+    if (numberOfKnots == 3) {
+      lowestQuantile <- .1
+      highestQuantile <- .9
+    } else if (numberOfKnots > 3 & numberOfKnots <= 6) {
+      lowestQuantile <- .05
+      highestQuantile <- .95
+    } else if (numberOfKnots == 7) {
+      lowestQuantile <- .025
+      highestQuantile <- .975
+    } else {
+        # use mgcv defaults
+        return(numberOfKnots)
+    }
+    knotQuantiles <- seq(
+      lowestQuantile,
+      highestQuantile,
+      length.out = numberOfKnots
+    )
+
+    knotLocation <- stats::quantile(
+      x = predictedProbabilities,
+      probs = knotQuantiles,
+      na.rm = TRUE
+    )
+
+    return(knotLocation)
+  }
+
+  for (k in numberOfKnots:3) {
+    if (k > 7) {
+      smoothFit <- tryCatch(
+        expr = {
+          mgcv::gam(
+            y ~ s(p, bs = 'cr', k = k, m = 2),
+            data = data,
+            family = stats::binomial()
+          )
+        },
+        error = function(e) {
+          return("Failed")
+        }
+      )
+    } else {
+      smoothFit <- tryCatch(
+        expr = {
+          mgcv::gam(
+            y ~ s(p, bs = 'cr', k = k, m = 2),
+            data = data,
+            knots = list(p = .defineKnots(p, k)),
+            family = stats::binomial()
+          )
+        },
+        error = function(e) {
+          return("Failed")
+        }
+      )
+    }
+    if (is.character(smoothFit)) {
       if (k > 3) {
         ParallelLogger::logInfo(paste0("Setting number of Knots to ", k, " led to estimation problems. Switching to nKnots = ", k-1))
       } else {
         ParallelLogger::logInfo(paste0('Unable to fit model'))
-        plot <- "Failed"
       }
+    } else {
+      break
     }
   }
 
-  # If the fit failed for all nKnots return "Failed"
-  if (smoothFitFail) {
-    return(plot)
+  if (is.character(smoothFit)){
+    return("Failed")
   }
 
-  xRange <- seq(0, p[length(p)], length.out = 1000)
-  pred <- stats::predict(smoothFit, xRange, se.fit = T, type = "lp")
-  predXRange <- stats::plogis(pred$linear.predictors)
-  ciSmooth <- data.frame(
-    lci = stats::plogis(pred$linear.predictors - 1.96 * pred$se.fit),
-    uci = stats::plogis(pred$linear.predictors + 1.96 * pred$se.fit)
+  xRange <- seq(min(p), max(p), length.out = 1e3)
+  predictWithSe <- stats::predict(smoothFit, newdata = data.frame(p = xRange), se.fit = TRUE)
+  smoothData <- data.frame(
+    xRange = xRange,
+    predXRange = stats::plogis(predictWithSe$fit),
+    lci = stats::plogis(predictWithSe$fit - 1.96 * predictWithSe$se.fit),
+    uci = stats::plogis(predictWithSe$fit + 1.96 * predictWithSe$se.fit)
   )
-
-  smoothData <- cbind(xRange, predXRange, ciSmooth)
   plot <- ggplot2::ggplot(
     data = smoothData,
     ggplot2::aes(
@@ -1340,17 +1384,17 @@ plotSmoothCalibrationRcs <- function(data, nKnots) {
         ymax = .data$uci
       ),
       fill = "blue",
-      alpha = 0.2
+      alpha = 0.2,
+      show.legend = FALSE
     ) +
-    ggplot2::geom_segment(
-      ggplot2::aes(
-        x = 0,
-        xend = 1,
-        y = 0,
-        yend = 1,
+    ggplot2::geom_abline(
+      mapping = ggplot2::aes(
+        slope = 1,
+        intercept = 0,
         color = "Ideal",
         linetype = "Ideal"
-      )
+      ),
+      show.legend = FALSE
     ) +
     ggplot2::scale_color_manual(
       name = "Models",
@@ -1361,7 +1405,7 @@ plotSmoothCalibrationRcs <- function(data, nKnots) {
       values = c(rcs = "solid", Ideal = "dashed")
     ) +
     ggplot2::labs(x = "", y = "Observed Probability")
-               
+
   return(plot)
 }
 
@@ -1397,43 +1441,44 @@ plotPredictionDistribution <- function(
   plots <- list()
   length(plots) <- length(evalTypes)
   
-  for(i in 1:length(evalTypes)){
+  for (i in 1:length(evalTypes)) {
     evalType <- evalTypes[i]
     x <- plpResult$performanceEvaluation$predictionDistribution %>% 
       dplyr::filter(.data[[typeColumn]] == evalType) 
   
-  non05 <- x$P05PredictedProbability[x$class==0]
-  non95 <- x$P95PredictedProbability[x$class==0]
-  one05 <- x$P05PredictedProbability[x$class==1]
-  one95 <- x$P95PredictedProbability[x$class==1]
-  
-  plots[[i]] <-   ggplot2::ggplot(x, ggplot2::aes(x=as.factor(.data$class),
-                                            ymin=.data$MinPredictedProbability,
-                                            lower=.data$P25PredictedProbability,
-                                            middle=.data$MedianPredictedProbability,
-                                            upper=.data$P75PredictedProbability, 
-                                            ymax=.data$MaxPredictedProbability, 
-                                            color=as.factor(.data$class))) + 
+  non05 <- x$P05PredictedProbability[x$class == 0]
+  non95 <- x$P95PredictedProbability[x$class == 0]
+  one05 <- x$P05PredictedProbability[x$class == 1]
+  one95 <- x$P95PredictedProbability[x$class == 1]
+  
+  plots[[i]] <- ggplot2::ggplot(x,
+    ggplot2::aes(x = as.factor(class),
+                 ymin = .data$MinPredictedProbability,
+                 lower = .data$P25PredictedProbability,
+                 middle = .data$MedianPredictedProbability,
+                 upper = .data$P75PredictedProbability, 
+                 ymax = .data$MaxPredictedProbability, 
+                 color = as.factor(.data$class))) + 
     ggplot2::coord_flip() +
-    ggplot2::geom_boxplot(stat="identity")  +
+    ggplot2::geom_boxplot(stat = "identity")  +
     ggplot2::scale_x_discrete("Class") + 
     ggplot2::scale_y_continuous("Predicted Probability") + 
-    ggplot2::theme(legend.position="none") +
-    ggplot2::geom_segment(ggplot2::aes(x = 0.9, y = non05, 
-                     xend = 1.1, yend = non05), color='red') +
-    ggplot2::geom_segment(ggplot2::aes(x = 0.9, y = non95, 
-                                       xend = 1.1, yend = non95), color='red') +
-  ggplot2::geom_segment(ggplot2::aes(x = 1.9, y = one05, 
-                                     xend = 2.1, yend = one05)) +
-    ggplot2::geom_segment(ggplot2::aes(x = 1.9, y = one95, 
-                                       xend = 2.1, yend = one95)) +
-    ggplot2::ggtitle(evalType)
+    ggplot2::theme(legend.position = "none") +
+    ggplot2::annotate("segment", x = 0.9, xend = 1.1, y = non05, yend = non05,
+                      color = "red") +
+    ggplot2::annotate("segment", x = 0.9, xend = 1.1, y = non95, yend = non95,
+                      color = "red") +
+    ggplot2::annotate("segment", x = 1.9, xend = 2.1, y = one05, yend = one05,
+                      color = "#00BFC4") +
+    ggplot2::annotate("segment", x = 1.9, xend = 2.1, y = one95, yend = one95,
+                      color = "#00BFC4") +
+     ggplot2::ggtitle(evalType)
   }
   
-  plot <- gridExtra::marrangeGrob(plots, nrow=length(plots), ncol=1)
+  plot <- gridExtra::marrangeGrob(plots, nrow = length(plots), ncol = 1)
   
-  if (!is.null(saveLocation)){
-    if(!dir.exists(saveLocation)){
+  if (!is.null(saveLocation)) {
+    if (!dir.exists(saveLocation)) {
       dir.create(saveLocation, recursive = T)
     }
     ggplot2::ggsave(file.path(saveLocation, fileName), plot, width = 5, height = 4.5, dpi = 400)
diff --git a/R/PopulationSettings.R b/R/PopulationSettings.R
index 63d9685be..098a784f7 100644
--- a/R/PopulationSettings.R
+++ b/R/PopulationSettings.R
@@ -134,7 +134,8 @@ createStudyPopulationSettings <- function(
     startAnchor = startAnchor,
     riskWindowEnd = riskWindowEnd,
     endAnchor = endAnchor, 
-    restrictTarToCohortEnd = restrictTarToCohortEnd)
+    restrictTarToCohortEnd = restrictTarToCohortEnd
+    )
   
   class(result) <- 'populationSettings'
   return(result)
@@ -190,7 +191,7 @@ createStudyPopulation <- function(
   restrictTarToCohortEnd <- populationSettings$restrictTarToCohortEnd
   
   # parameter checks
-  if(!class(plpData)%in%c('plpData')){
+  if(!inherits(x = plpData, what = c('plpData'))){
     ParallelLogger::logError('Check plpData format')
     stop('Wrong plpData input')
   }
@@ -214,13 +215,16 @@ createStudyPopulation <- function(
 
   if (is.null(population)) {
     population <- plpData$cohorts
+  } else {
+    population <- plpData$cohorts %>% 
+      dplyr::filter(.data$rowId %in% (population %>% dplyr::pull(.data$rowId)))
   }
   
-  # save the metadata (should have the cohortId, outcomeId, plpDataSettings and population settings)
+  # save the metadata (should have the ?targetId, outcomeId, plpDataSettings and population settings)
   metaData <- attr(population, "metaData")
-  metaData$plpDataSettings <- plpData$metaData$restrictPlpDataSettings
+  metaData$restrictPlpDataSettings <- plpData$metaData$restrictPlpDataSettings
   metaData$outcomeId <- outcomeId
-  metaData$populationSettings <- populationSettings
+  metaData$populationSettings <- populationSettings # this will overwrite an existing setting
   
   # set the existing attrition
   if(is.null(metaData$attrition)){
@@ -258,10 +262,10 @@ createStudyPopulation <- function(
     
   
   # get the outcomes during TAR
-  outcomeTAR <- population %>% 
-    dplyr::inner_join(plpData$outcomes, by ='rowId') %>% 
-    dplyr::filter(.data$outcomeId == get('oId'))  %>% 
-    dplyr::select(.data$rowId, .data$daysToEvent, .data$tarStart, .data$tarEnd) %>% 
+  outcomeTAR <- plpData$outcomes %>% 
+    dplyr::filter(.data$outcomeId == get("oId")) %>%
+    dplyr::inner_join(population, by = "rowId") %>%
+    dplyr::select("rowId", "daysToEvent", "tarStart", "tarEnd") %>% 
     dplyr::filter(.data$daysToEvent >= .data$tarStart & .data$daysToEvent <= .data$tarEnd)  
   
   # prevent warnings when no results left
@@ -270,11 +274,11 @@ createStudyPopulation <- function(
     dplyr::group_by(.data$rowId) %>%
     dplyr::summarise(first = min(.data$daysToEvent),
                      ocount = length(unique(.data$daysToEvent)))  %>% 
-    dplyr::select(.data$rowId, .data$first, .data$ocount)
+    dplyr::select("rowId", "first", "ocount")
   } else {
     outcomeTAR <- outcomeTAR %>% 
       dplyr::mutate(first = 0, ocount = 0) %>% 
-      dplyr::select(.data$rowId, .data$first, .data$ocount) 
+      dplyr::select("rowId", "first", "ocount") 
   }
   
   population <- population %>%
@@ -293,10 +297,12 @@ createStudyPopulation <- function(
   if (firstExposureOnly) {
     ParallelLogger::logTrace(paste("Restricting to first exposure"))
     
-    population <- population %>%
-      dplyr::arrange(.data$subjectId,.data$cohortStartDate) %>%
-      dplyr::group_by(.data$subjectId) %>%
-      dplyr::filter(dplyr::row_number(.data$subjectId)==1)
+    if (nrow(population) > dplyr::n_distinct(population$subjectId)) {
+      population <- population %>%
+        dplyr::arrange(.data$subjectId,.data$cohortStartDate) %>%
+        dplyr::group_by(.data$subjectId) %>%
+        dplyr::filter(dplyr::row_number(.data$subjectId)==1)
+    }
     
     attrRow <- population %>% dplyr::group_by() %>%
       dplyr::summarise(outcomeId = get('oId'),
@@ -328,17 +334,17 @@ createStudyPopulation <- function(
       ParallelLogger::logTrace("Removing subjects with prior outcomes (if any)")
     
     # get the outcomes during TAR
-    outcomeBefore <- population %>% 
-      dplyr::inner_join(plpData$outcomes, by ='rowId') %>% 
+    outcomeBefore <- plpData$outcomes %>% 
       dplyr::filter(outcomeId == get('oId'))  %>% 
-      dplyr::select(.data$rowId, .data$daysToEvent, .data$tarStart) %>% 
+      dplyr::inner_join(population, by = 'rowId') %>% 
+      dplyr::select("rowId", "daysToEvent", "tarStart") %>% 
       dplyr::filter(.data$daysToEvent < .data$tarStart & .data$daysToEvent > -get('priorOutcomeLookback') ) 
     
     if(nrow(as.data.frame(outcomeBefore))>0){
       outcomeBefore %>%
         dplyr::group_by(.data$rowId) %>%
         dplyr::summarise(first = min(.data$daysToEvent))  %>% 
-        dplyr::select(.data$rowId)
+        dplyr::select("rowId")
     }
       
     population <- population %>%
@@ -418,9 +424,9 @@ createStudyPopulation <- function(
     dplyr::mutate(timeAtRisk = .data$tarEnd - .data$tarStart + 1 ,
                   survivalTime = ifelse(.data$outcomeCount == 0, .data$tarEnd -.data$tarStart + 1, .data$first - .data$tarStart + 1),
                   daysToEvent = .data$first) %>%
-    dplyr::select(.data$rowId, .data$subjectId, .data$cohortId, .data$cohortStartDate, .data$daysFromObsStart,
-                  .data$daysToCohortEnd, .data$daysToObsEnd, .data$ageYear, .data$gender,
-                  .data$outcomeCount, .data$timeAtRisk, .data$daysToEvent, .data$survivalTime)
+    dplyr::select("rowId", "subjectId", "targetId", "cohortStartDate", "daysFromObsStart",
+                  "daysToCohortEnd", "daysToObsEnd", "ageYear", "gender",
+                  "outcomeCount", "timeAtRisk", "daysToEvent", "survivalTime")
 
     # check outcome still there
     if(sum(!is.na(population$daysToEvent))==0){
diff --git a/R/Predict.R b/R/Predict.R
index 0c17b88a2..14eff8787 100644
--- a/R/Predict.R
+++ b/R/Predict.R
@@ -44,24 +44,34 @@ predictPlp <- function(plpModel, plpData, population, timepoint){
   
   
   # do feature engineering/selection
-  plpData$covariateData <- do.call(
-    applyFeatureengineering, 
-    list(
-      covariateData = plpData$covariateData,
-      settings = plpModel$settings$featureEngineering
+  if(!is.null(plpModel$preprocessing$featureEngineering)){
+    plpData <- do.call(
+      applyFeatureengineering, 
+      list(
+        plpData = plpData,
+        settings = plpModel$preprocessing$featureEngineering
+      )
     )
-  )
+    featureEngineering <- T
+  } else{
+    featureEngineering <- F
+  }
   
   ParallelLogger::logTrace('did FE')
   
-  # do preprocessing
-  plpData$covariateData <- do.call(
-    applyTidyCovariateData, 
-    list(
-      covariateData = plpData$covariateData,
-      preprocessSettings = plpModel$settings$tidyCovariates
+  if(!is.null(plpModel$preprocessing$tidyCovariates)){
+    # do preprocessing
+    plpData$covariateData <- do.call(
+      applyTidyCovariateData, 
+      list(
+        covariateData = plpData$covariateData,
+        preprocessSettings = plpModel$preprocessing$tidyCovariates
+      )
     )
-  )
+    tidyCovariates <- T
+  } else{
+    tidyCovariates <- F
+  }
   
   ParallelLogger::logTrace('did tidy')
   
@@ -90,10 +100,14 @@ predictPlp <- function(plpModel, plpData, population, timepoint){
   
   # add metaData
   metaData$modelType <- attr(plpModel, 'modelType') #"binary", 
-  metaData$cohortId <- attr(population,'metaData')$cohortId
+  metaData$targetId <- attr(population,'metaData')$targetId
   metaData$outcomeId <- attr(population,'metaData')$outcomeId
   metaData$timepoint <- timepoint
   
+  # added information about running preprocessing/FE
+  metaData$tidyCovariates <- tidyCovariates
+  metaData$featureEngineering <- featureEngineering
+  
   attr(prediction, "metaData") <- metaData
   return(prediction)
 }
@@ -101,7 +115,7 @@ predictPlp <- function(plpModel, plpData, population, timepoint){
 
 
 applyFeatureengineering <- function(
-  covariateData,
+  plpData, 
   settings
 ){
   
@@ -112,12 +126,12 @@ applyFeatureengineering <- function(
   
   # add code for implementing the feature engineering
   for(set in settings){
-    set$settings$trainData <- covariateData
-    covariateData <- do.call(eval(parse(text = set$funct)), set$settings)
+    set$settings$trainData <- plpData
+    plpData <- do.call(eval(parse(text = set$funct)), set$settings)
   }
   
   # dont do anything for now
-  return(covariateData)
+  return(plpData)
   
 }
 
@@ -154,10 +168,10 @@ applyTidyCovariateData <- function(
   
   if(!is.null(maxs)){
     if('bins'%in%colnames(maxs)){
-      covariateData$maxes <- tibble::as_tibble(maxs)  %>% dplyr::rename(covariateId = .data$bins) %>% 
-        dplyr::rename(maxValue = .data$maxs)
+      covariateData$maxes <- dplyr::as_tibble(maxs)  %>% dplyr::rename(covariateId = "bins") %>% 
+        dplyr::rename(maxValue = "maxs")
     } else{
-      covariateData$maxes <- maxs #tibble::as_tibble(maxs)  %>% dplyr::rename(covariateId = bins)
+      covariateData$maxes <- maxs 
     }
     on.exit(covariateData$maxes <- NULL, add = TRUE)
     
@@ -170,8 +184,8 @@ applyTidyCovariateData <- function(
       dplyr::inner_join(covariateData$includeCovariates, by='covariateId') %>% # added as join
       dplyr::inner_join(covariateData$maxes, by = 'covariateId') %>%
       dplyr::mutate(value = 1.0*.data$covariateValue/.data$maxValue) %>%
-      dplyr::select(- .data$covariateValue) %>%
-      dplyr::rename(covariateValue = .data$value)
+      dplyr::select(-"covariateValue") %>%
+      dplyr::rename(covariateValue = "value")
   } else{
     newCovariateData$covariates <- covariateData$covariates %>% 
       dplyr::inner_join(covariateData$includeCovariates, by='covariateId')
diff --git a/R/PreprocessingData.R b/R/PreprocessingData.R
index d0e78b485..834d27a85 100644
--- a/R/PreprocessingData.R
+++ b/R/PreprocessingData.R
@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#' Create the settings for preprocessing the trainData using \code{ }.
+#' Create the settings for preprocessing the trainData.
 #'
 #' @details
 #' Returns an object of class \code{preprocessingSettings} that specifies how to preprocess the training data
@@ -66,6 +66,7 @@ preprocessData <- function (covariateData,
                             preprocessSettings){
   
   metaData <- attr(covariateData, "metaData")
+  preprocessSettingsInput <- preprocessSettings # saving this before adding covariateData
   
   checkIsClass(covariateData, c("CovariateData"))
   checkIsClass(preprocessSettings, c("preprocessSettings"))
@@ -77,15 +78,17 @@ preprocessData <- function (covariateData,
   preprocessSettings$covariateData <- covariateData
   covariateData <- do.call(FeatureExtraction::tidyCovariateData, preprocessSettings)
   
-  #update covariateRed
-  removed <- unique(
+  #update covariateRef
+  removed <- unique(c(
     attr(covariateData, "metaData")$deletedInfrequentCovariateIds,
     attr(covariateData, "metaData")$deletedRedundantCovariateIds
     )
+    )
   covariateData$covariateRef <- covariateData$covariateRef %>% 
     dplyr::filter(!.data$covariateId  %in% removed)
   
   metaData$tidyCovariateDataSettings <- attr(covariateData, "metaData")
+  metaData$preprocessSettings <- preprocessSettingsInput
   attr(covariateData, "metaData") <- metaData
   
   return(covariateData)
diff --git a/R/RClassifier.R b/R/RClassifier.R
index f08ed9f89..a74e2358b 100644
--- a/R/RClassifier.R
+++ b/R/RClassifier.R
@@ -1,5 +1,12 @@
 # this is a generic wrapper for training models using classifiers in R
-fitRclassifier <- function(trainData, param, search = 'grid', analysisId){
+fitRclassifier <- function(
+  trainData, 
+  modelSettings, 
+  search = 'grid', 
+  analysisId,
+  ...){
+  
+  param <- modelSettings$param
   
   if (!FeatureExtraction::isCovariateData(trainData$covariateData)){
     stop("Needs correct covariateData")
@@ -52,34 +59,37 @@ fitRclassifier <- function(trainData, param, search = 'grid', analysisId){
   result <- list(
     model = cvResult$model,
     
+    preprocessing = list(
+      featureEngineering = attr(trainData, "metaData")$featureEngineering,
+      tidyCovariates = attr(trainData$covariateData, "metaData")$tidyCovariateDataSettings, 
+      requireDenseMatrix = F
+    ),
+    
     prediction = prediction,
     
-    settings = list(
-      plpDataSettings = attr(trainData, "metaData")$plpDataSettings,
+    modelDesign = PatientLevelPrediction::createModelDesign(
+      targetId = attr(trainData, "metaData")$targetId,
+      outcomeId = attr(trainData, "metaData")$outcomeId,
+      restrictPlpDataSettings = attr(trainData, "metaData")$restrictPlpDataSettings,
       covariateSettings = attr(trainData, "metaData")$covariateSettings,
-      featureEngineering = attr(trainData$covariateData, "metaData")$featureEngineering,
-      tidyCovariates = attr(trainData$covariateData, "metaData")$tidyCovariateDataSettings, 
-      #covariateMap = covariateMap, this is in covariateImportance
-      requireDenseMatrix = F,
       populationSettings = attr(trainData, "metaData")$populationSettings,
-      modelSettings = list(
-        model = attr(param, 'settings')$trainRFunction, 
-        param = param,
-        finalModelParameters = cvResult$finalParam,
-        extraSettings = attr(param, 'settings')
-      ),
+      featureEngineeringSettings = attr(trainData$covariateData, "metaData")$featureEngineeringSettings,
+      preprocessSettings = attr(trainData$covariateData, "metaData")$preprocessSettings,
+      modelSettings = modelSettings,
       splitSettings = attr(trainData, "metaData")$splitSettings,
       sampleSettings = attr(trainData, "metaData")$sampleSettings
     ),
     
     trainDetails = list(
       analysisId = analysisId,
-      cdmDatabaseSchema = attr(trainData, "metaData")$cdmDatabaseSchema,
-      outcomeId = attr(trainData, "metaData")$outcomeId,
-      cohortId = attr(trainData, "metaData")$cohortId,
+      analysisSource = '', #TODO add from model
+      developmentDatabase = attr(trainData, "metaData")$cdmDatabaseName,
+      developmentDatabaseSchema = attr(trainData, "metaData")$cdmDatabaseSchema, 
       attrition = attr(trainData, "metaData")$attrition, 
-      trainingTime = comp,
+      trainingTime = paste(as.character(abs(comp)), attr(comp,'units')),
       trainingDate = Sys.Date(),
+      modelName = attr(param, 'settings')$trainRFunction, 
+      finalModelParameters = cvResult$finalParam,
       hyperParamSearch = hyperSummary
     ),
     
diff --git a/R/Recalibration.R b/R/Recalibration.R
index bc2493e49..2053f7ed7 100644
--- a/R/Recalibration.R
+++ b/R/Recalibration.R
@@ -39,17 +39,20 @@ recalibratePlpRefit <- function(
   newPopulation, 
   newData
 ){
-  if (is.null(newPopulation))
+  if (is.null(newPopulation)){
     stop("NULL population")
-  if (class(newData) != "plpData")
+  }
+  if (!inherits(x = newData, what =  "plpData")){
     stop("Incorrect plpData class")
-  if (class(plpModel) != "plpModel")
+  }
+  if (!inherits(x = plpModel, what = "plpModel")){
     stop("plpModel is not of class plpModel")
+  }
   
   #get selected covariates
   includeCovariateIds <- plpModel$covariateImportance %>% 
     dplyr::filter(.data$covariateValue != 0) %>% 
-    dplyr::select(.data$covariateId) %>% 
+    dplyr::select("covariateId") %>% 
     dplyr::pull()
   
   # check which covariates are included in new data
@@ -61,27 +64,42 @@ recalibratePlpRefit <- function(
   
   setLassoRefit <- setLassoLogisticRegression(
     includeCovariateIds = includeCovariateIds,
-    noShrinkage = noShrinkage
+    noShrinkage = noShrinkage, 
+    maxIterations = 10000 # increasing this due to test code often not converging
   )
   
-  newData$labels <- newPopulation #%>% 
-    #dplyr::select(
-    #  .data$rowId, 
-    #  .data$cohortStartDate,
-    #  .data$outcomeCount, 
-     # .data$survivalTime
-      #)
+  newData$labels <- newPopulation 
   
   newData$folds <- data.frame(
     rowId = newData$labels$rowId, 
     index = sample(2, length(newData$labels$rowId), replace = T)
     )
   
-  newModel <- fitPlp(
-    trainData = newData, 
-    modelSettings = setLassoRefit,
-    analysisId = 'recalibrationRefit'
+  # add dummy settings to fit model
+  attr(newData, "metaData")$outcomeId <- attr(newPopulation, 'metaData')$outcomeId
+  attr(newData, "metaData")$targetId <- attr(newPopulation, 'metaData')$targetId
+  attr(newData, "metaData")$restrictPlpDataSettings <- attr(newPopulation, 'metaData')$restrictPlpDataSettings
+  attr(newData, "metaData")$covariateSettings <- newData$metaData$covariateSettings
+  attr(newData, "metaData")$populationSettings <- attr(newPopulation, 'metaData')$populationSettings
+  attr(newData$covariateData, "metaData")$featureEngineeringSettings <- PatientLevelPrediction::createFeatureEngineeringSettings()
+  attr(newData$covariateData, "metaData")$preprocessSettings <- PatientLevelPrediction::createPreprocessSettings()
+  attr(newData, "metaData")$splitSettings <- PatientLevelPrediction::createDefaultSplitSetting()
+  attr(newData, "metaData")$sampleSettings <- PatientLevelPrediction::createSampleSettings()
+  
+  newModel <- tryCatch({
+    fitPlp(
+      trainData = newData, 
+      modelSettings = setLassoRefit,
+      analysisId = 'recalibrationRefit',
+      analysisPath = NULL
     )
+  }, 
+  error = function(e){ParallelLogger::logInfo(e); return(NULL)}
+  )
+  if(is.null(newModel)){
+    ParallelLogger::logInfo('Recalibration fit failed')
+    return(NULL)
+  }
   
   newModel$prediction$evaluationType <- 'recalibrationRefit'
 
@@ -103,8 +121,8 @@ recalibratePlpRefit <- function(
     adjust <- newModel$covariateImportance %>% 
       dplyr::filter(.data$covariateValue != 0) %>% 
       dplyr::select(
-        .data$covariateId, 
-        .data$covariateValue
+        "covariateId", 
+        "covariateValue"
       )
   } else{
     adjust <- c()
@@ -139,12 +157,13 @@ recalibratePlpRefit <- function(
 recalibratePlp <- function(prediction, analysisId, typeColumn = 'evaluationType',
                            method = c('recalibrationInTheLarge', 'weakRecalibration')){
   # check input:
-    if (class(prediction) != 'data.frame')
+    if (!inherits(x = prediction, what =  'data.frame')){
       stop("Incorrect prediction") 
+    }
   
-  if(!method  %in% c('recalibrationInTheLarge', 'weakRecalibration'))
+  if(!method  %in% c('recalibrationInTheLarge', 'weakRecalibration')){
     stop("Unknown recalibration method type. must be of type: recalibrationInTheLarge, weakRecalibration")
-  
+  }
   
   prediction <- do.call(method, list(prediction = prediction, columnType = typeColumn))
   
@@ -210,7 +229,8 @@ weakRecalibration <- function(prediction, columnType = 'evaluationType'){
     
     recalibrated <- prediction
     
-    baseline <- ifelse(is.null(attr(recalibrated, "baselineHazard")), 0.9, attr(recalibrated, "baselineHazard"))
+    # this will make the recalibration work if the baselineSurvival is missing
+    baseline <- ifelse(is.null(attr(recalibrated, "baselineSurvival")), 0.9, attr(recalibrated, "baselineSurvival"))
     ParallelLogger::logInfo(paste0('recal initial baseline hazard: ',baseline))
     
     offset <- ifelse(is.null(attr(recalibrated, "offset")), 0, attr(recalibrated, "offset"))
diff --git a/R/RunMultiplePlp.R b/R/RunMultiplePlp.R
index e9efd42c6..9e90f5452 100644
--- a/R/RunMultiplePlp.R
+++ b/R/RunMultiplePlp.R
@@ -25,15 +25,15 @@
 #' @param databaseDetails               The database settings created using \code{createDatabaseDetails()}
 #' @param modelDesignList                A list of model designs created using \code{createModelDesign()}
 #' @param onlyFetchData                  Only fetches and saves the data object to the output folder without running the analysis.
-#' @param splitSettings                  The train/validation/test splitting used by all analyses created using \code{createDefaultSplitSetting()}
 #' @param cohortDefinitions               A list of cohort definitions for the target and outcome cohorts
-#' @param logSettings                    The setting spexcifying the logging for the analyses created using \code{createLogSettings()}
+#' @param logSettings                    The setting specifying the logging for the analyses created using \code{createLogSettings()}
 #' @param saveDirectory                   Name of the folder where all the outputs will written to.
+#' @param sqliteLocation                 (optional) The location of the sqlite database with the results
 #' 
 #' @return
 #' A data frame with the following columns: \tabular{ll}{ \verb{analysisId} \tab The unique identifier
-#' for a set of analysis choices.\cr \verb{cohortId} \tab The ID of the target cohort populations.\cr
-#' \verb{outcomeId} \tab The ID of the outcomeId.\cr \verb{dataLocation} \tab The location where the plpData was saved \cr \verb{evaluationFolder} \tab The name of file containing the evaluation saved as a csv
+#' for a set of analysis choices.\cr \verb{targetId} \tab The ID of the target cohort populations.\cr
+#' \verb{outcomeId} \tab The ID of the outcomeId.\cr \verb{dataLocation} \tab The location where the plpData was saved 
 #'  \cr \verb{the settings ids} \tab The ids for all other settings used for model development.\cr }
 #'
 #' @export
@@ -44,104 +44,142 @@ runMultiplePlp <- function(
     createModelDesign(targetId = 1, outcomeId = 3, modelSettings = setLassoLogisticRegression())
   ),
   onlyFetchData = F,
-  splitSettings = createDefaultSplitSetting(
-    type = "stratified", 
-    testFraction = 0.25,
-    trainFraction = 0.75, 
-    splitSeed = 123, 
-    nfold = 3
-  ),
   cohortDefinitions = NULL,
   logSettings = createLogSettings(
     verbosity = "DEBUG", 
     timeStamp = T, 
     logName = "runPlp Log"
   ),
-  saveDirectory = getwd()
+  saveDirectory = getwd(),
+  sqliteLocation = file.path(saveDirectory, 'sqlite')
 ){
   
   #input checks
   checkIsClass(databaseDetails, c('databaseDetails'))
   checkIsClass(modelDesignList, c('list', 'modelDesign'))
   checkIsClass(onlyFetchData, 'logical')
-  checkIsClass(splitSettings, 'splitSettings')
   checkIsClass(logSettings, 'logSettings')
   checkIsClass(saveDirectory, 'character')
   if(!dir.exists(saveDirectory)){
     dir.create(saveDirectory, recursive = T)
   }
   
-  # get idList
-  idList <- getidList(modelDesignList = modelDesignList)
-  
-  # get settings data.frame
-  settingstable <- getSettingsTable(
-    modelDesignList = modelDesignList, 
-    idList = idList
-  )
+  settingstable <- convertToJson(modelDesignList,cohortDefinitions)
   
-  if(!is.null(cohortDefinitions)){
-    cohortNames <- data.frame(
-      targetName = getNames(cohortDefinitions, settingstable$targetId),
-      outcomeName = getNames(cohortDefinitions, settingstable$outcomeId)
-      )
-    settingstable <- cbind(cohortNames, settingstable)
+  if(nrow(settingstable) != length(modelDesignList)){
+    stop('Error in settingstable')
   }
   
-  utils::write.csv(settingstable, file.path(saveDirectory,'settings.csv'), row.names = F)
-  saveJsonFile(idList, file.path(saveDirectory,'settings.json'))
-  
-  # list(targetId, covariateSetting, outcomeIds, saveLocation)
-  dataSettings <- getDataSettings(settingstable)
+  # save the settings - TODO change this to save jsons in csv
+  utils::write.csv(
+    x = settingstable %>% dplyr::select(
+      "analysisId",
+      "targetId", 
+      "targetName",
+      "outcomeId", 
+      "outcomeName",
+      "dataLocation"
+      ), 
+    file.path(saveDirectory,'settings.csv'), 
+    row.names = F
+    )
+
+  # group the outcomeIds per combination of data extraction settings
+  dataSettings <- settingstable %>% 
+    dplyr::group_by(
+      .data$targetId,
+      .data$covariateSettings,
+      .data$restrictPlpDataSettings,
+      .data$dataLocation
+    ) %>% 
+    dplyr::summarise(
+      outcomeIds = paste(unique(.data$outcomeId), collapse = ',')
+      )
   
   # extract data
-  for(i in 1:length(dataSettings)){
-    dataExists <- length(dir(file.path(saveDirectory, dataSettings[[i]]$dataLocation)))>0
+  for(i in 1:nrow(as.data.frame(dataSettings))){
+    dataExists <- length(dir(file.path(saveDirectory, dataSettings$dataLocation[i])))>0
     if(!dataExists){
-      ParallelLogger::logInfo(paste('Extracting data for cohort', dataSettings[[i]]$targetId, 'to', file.path(saveDirectory, dataSettings[[i]]$dataLocation)))
+      ParallelLogger::logInfo(paste('Extracting data for cohort', dataSettings$targetId[i], 'to', file.path(saveDirectory, dataSettings$dataLocation[i])))
       
-      databaseDetails$cohortId <- dataSettings[[i]]$targetId
-      databaseDetails$outcomeIds <- dataSettings[[i]]$outcomeIds
+      databaseDetails$targetId <- dataSettings$targetId[i]
+      databaseDetails$outcomeIds <- strsplit(dataSettings$outcomeIds[i], ',')[[1]]
       
       plpDataSettings <- list(
         databaseDetails = databaseDetails,
-        covariateSettings = getSettingFromId(idList, type = 'covariateSettings', dataSettings[[i]]$covariateSettings),
-        restrictPlpDataSettings = getSettingFromId(idList, type = 'restrictPlpDataSettings', dataSettings[[i]]$restrictPlpDataSettings)
-        )
-      
+        covariateSettings = ParallelLogger::convertJsonToSettings(dataSettings$covariateSettings[i]),
+        restrictPlpDataSettings = ParallelLogger::convertJsonToSettings(dataSettings$restrictPlpDataSettings[i])
+      )
       
       plpData <- tryCatch(
         {do.call(getPlpData, plpDataSettings)},
         error = function(e){ParallelLogger::logInfo(e); return(NULL)}
       )
       if(!is.null(plpData)){
-        savePlpData(plpData, file.path(saveDirectory, dataSettings[[i]]$dataLocation))
+        savePlpData(plpData, file.path(saveDirectory, dataSettings$dataLocation[i]))
       }
     } else{
-      ParallelLogger::logInfo(paste('Data for cohort', dataSettings[[i]]$targetId, 'exists at', file.path(saveDirectory, dataSettings[[i]]$dataLocation)))
+      ParallelLogger::logInfo(paste('Data for target', dataSettings$targetId[i], 'exists at', file.path(saveDirectory, dataSettings$dataLocation[i])))
     }
   }
   
-  # runPlp
+  # runDiagnosis - NEW
   if(!onlyFetchData){
-    for(i in 1:nrow(settingstable)){
+    for(i in 1:nrow(as.data.frame(settingstable))){
       modelDesign <- modelDesignList[[i]]
-      settings <- settingstable[i,]
+      settings <- settingstable[i,] # just the data locations?
       
       dataExists <- length(dir(file.path(saveDirectory, settings$dataLocation)))>0
       
       if(dataExists){
-        plpData <- PatientLevelPrediction::loadPlpData(file.path(saveDirectory, settings$dataLocation))
+        analysisExists <- file.exists(file.path(saveDirectory, settings$analysisId,'diagnosePlp.rds'))
         
-        analysisExists <- file.exists(file.path(saveDirectory, settings$analysisId,'plpResult', 'runPlp.rds'))
         if(!analysisExists){
+          plpData <- PatientLevelPrediction::loadPlpData(file.path(saveDirectory, settings$dataLocation))
+          diagnosePlpSettings <- list(
+            plpData = plpData,
+            outcomeId = modelDesign$outcomeId,
+            analysisId = settings$analysisId,
+            populationSettings = modelDesign$populationSettings,
+            splitSettings = modelDesign$splitSettings,
+            sampleSettings = modelDesign$sampleSettings,
+            featureEngineeringSettings = modelDesign$featureEngineeringSettings,
+            preprocessSettings = modelDesign$preprocessSettings,
+            modelSettings = modelDesign$modelSettings,
+            logSettings = logSettings,
+            saveDirectory = saveDirectory
+          )
           
+          diagnose <- tryCatch(
+            {do.call(diagnosePlp, diagnosePlpSettings)},
+            error = function(e){ParallelLogger::logInfo(e); return(NULL)}
+          )
+        } else{
+          ParallelLogger::logInfo(paste('Diagnosis ', settings$analysisId, 'exists at', file.path(saveDirectory, settings$analysisId)))
+        }
+      }
+    }
+  }
+  
+  # runPlp
+  if(!onlyFetchData){
+    for(i in 1:nrow(as.data.frame(settingstable))){
+      modelDesign <- modelDesignList[[i]]
+      settings <- settingstable[i,] # just the data locations?
+      
+      dataExists <- length(dir(file.path(saveDirectory, settings$dataLocation)))>0
+      
+      if(dataExists){
+        analysisExists <- file.exists(file.path(saveDirectory, settings$analysisId,'plpResult', 'runPlp.rds'))
+        
+        if(!analysisExists){
+          plpData <- PatientLevelPrediction::loadPlpData(file.path(saveDirectory, settings$dataLocation))
           runPlpSettings <- list(
-            plpData = plpData,
+            plpData = quote(plpData),
             outcomeId = modelDesign$outcomeId,
             analysisId = settings$analysisId,
             populationSettings = modelDesign$populationSettings,
-            splitSettings = splitSettings,
+            splitSettings = modelDesign$splitSettings,
             sampleSettings = modelDesign$sampleSettings,
             featureEngineeringSettings = modelDesign$featureEngineeringSettings,
             preprocessSettings = modelDesign$preprocessSettings,
@@ -159,10 +197,24 @@ runMultiplePlp <- function(
           ParallelLogger::logInfo(paste('Analysis ', settings$analysisId, 'exists at', file.path(saveDirectory, settings$analysisId)))
         }
       }
-    }
-    
-    
+    } # end run per setting
+  }
+  
+  # [TODO] add code to create sqlite database and populate with results...
+  if(!onlyFetchData){
+    insertResultsToSqlite(
+      resultLocation = saveDirectory, 
+      cohortDefinitions = cohortDefinitions,
+      databaseList = createDatabaseList(
+        cdmDatabaseSchemas = databaseDetails$cohortDatabaseSchema,
+        cdmDatabaseNames = databaseDetails$cdmDatabaseName,
+        databaseRefIds = databaseDetails$cdmDatabaseId
+      ),
+      sqliteLocation = sqliteLocation
+    )
   }
+  
+  
   return(invisible(settingstable))
 }
 
@@ -181,6 +233,7 @@ runMultiplePlp <- function(
 #' @param sampleSettings                  Either NULL or an object of class \code{sampleSettings} with the over/under sampling settings used for model development
 #' @param preprocessSettings              Either NULL or an object of class \code{preprocessSettings} created using \code{createPreprocessingSettings()}
 #' @param modelSettings                   The model settings such as \code{setLassoLogisticRegression()}
+#' @param splitSettings                  The train/validation/test splitting used by all analyses created using \code{createDefaultSplitSetting()}
 #' @param runCovariateSummary             Whether to run the covariateSummary
 #' 
 #' @return
@@ -197,6 +250,13 @@ createModelDesign <- function(
   sampleSettings = NULL,
   preprocessSettings = NULL,
   modelSettings = NULL,
+  splitSettings = createDefaultSplitSetting(
+    type = "stratified", 
+    testFraction = 0.25,
+    trainFraction = 0.75, 
+    splitSeed = 123, 
+    nfold = 3
+  ),
   runCovariateSummary = T
 ){
   
@@ -206,21 +266,30 @@ createModelDesign <- function(
   checkIsClass(populationSettings, c('populationSettings'))
   checkIsClass(restrictPlpDataSettings, 'restrictPlpDataSettings')
   checkIsClass(covariateSettings, c('covariateSettings', 'list'))
+  checkIsClass(splitSettings, 'splitSettings')
   
   useFE <- F
   if(!is.null(featureEngineeringSettings)){
-    checkIsClass(featureEngineeringSettings, c('featureEngineeringSettings'))
+    if(inherits(featureEngineeringSettings, 'featureEngineeringSettings')){
+      featureEngineeringSettings <- list(featureEngineeringSettings)
+    }
+    lapply(featureEngineeringSettings, function(x) checkIsClass(x, c('featureEngineeringSettings')))
     useFE <- T
   } else{
-    featureEngineeringSettings <- createFeatureEngineeringSettings(type = "none")
+    featureEngineeringSettings <- list(createFeatureEngineeringSettings(type = "none"))
   }
   
   useSample <- F
   if(!is.null(sampleSettings)){
-    checkIsClass(sampleSettings, c('sampleSettings'))
+    
+    if(inherits(sampleSettings, 'sampleSettings')){
+      sampleSettings <- list(sampleSettings)
+    }
+    lapply(sampleSettings, function(x) checkIsClass(x, c('sampleSettings')))
+    
     useSample <- T
   } else{
-    sampleSettings <- createSampleSettings(type = "none")
+    sampleSettings <- list(createSampleSettings(type = "none"))
   }
   
   usePreprocess <- F
@@ -246,6 +315,7 @@ createModelDesign <- function(
     featureEngineeringSettings = featureEngineeringSettings,
     preprocessSettings = preprocessSettings,
     modelSettings = modelSettings,
+    splitSettings = splitSettings,
     executeSettings = createExecuteSettings(
       runSplitData = T,
       runSampleData = useSample,
@@ -269,6 +339,7 @@ createModelDesign <- function(
 #' This function creates a json file with the modelDesignList saved
 #' 
 #' @param modelDesignList          A list of modelDesigns created using \code{createModelDesign()}
+#' @param cohortDefinitions        A list of the cohortDefinitions (generally extracted from ATLAS)
 #' @param saveDirectory            The directory to save the modelDesignList settings
 #'                                       
 #' @examples
@@ -288,20 +359,17 @@ savePlpAnalysesJson <- function(
   createModelDesign(targetId = 1, outcomeId = 2, modelSettings = setLassoLogisticRegression()), 
   createModelDesign(targetId = 1, outcomeId = 3, modelSettings = setLassoLogisticRegression())
   ),
+  cohortDefinitions = NULL,
+  # add cohortDefinitions
   saveDirectory = NULL
   ){
   
-  if(class(modelDesignList) == 'modelDesign'){
+  if(inherits(modelDesignList, 'modelDesign')){
     modelDesignList <- list(modelDesignList)
   }
   
   lapply(modelDesignList, function(x){checkIsClass(x, 'modelDesign')})
 
-  
-  # save this as a json
-  modelDesignList <- lapply(modelDesignList, function(x) prepareToJson(x))
-  jsonSettings <- list(analyses = modelDesignList) # TODO: rename this ModelDesignList?
-  
   if(!is.null(saveDirectory)){
     checkIsClass(saveDirectory, 'character')
     
@@ -309,21 +377,27 @@ savePlpAnalysesJson <- function(
       dir.create(saveDirectory, recursive = T)
     }
     
-    modelDesignList <- jsonlite::toJSON(
-      x = jsonSettings, 
-      pretty = T, 
-      digits = 23, 
-      auto_unbox=TRUE, 
-      null = "null"
-      )
-    write(modelDesignList, file.path(saveDirectory,"predictionAnalysisList.json"))
-    
-    # should we add splitSettings to this and the input?
+    ParallelLogger::saveSettingsToJson(
+      object = list(
+        plpVersion = as.character(utils::packageVersion("PatientLevelPrediction")),
+        analyses = modelDesignList,
+        cohortDefinitions = cohortDefinitions
+      ), 
+      fileName = file.path(saveDirectory,"predictionAnalysisList.json")
+    )
     
     return(file.path(saveDirectory,"predictionAnalysisList.json")) 
   }
   
-  return(jsonSettings)
+  return(
+    ParallelLogger::convertSettingsToJson(
+      object = list(
+        plpVersion = as.character(utils::packageVersion("PatientLevelPrediction")),
+        analyses = modelDesignList,
+        cohortDefinitions = cohortDefinitions
+      )
+    )
+  )
 }
 
 
@@ -350,42 +424,9 @@ loadPlpAnalysesJson <- function(
     ParallelLogger::logError('Invalid directory - does not exist')
   }
   
-  if(!file.exists(file.path(jsonFileLocation))){
-    ParallelLogger::logError('predictionAnalysisList.json not found ')
-  }
-  
-  
-  json <- tryCatch(
-    {readChar(jsonFileLocation, file.info(jsonFileLocation)$size)},
-    error= function(cond) {
-      ParallelLogger::logInfo('Issue with loading json file...');
-      ParallelLogger::logError(cond)
-    })
-  json <- tryCatch(
-    {jsonlite::fromJSON(json, simplifyVector = T, simplifyDataFrame = F, simplifyMatrix = T)},
-    error = function(cond) {
-      ParallelLogger::logInfo('Issue with parsing json object...');
-      ParallelLogger::logError(cond)
-    })
-  json$analyses <- tryCatch(
-    {lapply(json$analyses, function(x) prepareToRlist(x))},
-    error = function(cond) {
-      ParallelLogger::logInfo('Issue converting json to R list...');
-      ParallelLogger::logError(cond)
-    })
-  
-  # if splitSettings in json
-  if('splitSettings' %in% names(json)){
-    # update the splitsetting (move this into load/saveplpAnalysis)
-    if('attributes' %in% names(json$splitSettings)){
-      atts <- json$splitSettings$attributes
-      json$splitSettings$attributes <- NULL
-      attributes(json$splitSettings) <- atts
-    }
-  }
-  
-  return(json)
+  rList <- ParallelLogger::loadSettingsFromJson(fileName = jsonFileLocation)
   
+  return(rList)
 }
 
 
@@ -401,9 +442,10 @@ loadPlpAnalysesJson <- function(
 #' are found and the connection and database settings for the new data
 #' 
 #' @param analysesLocation                The location where the multiple plp analyses are
-#' @param validationDatabaseDetails       The validation database settings created using \code{createDatabaseDetails()}
+#' @param validationDatabaseDetails       A single or list of validation database settings created using \code{createDatabaseDetails()}
 #' @param validationRestrictPlpDataSettings  The settings specifying the extra restriction settings when extracting the data created using \code{createRestrictPlpDataSettings()}.
 #' @param recalibrate                      A vector of recalibration methods (currently supports 'RecalibrationintheLarge' and/or 'weakRecalibration')
+#' @param cohortDefinitions           A list of cohortDefinitions
 #' @param saveDirectory               The location to save to validation results
 #' 
 #' @export 
@@ -412,13 +454,21 @@ validateMultiplePlp <- function(
   validationDatabaseDetails,
   validationRestrictPlpDataSettings = createRestrictPlpDataSettings(),
   recalibrate = NULL,
+  cohortDefinitions = NULL,
   saveDirectory = NULL
   ){
 
   # add input checks 
   checkIsClass(analysesLocation, 'character')
   
-  checkIsClass(validationDatabaseDetails, 'databaseDetails')
+  if(inherits(validationDatabaseDetails, 'databaseDetails')){
+    validationDatabaseDetails <- list(validationDatabaseDetails)
+  }
+  lapply(
+    validationDatabaseDetails, 
+    function(x){checkIsClass(x, 'databaseDetails')}
+    )
+  
   checkIsClass(validationRestrictPlpDataSettings, 'restrictPlpDataSettings')
   
   checkIsClass(recalibrate, c('character', 'NULL'))
@@ -443,7 +493,7 @@ validateMultiplePlp <- function(
     
     ParallelLogger::logInfo(paste0('Evaluating model in ',modelSettings ))
     
-    if(dir.exists(file.path(modelSettings[i],'plpResult'))){
+    if(dir.exists(file.path(modelSettings,'plpResult'))){
       ParallelLogger::logInfo(paste0('plpResult found in ',modelSettings ))
       
       plpModel <- loadPlpModel(file.path(modelSettings,'plpResult','model'))
@@ -455,7 +505,8 @@ validateMultiplePlp <- function(
             validationDatabaseDetails = validationDatabaseDetails,
             validationRestrictPlpDataSettings = validationRestrictPlpDataSettings,
             settings = createValidationSettings(
-              recalibrate = recalibrate
+              recalibrate = recalibrate,
+              runCovariateSummary = F
             ),
             outputFolder = saveLocation
           )},
@@ -466,167 +517,111 @@ validateMultiplePlp <- function(
     }
   }
   
-}
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-# HELPERS
-#===============================
-getidList <- function(modelDesignList){
-  
-  types <- c(
-    'targetId', 
-    'outcomeId', 
-    'restrictPlpDataSettings',
-    'covariateSettings', 
-    'populationSettings', 
-    'sampleSettings',
-    'featureEngineeringSettings',
-    'preprocessSettings',
-    'modelSettings',
-    'executeSettings'
-  )
-  
-  idList <- list()
-  length(idList) <- length(types)
-  names(idList) <- types
+  # add to sqlite database - needed for shiny app
+  #=======================
   
-  for(type in types){
-    idList[[type]] <- getSettingValues(modelDesignList, type = type )
-  }
-  
-  return(idList)
-}
-
-
-getSettingValues <- function(modelDesignList, type = 'cohortId' ){
-  
-  if(class(modelDesignList) == 'list'){
-    values <- unique(unlist(lapply(modelDesignList, function(x)jsonlite::serializeJSON(x[[type]])))
-      )
+  if(saveLocation == file.path(analysesLocation, 'Validation')){
+    ParallelLogger::logInfo('Saving validation results into the development sqlite database')
+    sqliteLocation <- file.path(analysesLocation, 'sqlite')
   } else{
-    values <- jsonlite::serializeJSON(modelDesignList[[type]])
+    ParallelLogger::logInfo('Saving validation results into validation sqlite')
+    sqliteLocation <- file.path(saveDirectory,'sqlite')
   }
   
-  if(! type %in% c('targetId', 'outcomeId')  ){
-    result <- data.frame(
-      value = values,
-      id = 1:length(values)
-    )
-  } else{
-    result <- data.frame(
-      value = sapply(values, function(x) jsonlite::unserializeJSON(x)),
-      id = sapply(values, function(x) jsonlite::unserializeJSON(x))
-    )
+  for(validationDatabaseDetail in validationDatabaseDetails){
+    tryCatch({
+      insertResultsToSqlite(
+        resultLocation = file.path(saveLocation, validationDatabaseDetail$cdmDatabaseName), 
+        cohortDefinitions = cohortDefinitions,
+        databaseList = createDatabaseList(
+          cdmDatabaseSchemas = validationDatabaseDetail$cdmDatabaseSchema,
+          cdmDatabaseNames = validationDatabaseDetail$cdmDatabaseName,
+          databaseRefIds = validationDatabaseDetail$cdmDatabaseId 
+        ),
+        sqliteLocation = sqliteLocation
+      )
+    })
   }
   
-  return(result)
 }
 
-# get the ids for the model design settings
-getIdsForSetting <- function(modelDesign, idList){
+convertToJson <-function(
+  modelDesignList,
+  cohortDefinitions = NULL
+){
   
-  ids <- c()
+  convertToJsonString <- function(x){as.character(ParallelLogger::convertSettingsToJson(x))}
   
-  for(settingType in names(idList)){
+  if(is.null(cohortDefinitions)){
     
-    if(!settingType %in% c('targetId', 'outcomeId')){
-    # get the index of the setting matching the design setting
-      ind <- which(idList[[settingType]]$value == jsonlite::serializeJSON(modelDesign[[settingType]]))
-    } else{
-      ind <- which(idList[[settingType]]$value == modelDesign[[settingType]])
-    }
-    # get the id
-    id <- idList[[settingType]]$id[ind]
+    cohortIds <- unlist(
+      lapply(
+        X = 1:length(modelDesignList), 
+        FUN = function(i){
+          c(
+            modelDesignList[[i]]$targetId,
+            modelDesignList[[i]]$outcomeId
+          )
+        }
+      )
+    )
+    cohortIds <- unique(cohortIds)
     
-    ids <- c(ids, id)
+    cohortDefinitions <- data.frame(
+      cohortId = cohortIds,
+      cohortName = paste0('Cohort: ', cohortIds)
+    )
+      
+  } else{
+    cohortDefinitions <- cohortDefinitions %>% 
+      dplyr::select(
+        "cohortId", 
+        "cohortName"
+        )
   }
   
-  names(ids) <- names(idList)
-  
-  return(ids)
-}
-
-
-# this creates a data.frame with the analysisId and settingsId for each analysis
-# need to add the data location to this
-getSettingsTable <- function(modelDesignList, idList){
-  
-  result <- lapply(modelDesignList, function(x) getIdsForSetting(x, idList) )
-  settingsTable <- do.call(rbind, result)
-  settingsTable <- as.data.frame(settingsTable)
-  
-  settingsTable$analysisId <- paste0('Analysis_', 1:nrow(settingsTable))
-  
-  settingsTable$dataLocation <- paste0('T_',settingsTable$targetId, '_L_', settingsTable$covariateSettings*settingsTable$restrictPlpDataSettings)
+  result <- data.frame(
+    analysisId = paste0('Analysis_', 1:length(modelDesignList)),
+    targetId = unlist(lapply(modelDesignList, function(x) ifelse(is.null(x$targetId), x$cohortId, x$targetId))),
+    outcomeId = unlist(lapply(modelDesignList, function(x) x$outcomeId)),
+    covariateSettings = unlist(lapply(modelDesignList, function(x) convertToJsonString(x$covariateSettings))),
+    restrictPlpDataSettings = unlist(lapply(modelDesignList, function(x)  convertToJsonString(x$restrictPlpDataSettings))),
+    populationSettings = unlist(lapply(modelDesignList, function(x)  convertToJsonString(x$populationSettings))),
+    sampleSettings = unlist(lapply(modelDesignList, function(x)  convertToJsonString(x$sampleSettings))),
+    splitSettings = unlist(lapply(modelDesignList, function(x)  convertToJsonString(x$splitSettings))),
+    featureEngineeringSettings = unlist(lapply(modelDesignList, function(x)  convertToJsonString(x$featureEngineeringSettings))),
+    preprocessSettings = unlist(lapply(modelDesignList, function(x)  convertToJsonString(x$preprocessSettings))),
+    modelSettings = unlist(lapply(modelDesignList, function(x)  convertToJsonString(x$modelSettings))),
+    executeSettings = unlist(lapply(modelDesignList, function(x)  convertToJsonString(x$executeSettings)))
+  )
   
-  return(settingsTable)
-}
-
-
-getSettingFromId <- function(
-  idList, 
-  type, 
-  id
-){
-  ind <- which(idList[[type]]$id == id)
-  if(!type %in% c('targetId', 'outcomeId')){
-    return(jsonlite::unserializeJSON(as.character(idList[[type]]$value[[ind]])))
-  } else{
-    return(idList[[type]]$value[[ind]])
-  }
-}
-
-
-getDataSettings <- function(settingstable){
-  
-  combos <- settingstable %>% 
-    dplyr::distinct(.data$targetId,.data$covariateSettings,.data$restrictPlpDataSettings,.data$dataLocation)
-  
-  result <- list()
-  length(result) <- nrow(combos)
-  for(i in 1:nrow(combos)){
-    result[[i]] <- list(
-      targetId = combos$targetId[i],
-      covariateSettings = combos$covariateSettings[i],
-      restrictPlpDataSettings = combos$restrictPlpDataSettings[i],
-      dataLocation = combos$dataLocation[i],
-      outcomeIds = settingstable %>%  
-        dplyr::filter(.data$dataLocation == combos$dataLocation[i]) %>%  
-        dplyr::select(.data$outcomeId) %>%  
-        dplyr::pull()
+    result <- result %>% 
+      dplyr::left_join(cohortDefinitions, by = c("outcomeId" = "cohortId")) %>%
+      dplyr::rename(outcomeName = "cohortName") %>%
+      dplyr::left_join(cohortDefinitions, by = c('targetId' = 'cohortId')) %>%
+      dplyr::rename(targetName = "cohortName") # new
+  
+  # get the names
+  uniqueSettings <-  result %>% 
+    dplyr::distinct(
+      .data$targetId, 
+      .data$covariateSettings, 
+      .data$restrictPlpDataSettings
+    ) %>%
+    dplyr::group_by(.data$targetId) %>% 
+    dplyr::mutate(dataLocation = paste0('targetId_',.data$targetId, '_L', dplyr::row_number()))
+  
+  # add the data names
+  result <- result %>% 
+    dplyr::left_join(
+      uniqueSettings, 
+      by = c(
+        "targetId" = "targetId",
+        "covariateSettings" = "covariateSettings",
+        "restrictPlpDataSettings" = "restrictPlpDataSettings"
+      )
     )
-  }
+  
   return(result)
 }
 
-getNames <- function(
-  cohortDefinitions, 
-  ids
-){
-  
-  idNames <- lapply(cohortDefinitions, function(x) c(x$id, x$name))
-  idNames <- do.call(rbind, idNames)
-  colnames(idNames) <- c('id', 'name')
-  idNames <- as.data.frame(idNames)
-  
-  nams <- c()
-  for(id in ids){
-    nams <- c(nams, idNames$name[idNames$id == id])
-  }
-  
-  return(nams)
-  
-}
diff --git a/R/RunPlp.R b/R/RunPlp.R
index 40fe09247..2d7119388 100644
--- a/R/RunPlp.R
+++ b/R/RunPlp.R
@@ -31,7 +31,8 @@
 #' develop and internally validate a model for the specified outcomeId.
 #'
 #' @param plpData                    An object of type \code{plpData} - the patient level prediction
-#'                                   data extracted from the CDM.
+#'                                   data extracted from the CDM.  Can also include an initial population as 
+#'                                   plpData$popualtion.
 #' @param outcomeId                  (integer) The ID of the outcome.                                       
 #' @param analysisId                 (integer) Identifier for the analysis. It is used to create, e.g., the result folder. Default is a timestamp.
 #' @param analysisName               (character) Name for the analysis
@@ -48,17 +49,12 @@
 #'                                   and whether to normalise the covariates before training  
 #' @param modelSettings              An object of class \code{modelSettings} created using one of the function:
 #'                                         \itemize{
-#'                                         \item{setLassoLogisticRegression()}{ A lasso logistic regression model}
-#'                                         \item{setGradientBoostingMachine()}{ A gradient boosting machine}
-#'                                         \item{setAdaBoost()}{ An ada boost model}
-#'                                         \item{setRandomForest()}{ A random forest model}
-#'                                         \item{setDecisionTree()}{ A decision tree model}
-#'                                         \item{setCovNN())}{ A convolutional neural network model}
-#'                                         \item{setCIReNN()}{ A recurrent neural network model}
-#'                                         \item{setMLP()}{ A neural network model}
-#'                                         \item{setDeepNN()}{ A deep neural network model}
-#'                                         \item{setKNN()}{ A KNN model}
-#'                                         
+#'                                         \item setLassoLogisticRegression() A lasso logistic regression model
+#'                                         \item setGradientBoostingMachine() A gradient boosting machine
+#'                                         \item setAdaBoost() An ada boost model
+#'                                         \item setRandomForest() A random forest model
+#'                                         \item setDecisionTree() A decision tree model
+#'                                         \item setKNN() A KNN model
 #'                                         } 
 #' @param logSettings                An object of \code{logSettings} created using \code{createLogSettings} 
 #'                                   specifying how the logging is done                                                                            
@@ -70,13 +66,12 @@
 #' An object containing the following:
 #'
 #'  \itemize{
-#'           \item{inputSettings}{A list containing all the settings used to develop the model}
-#'           \item{model}{ The developed model of class \code{plpModel}}
-#'           \item{executionSummary}{ A list containing the hardward details, R package details and execution time}
-#'           \item{performanceEvaluation}{ Various internal performance metrics in sparse format}
-#'           \item{prediction}{ The plpData cohort table with the predicted risks added as a column (named value)}
-#'           \item{covariateSummary)}{ A characterization of the features for patients with and without the outcome during the time at risk}
-#'           \item{analysisRef}{ A list with details about the analysis}
+#'           \item model The developed model of class \code{plpModel}
+#'           \item executionSummary A list containing the hardward details, R package details and execution time
+#'           \item performanceEvaluation Various internal performance metrics in sparse format
+#'           \item prediction The plpData cohort table with the predicted risks added as a column (named value)
+#'           \item covariateSummary A characterization of the features for patients with and without the outcome during the time at risk
+#'           \item analysisRef A list with details about the analysis
 #'           } 
 #'
 #'
@@ -258,7 +253,7 @@ runPlp <- function(
   tryCatch({
     printHeader(
       plpData, 
-      plpData$metaData$databaseDetails$cohortId, 
+      plpData$metaData$databaseDetails$targetId, 
       outcomeId, 
       analysisId, 
       analysisName,
@@ -267,20 +262,23 @@ runPlp <- function(
   })
   
   # create the population
-  population <- tryCatch(
-    {
-      do.call(
-        createStudyPopulation, 
-        list(
-          plpData = plpData,
-          outcomeId = outcomeId,
-          populationSettings = populationSettings
-        )
-      )
-    },
+  if(!is.null(plpData$population)) {
+    ParallelLogger::logInfo('Using existing population')
+    population <- plpData$population
+  } else {
+    ParallelLogger::logInfo('Creating population')
+    population <- tryCatch({
+      do.call(createStudyPopulation,
+              list(plpData = plpData,
+                   outcomeId = outcomeId,
+                   populationSettings = populationSettings,
+                   population = plpData$population
+                   )
+      )},
     error = function(e){ParallelLogger::logError(e); return(NULL)}
-  )
-  
+    )
+  }
+    
   if(is.null(population)){
     stop('population NULL')
   }
@@ -364,7 +362,8 @@ runPlp <- function(
     settings <- list(
       trainData = data$Train, 
       modelSettings = modelSettings,
-      analysisId = analysisId
+      analysisId = analysisId,
+      analysisPath = analysisPath
     )
     
     ParallelLogger::logInfo(sprintf('Training %s model',settings$modelSettings$name))  
@@ -438,22 +437,29 @@ runPlp <- function(
     
     variableImportance <- plpData$covariateData$covariateRef %>% 
       dplyr::mutate(covariateValue = 0) %>% 
-      dplyr::select(.data$covariateId, .data$covariateValue) %>% 
+      dplyr::select("covariateId", "covariateValue") %>% 
       dplyr::collect()
     if(!is.null(model)){
       if(!is.null(model$covariateImportance)){
-        variableImportance <- model$covariateImportance %>% dplyr::select(.data$covariateId, .data$covariateValue)
+        variableImportance <- model$covariateImportance %>% 
+          dplyr::select("covariateId", "covariateValue")
       }
     }
     
+    # apply FE if it is used
+    featureEngineering <- NULL
+    if(!is.null(model)){
+      featureEngineering <- model$preprocessing$featureEngineering
+    }
+    
     covariateSummaryResult <- do.call(covariateSummary,   
       list(
         covariateData = plpData$covariateData,
-        cohort = population %>% dplyr::select(.data$rowId),
-        labels = population %>% dplyr::select(.data$rowId, .data$outcomeCount), 
+        cohort = population %>% dplyr::select("rowId"),
+        labels = population %>% dplyr::select("rowId", "outcomeCount"), 
         strata = strata,
         variableImportance = variableImportance,
-        featureEngineering = NULL
+        featureEngineering = featureEngineering
         )
     )
   
diff --git a/R/RunPlpHelpers.R b/R/RunPlpHelpers.R
index 4073354c1..607e1872c 100644
--- a/R/RunPlpHelpers.R
+++ b/R/RunPlpHelpers.R
@@ -1,4 +1,4 @@
-printHeader <- function(plpData, cohortId, outcomeId , analysisId, analysisName, ExecutionDateTime){
+printHeader <- function(plpData, targetId, outcomeId , analysisId, analysisName, ExecutionDateTime){
   
   ParallelLogger::logInfo(paste0('Patient-Level Prediction Package version ', utils::packageVersion("PatientLevelPrediction")))
   
@@ -8,9 +8,13 @@ printHeader <- function(plpData, cohortId, outcomeId , analysisId, analysisName,
   ParallelLogger::logInfo(sprintf('%-20s%s', 'AnalysisName: ',analysisName))
   
   # add header to analysis log
-  ParallelLogger::logInfo(sprintf('%-20s%s', 'CohortID: ', cohortId))
+  ParallelLogger::logInfo(sprintf('%-20s%s', 'TargetID: ', targetId))
   ParallelLogger::logInfo(sprintf('%-20s%s', 'OutcomeID: ', outcomeId))
   ParallelLogger::logInfo(sprintf('%-20s%s', 'Cohort size: ', nrow(plpData$cohorts)))
+  if(!is.null(plpData$population)){
+    ParallelLogger::logInfo(sprintf('%-20s%s', 'Initial population size: ', nrow(plpData$population)))
+    ParallelLogger::logInfo(sprintf('%-20s%s', 'Initial cases: ', sum(plpData$population$outcomeCount>0)))
+  }
   ParallelLogger::logInfo(sprintf('%-20s%s', 'Covariates: ', nrow(plpData$covariateData$covariateRef)))
  ## ParallelLogger::logInfo(sprintf('%-20s%s', 'Population size: ', nrow(population)))
  ## ParallelLogger::logInfo(sprintf('%-20s%s', 'Cases: ', sum(population$outcomeCount>0)))
@@ -34,13 +38,13 @@ checkInputs <- function(inputs) {
     )
     
     # check class is correct
-    if(class(inputs[[inputName]]) != inputName && class(inputs[[inputName]]) != 'list'){
+    if(!inherits(x = inputs[[inputName]], what = c(inputName,'list'))){
       ParallelLogger::logError(paste0('Incorrect ', inputName))
       stop('Bad input')
     } 
     
-    if(class(inputs[[inputName]]) == 'list'){
-      if(unique(unlist(lapply(inputs[[inputName]], class))) != inputName){
+    if(inherits(x = inputs[[inputName]], what = 'list')){
+      if(sum(unlist(lapply(inputs[[inputName]], function(obj){inherits(x = obj, what = inputName)}))) != length(inputs[[inputName]])){
         ParallelLogger::logError(paste0('Incorrect ', inputName))
         stop('Bad input list')
       }
diff --git a/R/Sampling.R b/R/Sampling.R
index a77f77431..f97b024e0 100644
--- a/R/Sampling.R
+++ b/R/Sampling.R
@@ -22,9 +22,9 @@
 #' Returns an object of class \code{sampleSettings} that specifies the sampling function that will be called and the settings
 #'
 #' @param type              (character) Choice of:  \itemize{
-#'                                         \item{'none'}{ No sampling is applied - this is the default }
-#'                                         \item{'underSample')}{Undersample the non-outcome class to make the data more ballanced}
-#'                                         \item{'overSample'}{Oversample the outcome class by adding in each outcome multiple times}
+#'                                         \item 'none' No sampling is applied - this is the default
+#'                                         \item 'underSample' Undersample the non-outcome class to make the data more ballanced
+#'                                         \item 'overSample' Oversample the outcome class by adding in each outcome multiple times
 #'                                         } 
 #' @param numberOutcomestoNonOutcomes   (numeric) An numeric specifying the require number of non-outcomes per outcome 
 #' @param sampleSeed         (numeric) A seed to use when splitting the data for reproducibility (if not set a random number will be generated)
@@ -46,7 +46,7 @@ createSampleSettings <- function(type = 'none',
   
   sampleSettings <- list(
     numberOutcomestoNonOutcomes = numberOutcomestoNonOutcomes,
-    sampleSeed  = sampleSeed 
+    sampleSeed  = ifelse(type == 'none', 1, sampleSeed) # to make it the same for none
     )
   
   if(type == 'none'){
@@ -71,7 +71,7 @@ sampleData <- function(trainData, sampleSettings){
   ParallelLogger::logInfo('Starting data sampling')
   
   # if a single setting, make it a list
-  if(class(sampleSettings) == 'sampleSettings'){
+  if(inherits(sampleSettings,'sampleSettings')){
     sampleSettings <- list(sampleSettings)
   }
   
@@ -85,7 +85,7 @@ sampleData <- function(trainData, sampleSettings){
   
   ParallelLogger::logInfo('Finished data sampling')
   
-  metaData$sampleSettings <- sampleSetting
+  metaData$sampleSettings <- sampleSettings
   
   attr(trainData, "metaData") <- metaData
   return(trainData)
@@ -95,6 +95,19 @@ sampleData <- function(trainData, sampleSettings){
 
 sameData <- function(trainData, ...){
   ParallelLogger::logInfo('No sampling - returning same data')
+  
+  # add attribute for FE
+  featureEngeering <- list(
+    funct = 'sameData',
+    settings = list(
+      none = T
+    )
+  )
+  attr(trainData, 'metaData')$featureEngineering = listAppend(
+    attr(trainData, 'metaData')$featureEngineering,
+    featureEngeering
+  )
+  
   return(trainData)
 }
 
@@ -138,6 +151,7 @@ underSampleData <- function(trainData, sampleSettings){
   
   # filter to these patients 
   sampleTrainData <- list()
+  class(sampleTrainData) <- 'plpData'
   sampleTrainData$labels <- trainData$labels %>% dplyr::filter(.data$rowId %in% pplOfInterest)
   sampleTrainData$folds <- trainData$folds %>% dplyr::filter(.data$rowId %in% pplOfInterest)
   
@@ -179,6 +193,7 @@ overSampleData <- function(trainData, sampleSettings){
                                  sum(population$outcomeCount == 0), ' non-outcomes'))
   
   sampleTrainData <- list()
+  class(sampleTrainData) <- 'plpData'
   sampleTrainData$labels <- trainData$labels %>% dplyr::collect()
   sampleTrainData$folds <- trainData$folds %>% dplyr::collect()
   
@@ -238,5 +253,5 @@ overSampleData <- function(trainData, sampleSettings){
   
   class(sampleTrainData$covariateData) <- 'CovariateData'
   
-  return(trainData)
+  return(sampleTrainData)
 }
diff --git a/R/SaveLoadPlp.R b/R/SaveLoadPlp.R
index 85f76a30c..47353fcf3 100644
--- a/R/SaveLoadPlp.R
+++ b/R/SaveLoadPlp.R
@@ -35,12 +35,15 @@
 #'
 #' @export
 savePlpData <- function(plpData, file, envir=NULL, overwrite=F) {
-  if (missing(plpData))
+  if (missing(plpData)){
     stop("Must specify plpData")
-  if (missing(file))
+  }
+  if (missing(file)){
     stop("Must specify file")
-  if (!class(plpData) %in% c("plpData","plpData.libsvm"  ))
+  }
+  if (!inherits(x = plpData, what =  c("plpData"))){
     stop("Data not of class plpData")
+  }
   if(dir.exists(file.path(file, "covariates"))){
     stop('Folder to save covariates already exists...')
   }
@@ -108,12 +111,15 @@ loadPlpData <- function(file, readOnly = TRUE) {
 #'
 #' @export
 savePlpModel <- function(plpModel, dirPath){
-  if (missing(plpModel))
+  if (missing(plpModel)){
     stop("Must specify plpModel")
-  if (missing(dirPath))
+  }
+  if (missing(dirPath)){
     stop("Must specify directory path")
-  if (class(plpModel) != "plpModel")
+  }
+  if (!inherits(x = plpModel, what =  "plpModel")){
     stop("Not a plpModel")
+  }
   
   if(!dir.exists(dirPath)){
     ParallelLogger::logInfo('Creating directory to save model')
@@ -129,45 +135,82 @@ savePlpModel <- function(plpModel, dirPath){
   
   # save the trainDetails
   if(!is.null(plpModel$trainDetails)){
-  plpModel$trainDetails$trainingTime <- paste(as.character(plpModel$trainDetails$trainingTime), attr(plpModel$trainDetails$trainingTime,'units'))
-  saveJsonFile(
-    rObject = plpModel$trainDetails, 
-    file = file.path(dirPath, 'trainDetails.json')
+    ParallelLogger::saveSettingsToJson(
+    object = plpModel$trainDetails, 
+    fileName = file.path(dirPath, 'trainDetails.json')
   )
   }
   
   # save the validationDetails
   if(!is.null(plpModel$validationDetails)){
-    plpModel$validationDetails$validationDate <- paste(as.character(plpModel$validationDetails$validationDate), attr(plpModel$validationDetails$validationDate,'units'))
-    saveJsonFile(
-      rObject = plpModel$validationDetails, 
-      file = file.path(dirPath, 'validationDetails.json')
+    ParallelLogger::saveSettingsToJson(
+      object = plpModel$validationDetails, 
+      fileName = file.path(dirPath, 'validationDetails.json')
     )
   }
   
   
   # save the settings
-  saveJsonFile(
-    rObject = plpModel$settings, 
-    file = file.path(dirPath, 'settings.json')
+  ParallelLogger::saveSettingsToJson(
+    object = plpModel$modelDesign, 
+    fileName = file.path(dirPath, 'modelDesign.json')
+  )
+  
+  if(!is.null(plpModel$preprocessing)){
+    
+    # cheap fix to get past bug in ParallelLogger::saveSettingsToJson with tibbles
+    plpModel$preprocessing$tidyCovariates$normFactors <- 
+      as.data.frame(plpModel$preprocessing$tidyCovariates$normFactors)
+    
+    ParallelLogger::saveSettingsToJson(
+    object = plpModel$preprocessing, 
+    fileName = file.path(dirPath, 'preprocessing.json')
+  )
+  }
+  
+  
+  # save the model part function to file 
+  saveModelPart(
+    model = plpModel$model, 
+    savetype = attr(plpModel, 'saveType'),
+    dirPath = dirPath
+    )
+  
+  # save the attributes of plpModel
+  modelAttributes <- attributes(plpModel)
+  modelAttributes$names <- NULL
+  ParallelLogger::saveSettingsToJson(
+    object = modelAttributes, 
+    fileName = file.path(dirPath, 'attributes.json')
   )
   
+  return(dirPath)
+}
+
+
+saveModelPart <- function(model, savetype, dirPath){
   # save the model based on saveType
-  if(attr(plpModel, 'saveType') == "xgboost"){
-    xgboost::xgb.save(model = plpModel$model, fname = file.path(dirPath, "model.json"))
-  } else if(attr(plpModel, 'saveType') == "RtoJson"){
-    saveJsonFile(
-      rObject = plpModel$model, 
-      file = file.path(dirPath, 'model.json')
+  if(savetype == "xgboost"){
+    xgboost::xgb.save(
+      model = model, 
+      fname = file.path(dirPath, "model.json")
+    )
+  } else if(savetype == "lightgbm"){
+    lightgbm::lgb.save(booster = model,
+                       filename = file.path(dirPath, "model.json"))
+  } else if(savetype == "RtoJson"){
+    ParallelLogger::saveSettingsToJson(
+      object = model, 
+      fileName = file.path(dirPath, 'model.json')
     )
-  } else if(attr(plpModel, 'saveType') == "file"){
+  } else if(savetype == "file"){
     # move the model into model
     if(!dir.exists(file.path(dirPath, 'model'))){
       dir.create(file.path(dirPath, 'model'), recursive = T)
     }
-    for(file in dir(plpModel$model)){   
+    for(file in dir(model)){   
       file.copy(
-        file.path(plpModel$model,file), 
+        file.path(model,file), 
         file.path(dirPath,'model'), 
         overwrite = TRUE,  
         recursive = FALSE,
@@ -178,15 +221,6 @@ savePlpModel <- function(plpModel, dirPath){
     ParallelLogger::logWarn('Not sure how to save model - invalid saveType')
   }
   
-  # save the attributes of plpModel
-  modelAttributes <- attributes(plpModel)
-  modelAttributes$names <- NULL
-  saveJsonFile(
-    rObject = modelAttributes, 
-    file = file.path(dirPath, 'attributes.json')
-  )
-  
-  return(dirPath)
 }
 
 
@@ -206,7 +240,7 @@ loadPlpModel <- function(dirPath) {
   
   plpModel <- list()
   modelAttributes <- tryCatch(
-    loadJsonFile(file.path(dirPath, 'attributes.json')),
+    ParallelLogger::loadSettingsFromJson(file.path(dirPath, 'attributes.json')),
     error = function(e){NULL}
   )
   
@@ -224,27 +258,46 @@ loadPlpModel <- function(dirPath) {
   
   if(file.exists(file.path(dirPath, "trainDetails.json"))){
     plpModel$trainDetails <- tryCatch(
-      loadJsonFile(file.path(dirPath, "trainDetails.json")),
+      ParallelLogger::loadSettingsFromJson(file.path(dirPath, "trainDetails.json")),
       error = function(e){NULL}
     )
   }
   if(file.exists(file.path(dirPath, "validationDetails.json"))){
     plpModel$validationDetails <- tryCatch(
-      loadJsonFile(file.path(dirPath, "validationDetails.json")),
+      ParallelLogger::loadSettingsFromJson(file.path(dirPath, "validationDetails.json")),
       error = function(e){NULL}
     )
   }
   
-  plpModel$settings <- tryCatch(
-    loadJsonFile(file.path(dirPath, "settings.json")),
+  plpModel$modelDesign <- tryCatch(
+    ParallelLogger::loadSettingsFromJson(file.path(dirPath, "modelDesign.json")),
     error = function(e){NULL}
   )
   
+  # we don't use "preprocess" anymore, should be "preprocessing", 
+  # but leave this here if loading an older model
+  if(file.exists(file.path(dirPath, "preprocess.json"))){
+    plpModel$preprocessing <- tryCatch(
+      ParallelLogger::loadSettingsFromJson(file.path(dirPath, "preprocess.json")),
+      error = function(e){NULL}
+    )
+  }
+  if(file.exists(file.path(dirPath, "preprocessing.json")) & is.null(plpModel$preprocessing)){
+    plpModel$preprocessing <- tryCatch(
+      ParallelLogger::loadSettingsFromJson(file.path(dirPath, "preprocessing.json")),
+      error = function(e){NULL}
+    )
+  }
+  
+  
   if(attr(plpModel, 'saveType') == "xgboost"){
     ensure_installed("xgboost")
     plpModel$model <- xgboost::xgb.load(file.path(dirPath, "model.json"))
+  } else if(attr(plpModel, 'saveType') == "lightgbm"){
+    ensure_installed("lightgbm")
+    plpModel$model <- lightgbm::lgb.load(file.path(dirPath, "model.json"))
   } else if(attr(plpModel, 'saveType') %in% c("RtoJson")){
-    plpModel$model <- loadJsonFile(file.path(dirPath, "model.json"))
+    plpModel$model <- ParallelLogger::loadSettingsFromJson(file.path(dirPath, "model.json"))
   } else{
     plpModel$model <- file.path(dirPath, 'model')
   }
@@ -252,20 +305,6 @@ loadPlpModel <- function(dirPath) {
   return(plpModel)
 }
 
-saveJsonFile <- function(rObject, file){
-  
-  jsonObject  <- jsonlite::serializeJSON(rObject, digits = 23)
-  write(jsonObject, file)
-}
-
-loadJsonFile <- function(fileName) {
-  
-  jsonObject <- readChar(fileName, file.info(fileName)$size)
-  rObject <- jsonlite::unserializeJSON(jsonObject)
-  
-  return(rObject)
-}
-
 
 #' Saves the prediction dataframe to RDS
 #'
@@ -279,7 +318,10 @@ loadJsonFile <- function(fileName) {
 #' @export
 savePrediction <- function(prediction, dirPath, fileName='prediction.rds'){
   #TODO check inupts
-  saveJsonFile(prediction, file=file.path(dirPath,fileName))
+  ParallelLogger::saveSettingsToJson(
+    object = prediction, 
+    fileName = file.path(dirPath,fileName)
+    )
   
   return(file.path(dirPath,fileName))
 }
@@ -294,7 +336,7 @@ savePrediction <- function(prediction, dirPath, fileName='prediction.rds'){
 #' @export
 loadPrediction <- function(fileLocation){
   #TODO check inupts
-  prediction <- loadJsonFile(fileName = fileLocation)
+  prediction <- ParallelLogger::loadSettingsFromJson(fileName = fileLocation)
   return(prediction)
 }
 
@@ -308,14 +350,16 @@ loadPrediction <- function(fileLocation){
 #' 
 #' @export
 savePlpResult <- function(result, dirPath){
-  if (missing(result))
+  if (missing(result)){
     stop("Must specify runPlp output")
-  if (missing(dirPath))
+  }
+  if (missing(dirPath)){
     stop("Must specify directory location")
-  #if (class(plpModel) != "plpModel")
-  #  stop("Not a plpModel")
-  
-  if(!dir.exists(dirPath)) dir.create(dirPath, recursive = T)
+  }
+
+  if(!dir.exists(dirPath)){
+    dir.create(dirPath, recursive = T)
+  }
   
   savePlpModel(result$model, dirPath=file.path(dirPath,'model') )
   result$model <- NULL
@@ -332,16 +376,20 @@ savePlpResult <- function(result, dirPath){
 #' 
 #' @export
 loadPlpResult <- function(dirPath){
-  if (!file.exists(dirPath))
+  if (!file.exists(dirPath)){
     stop(paste("Cannot find folder", dirPath))
-  if (!file.info(dirPath)$isdir)
+  }
+  if (!file.info(dirPath)$isdir){
     stop(paste("Not a folder", dirPath))
+  }
   
   result <- readRDS(file.path(dirPath, "runPlp.rds"))
   result$model = loadPlpModel(file.path(dirPath, "model"))
-
-  class(result) <- "runPlp"
   
+  if (is.null(class(result))) {
+    class(result) <- 'runPlp'
+  }
+
   return(result)
   
 }
@@ -362,7 +410,12 @@ savePlpShareable <- function(result, saveDirectory, minCellCount = 10){
   if(!dir.exists(saveDirectory)) dir.create(saveDirectory, recursive = T)
   
   #executionSummary
-  saveJsonFile(result$executionSummary, file.path(saveDirectory, 'executionSummary.json'))
+  result$executionSummary$PackageVersion$packageVersion <- as.character(result$executionSummary$PackageVersion$packageVersion)
+  result$executionSummary$PlatformDetails$RAM <- as.character(result$executionSummary$PlatformDetails$RAM)
+  ParallelLogger::saveSettingsToJson(
+    object = result$executionSummary, 
+    fileName = file.path(saveDirectory, 'executionSummary.json')
+    )
   
   #save model as json files
   savePlpModel(result$model, file.path(saveDirectory, 'model'))
@@ -399,7 +452,10 @@ savePlpShareable <- function(result, saveDirectory, minCellCount = 10){
   }
   
   #analysisRef
-  saveJsonFile(result$analysisRef, file.path(saveDirectory, 'analysisRef.json'))
+  ParallelLogger::saveSettingsToJson(
+    object = result$analysisRef, 
+    fileName = file.path(saveDirectory, 'analysisRef.json')
+    )
   
   return(invisible(saveDirectory))
 }
@@ -445,7 +501,7 @@ loadPlpShareable <- function(loadDirectory){
   result$model <- loadPlpModel(file.path(loadDirectory,'model'))
   
   #executionSummary
-  result$executionSummary <- tryCatch({loadJsonFile(fileName = file.path(loadDirectory, 'executionSummary.json'))}, error = function(e){return(NULL)})
+  result$executionSummary <- tryCatch({ParallelLogger::loadSettingsFromJson(fileName = file.path(loadDirectory, 'executionSummary.json'))}, error = function(e){return(NULL)})
   
   #performanceEvaluation
   result$performanceEvaluation <- list()
@@ -459,7 +515,7 @@ loadPlpShareable <- function(loadDirectory){
   result$covariateSummary <- utils::read.csv(file = file.path(loadDirectory,'covariateSummary.csv'))
 
   #analysisRef
-  result$analysisRef <- tryCatch({loadJsonFile(fileName = file.path(loadDirectory, 'analysisRef.json'))}, error = function(e){return(NULL)})
+  result$analysisRef <- tryCatch({ParallelLogger::loadSettingsFromJson(fileName = file.path(loadDirectory, 'analysisRef.json'))}, error = function(e){return(NULL)})
   
   class(result) <- "runPlp"
   return(result)
@@ -497,3 +553,159 @@ removeCellCount <- function(
   return(data)
 }
 
+
+# add test for this - cant save json to csv - remove this...
+#' Exports all the results from a database into csv files
+#'
+#' @details
+#' Extracts the results from a database into a set of csv files
+#'
+#' @param conn  The connection to the database with the results
+#' @param connectionDetails                    The connectionDetails for the result database
+#' @param databaseSchemaSettings         The result database schema settings
+#' @param csvFolder      Location to save the csv files
+#' @param minCellCount   The min value to show in cells that are sensitive (values less than this value will be replaced with -1)
+#' @param sensitiveColumns A named list (name of table columns belong to) with a list of columns to apply the minCellCount to.
+#' @param fileAppend     If set to a string this will be appended to the start of the csv file names
+#' 
+#' @export
+extractDatabaseToCsv <- function(
+  conn = NULL,
+  connectionDetails,
+  databaseSchemaSettings = createDatabaseSchemaSettings(resultSchema = 'main'),
+  csvFolder,
+  minCellCount = 5,
+  sensitiveColumns = getPlpSensitiveColumns(),
+  fileAppend = NULL
+  ){
+  
+  ensure_installed('readr')
+  
+  # check inputs
+  if(!is.null(fileAppend)){
+    fileAppend <- paste0(gsub('_','',gsub(' ','', fileAppend)), '_')
+  }
+  
+  if(is.null(conn)){
+    # connect
+    conn <- DatabaseConnector::connect(connectionDetails)
+    on.exit(DatabaseConnector::disconnect(conn))
+  }
+  
+  # create the folder to save the csv files
+  if(!dir.exists(csvFolder)){
+    dir.create(csvFolder, recursive = T)
+  }
+  
+  # get the table names using the function in uploadToDatabase.R
+  tables <- getPlpResultTables()
+  
+  # extract result per table - give option to extract from different cohort/database tables?
+  modelLocations <- list()
+  for(table in tables){
+    sql <- "select * from @resultSchema.@appendtotable@tablename"
+    sql <- SqlRender::render(
+      sql, 
+      resultSchema = databaseSchemaSettings$resultSchema,
+      appendtotable = databaseSchemaSettings$stringAppendToResultSchemaTables,
+      tablename = table 
+    )
+    sql <- SqlRender::translate(
+      sql = sql, 
+      targetDialect = databaseSchemaSettings$targetDialect, 
+      tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema)
+    result <- DatabaseConnector::querySql(conn, sql)
+    
+    # get the model locations
+    if(table == 'MODELS'){
+      modelLocations <- result$PLP_MODEL_FILE
+    }
+    
+    # lower case for consistency in sharing csv results
+    colnames(result) <- tolower(colnames(result))
+    
+    # TODO: add min cell count filter here
+    if(tolower(table) %in% names(sensitiveColumns)){
+      result <- applyMinCellCount(
+        tableName = table,
+        sensitiveColumns = sensitiveColumns,
+        result = result,
+        minCellCount = minCellCount
+        )
+    }
+    
+    # save the results as a csv
+    readr::write_excel_csv(
+      x = result, 
+      file = file.path(csvFolder, paste0(fileAppend,tolower(table),'.csv'))
+      )
+  }
+  
+  
+  # load plpModels from database file and save into csv file
+  if(length(modelLocations)>0){
+    if(!dir.exists(file.path(csvFolder, 'models'))){
+      dir.create(file.path(csvFolder, 'models'), recursive = T)
+    }
+    for(modelLocation in modelLocations){
+      modelLocAppend <- strsplit(x = modelLocation, split = '/')[[1]][length(strsplit(x = modelLocation, split = '/')[[1]])]
+      plpModel <- tryCatch(
+        {
+          PatientLevelPrediction::loadPlpModel(file.path(modelLocation))
+        }, error = function(e){ParallelLogger::logInfo(e); return(NULL)}
+      )
+      if(!is.null(plpModel)){
+        PatientLevelPrediction::savePlpModel(plpModel, file.path(csvFolder, 'models', modelLocAppend))
+      }
+    }
+  }
+  
+  return(invisible(NULL))
+  
+}
+
+
+getPlpSensitiveColumns <- function(){
+  
+  result <- list(
+    
+    prediction_distribution = list(
+        c('person_count')
+      ),
+    covariate_summary = list(
+        c('covariate_count'),
+        c('with_no_outcome_covariate_count', 'with_outcome_covariate_count')
+    ),
+    calibration_summary = list(
+        c('person_count_at_risk', 'person_count_with_outcome')
+    ),
+    
+    demographic_summary = list(
+        c('person_count_at_risk'),
+        c('person_count_with_outcome')
+    )
+ 
+  )
+  
+  return(result)
+}
+
+
+applyMinCellCount <- function(
+  tableName,
+  sensitiveColumns,
+  result,
+  minCellCount
+){
+  
+  columnsToCensor <- sensitiveColumns[[tableName]]
+  
+  for(columns in columnsToCensor){
+    rowInd <- apply(result[,columns, drop = F] < minCellCount, 1, sum) > 0
+    if(sum(rowInd) > 0){
+      result[rowInd , columns] <- -1
+    }
+  }
+  
+  return(result)
+}
diff --git a/R/Simulation.R b/R/Simulation.R
index 8157238e2..b1d115c14 100644
--- a/R/Simulation.R
+++ b/R/Simulation.R
@@ -120,7 +120,7 @@ simulatePlpData <- function(plpDataSimulationProfile, n = 10000) {
   class(covariateData) <- "CovariateData"
   
   writeLines("Generating cohorts")
-  cohorts <- data.frame(rowId = 1:n, subjectId = 2e+10 + (1:n), cohortId = 1)
+  cohorts <- data.frame(rowId = 1:n, subjectId = 2e+10 + (1:n), targetId = 1)
   breaks <- cumsum(plpDataSimulationProfile$timePrevalence)
   r <- stats::runif(n)
   cohorts$time <- as.numeric(as.character(cut(r, breaks = c(0, breaks), labels = names(breaks))))
@@ -135,7 +135,10 @@ simulatePlpData <- function(plpDataSimulationProfile, n = 10000) {
   writeLines("Generating outcomes")
   allOutcomes <- data.frame()
   for (i in 1:length(plpDataSimulationProfile$metaData$outcomeIds)) {
-    prediction <- predictCyclopsType(plpDataSimulationProfile$outcomeModels[[i]],
+    coefficients <- data.frame(betas=as.numeric(plpDataSimulationProfile$outcomeModels[[i]]),
+                               covariateIds=names(plpDataSimulationProfile$outcomeModels[[i]])
+                               )
+    prediction <- predictCyclopsType(coefficients,
                                    cohorts,
                                    covariateData,
                                    modelType = "poisson")
@@ -152,28 +155,25 @@ simulatePlpData <- function(plpDataSimulationProfile, n = 10000) {
  covariateData$coefficients <- NULL
  
  # add indexes for covariate summary
- RSQLite::dbExecute(covariateData, "CREATE INDEX covsum_rowId ON covariates(rowId)")
- RSQLite::dbExecute(covariateData, "CREATE INDEX covsum_covariateId ON covariates(covariateId)")
- 
-  
+ Andromeda::createIndex(tbl = covariateData$covariates, columnNames = 'rowId', indexName = 'covsum_rowId')
+ Andromeda::createIndex(tbl = covariateData$covariates, columnNames = 'covariateId', indexName = 'covsum_covariateId')
+
   # Remove rownames else they will be copied to the ffdf objects:
   metaData = list()
   
   metaData$databaseDetails <- list(
-    cdmDatabaseSchema = 'Profile',
+    cdmDatabaseSchema = 'CDM_SCHEMA',
+    cdmDatabaseName = "CDM_NAME",
     outcomeDatabaseSchema = NULL,
     cohortDatabaseSchema = NULL,
     connectionDetails = NULL,
     outcomeTable = NULL,
     cohortTable = NULL,
     cdmVersion = 5,
-    cohortId = 1,
+    targetId = 1,
     outcomeIds = c(2,3)
   )
-  metaData$restrictPlpDataSettings <- list(
-    studyStartDate = NULL,
-    studyEndDate = NULL
-    )
+  metaData$restrictPlpDataSettings <- PatientLevelPrediction::createRestrictPlpDataSettings()
   metaData$covariateSettings <- FeatureExtraction::createCovariateSettings(useDemographicsAgeGroup = T)
   
   
@@ -181,13 +181,13 @@ simulatePlpData <- function(plpDataSimulationProfile, n = 10000) {
                                       targetCount=nrow(cohorts), uniquePeople=nrow(cohorts), 
                                       outcomes=nrow(outcomes))
   attr(cohorts, "metaData") <- list(
-    cohortId = 1, 
+    targetId = 1, 
     attrition = attrition
     )
   
   attr(allOutcomes, "metaData") <- data.frame(outcomeIds = c(2,3))
   
-  attr(covariateData, "metaData") <- list(populationSize = n)
+  attr(covariateData, "metaData") <- list(populationSize = n, cohortId = 1)
   
   result <- list(cohorts = cohorts,
                  outcomes = allOutcomes,
diff --git a/R/SklearnClassifier.R b/R/SklearnClassifier.R
index 6db987a5a..850e7b130 100644
--- a/R/SklearnClassifier.R
+++ b/R/SklearnClassifier.R
@@ -1,4 +1,4 @@
-# @file PythonClassifier.R
+# @file SklearnClassifier.R
 #
 # Copyright 2021 Observational Health Data Sciences and Informatics
 #
@@ -16,15 +16,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-fitSklearn <- function(
-  trainData,
-  param,
-  search = "grid",
-  analysisId,
-  ...) {
+fitSklearn <- function(trainData,
+                       modelSettings,
+                       search = "grid",
+                       analysisId,
+                       ...) {
+  param <- modelSettings$param
   
   # check covariate data
-  if(!FeatureExtraction::isCovariateData(trainData$covariateData)){stop("Needs correct covariateData")}
+  if (!FeatureExtraction::isCovariateData(trainData$covariateData)) {
+    stop("Needs correct covariateData")
+  }
   
   # get the settings from the param
   pySettings <- attr(param, 'settings')
@@ -34,8 +36,9 @@ fitSklearn <- function(
   
   start <- Sys.time()
   
-  if(!is.null(trainData$folds)){
-    trainData$labels <- merge(trainData$labels, trainData$fold, by = 'rowId')
+  if (!is.null(trainData$folds)) {
+    trainData$labels <-
+      merge(trainData$labels, trainData$fold, by = 'rowId')
   }
   
   # convert the data to a sparse R Matrix and then use reticulate to convert to python sparse
@@ -48,15 +51,15 @@ fitSklearn <- function(
   
   # save the model to outLoc
   outLoc <- createTempModelLoc()
-
+  
   # functions does CV and fits final models
   # returns: prediction (Train/CV),
   #          finalParam (optimal hyper-parameters)
-  #          variableImportance (final model) 
+  #          variableImportance (final model)
   #          paramGridSearch list with performance and params for complete grid search
   # at the moment it uses AUC as performance but this could be modified to let user
   # specify the performance metric to optimise
-  cvResult <- do.call( 
+  cvResult <- do.call(
     what = gridCvPython,
     args = list(
       matrixData = matrixData,
@@ -64,58 +67,66 @@ fitSklearn <- function(
       seed = pySettings$seed,
       requiresDenseMatrix = pySettings$requiresDenseMatrix,
       modelName = pySettings$name,
-      pythonImport = pySettings$pythonImport,
-      pythonImportSecond = pySettings$pythonImportSecond,
-      pythonClassifier = pySettings$pythonClassifier,
+      pythonModule = pySettings$pythonModule,
+      pythonClass = pySettings$pythonClass,
       modelLocation = outLoc,
       paramSearch = param,
-      saveToJson = pySettings$saveToJson
-      )
+      saveToJson = attr(param, 'saveToJson')
     )
+  )
+  
+  hyperSummary <-
+    do.call(rbind,
+            lapply(cvResult$paramGridSearch, function(x)
+              x$hyperSummary))
   
-  hyperSummary <- do.call(rbind, lapply(cvResult$paramGridSearch, function(x) x$hyperSummary))
-
   prediction <- cvResult$prediction
- 
+  
   variableImportance <- cvResult$variableImportance
   variableImportance[is.na(variableImportance)] <- 0
   
   incs <- rep(1, nrow(covariateRef))
   covariateRef$included <- incs
-  covariateRef$covariateValue <- unlist(variableImportance) # check this is correct order
+  covariateRef$covariateValue <-
+    unlist(variableImportance) # check this is correct order
   
   comp <- start - Sys.time()
   
   result <- list(
     model = file.path(outLoc),
     
+    preprocessing = list(
+      featureEngineering = attr(trainData, "metaData")$featureEngineering,
+      tidyCovariates = attr(trainData$covariateData, "metaData")$tidyCovariateDataSettings,
+      requireDenseMatrix = attr(param, 'settings')$requiresDenseMatrix
+    ),
+    
     prediction = prediction,
     
-    settings = list(
-      plpDataSettings = attr(trainData, "metaData")$plpDataSettings,
+    modelDesign = PatientLevelPrediction::createModelDesign(
+      targetId = attr(trainData, "metaData")$targetId,
+      outcomeId = attr(trainData, "metaData")$outcomeId,
+      restrictPlpDataSettings = attr(trainData, "metaData")$restrictPlpDataSettings,
       covariateSettings = attr(trainData, "metaData")$covariateSettings,
       populationSettings = attr(trainData, "metaData")$populationSettings,
-      featureEngineering = attr(trainData$covariateData, "metaData")$featureEngineering,
-      tidyCovariates = attr(trainData$covariateData, "metaData")$tidyCovariateDataSettings, 
-      requireDenseMatrix = attr(param, 'settings')$requiresDenseMatrix,
-      modelSettings = list(
-        model = pySettings$name, 
-        param = param,
-        finalModelParameters = cvResult$finalParam,
-        extraSettings = attr(param, 'settings')
-      ),
+      featureEngineeringSettings = attr(trainData$covariateData, "metaData")$featureEngineeringSettings,
+      preprocessSettings = attr(trainData$covariateData, "metaData")$preprocessSettings,
+      modelSettings = modelSettings,
       splitSettings = attr(trainData, "metaData")$splitSettings,
       sampleSettings = attr(trainData, "metaData")$sampleSettings
     ),
     
     trainDetails = list(
       analysisId = analysisId,
-      cdmDatabaseSchema = attr(trainData, "metaData")$cdmDatabaseSchema,
-      outcomeId = attr(trainData, "metaData")$outcomeId,
-      cohortId = attr(trainData, "metaData")$cohortId,
-      attrition = attr(trainData, "metaData")$attrition, 
-      trainingTime = comp,
+      analysisSource = '',
+      #TODO add from model
+      developmentDatabase = attr(trainData, "metaData")$cdmDatabaseName,
+      developmentDatabaseSchema = attr(trainData, "metaData")$cdmDatabaseSchema,
+      attrition = attr(trainData, "metaData")$attrition,
+      trainingTime = paste(as.character(abs(comp)), attr(comp, 'units')),
       trainingDate = Sys.Date(),
+      modelName = pySettings$name,
+      finalModelParameters = cvResult$finalParam,
       hyperParamSearch = hyperSummary
     ),
     
@@ -125,81 +136,80 @@ fitSklearn <- function(
   class(result) <- "plpModel"
   attr(result, "predictionFunction") <- "predictPythonSklearn"
   attr(result, "modelType") <- "binary"
-  attr(result, "saveType") <- attr(param, 'saveType')
+  attr(result, "saveType") <-
+    attr(param, 'saveType') # in save/load plp
+  attr(result, "saveToJson") <-
+    attr(param, 'saveToJson') # when saving in reticulate
   
   return(result)
 }
 
 
-predictPythonSklearn <- function(
-  plpModel, 
-  data, 
-  cohort
-  ){
-  
-  if(class(data) == 'plpData'){
+predictPythonSklearn <- function(plpModel,
+                                 data,
+                                 cohort) {
+  if (inherits(data, 'plpData')) {
     # convert
     matrixObjects <- toSparseM(
-      plpData = data, 
+      plpData = data,
       cohort = cohort,
-      map = plpModel$covariateImportance %>% 
-        dplyr::select(.data$columnId, .data$covariateId)
+      map = plpModel$covariateImportance %>%
+        dplyr::select("columnId", "covariateId")
     )
     
     newData <- matrixObjects$dataMatrix
     cohort <- matrixObjects$labels
     
-  }else{
+  } else{
     newData <- data
   }
-
-  # import
-  os <- reticulate::import('os')
   
   # load model
-  if(plpModel$settings$modelSettings$extraSettings$saveToJson){
-    skljson <- reticulate::import('sklearn_json')
-    modelLocation <- reticulate::r_to_py(paste0(plpModel$model,"\\model.json"))
-    model <- skljson$from_json(modelLocation)
+  if (attr(plpModel, 'saveToJson')) {
+    modelLocation <-
+      reticulate::r_to_py(file.path(plpModel$model, "model.json"))
+    model <- sklearnFromJson(path = modelLocation)
   } else{
-    joblib <- reticulate::import('joblib')
-    modelLocation <- reticulate::r_to_py(paste0(plpModel$model,"\\model.pkl"))
-    model <- joblib$load(os$path$join(modelLocation)) 
+    os <- reticulate::import('os')
+    joblib <- reticulate::import('joblib', convert = FALSE)
+    modelLocation <-
+      reticulate::r_to_py(file.path(plpModel$model, "model.pkl"))
+    model <- joblib$load(os$path$join(modelLocation))
   }
+  included <-
+    plpModel$covariateImportance$columnId[plpModel$covariateImportance$included >
+                                            0] # does this include map?
+  pythonData <- reticulate::r_to_py(newData[, included, drop = F])
   
-  included <- plpModel$covariateImportance$columnId[plpModel$covariateImportance$included>0] # does this include map?
-  pythonData <- reticulate::r_to_py(newData[,included, drop = F])
-
   # make dense if needed
-  if(plpModel$settings$requireDenseMatrix){
+  if (plpModel$preprocessing$requireDenseMatrix) {
     pythonData <- pythonData$toarray()
   }
   
   cohort <- predictValues(
-    model = model, 
-    data = pythonData, 
-    cohort = cohort, 
+    model = model,
+    data = pythonData,
+    cohort = cohort,
     type = attr(plpModel, 'modelType')
   )
-
+  
   return(cohort)
 }
 
-predictValues <- function(model, data, cohort, type = 'binary'){
-  
+predictValues <- function(model, data, cohort, type = 'binary') {
   predictionValue  <- model$predict_proba(data)
-  cohort$value <- predictionValue[,2]
+  cohort$value <- reticulate::py_to_r(predictionValue)[, 2]
   
-  cohort <- cohort %>% 
-    dplyr::select(-.data$rowId) %>%
-    dplyr::rename(rowId = .data$originalRowId)
+  cohort <- cohort %>%
+    dplyr::select(-"rowId") %>%
+    dplyr::rename(rowId = "originalRowId")
   
   attr(cohort, "metaData")$modelType <-  type
   
   return(cohort)
 }
 
-checkPySettings <- function(settings){
+checkPySettings <- function(settings) {
   checkIsClass(settings$seed, c('numeric', 'integer'))
   ParallelLogger::logDebug(paste0('classifier seed: ', settings$seed))
   
@@ -209,40 +219,26 @@ checkPySettings <- function(settings){
   checkIsClass(settings$name, c('character'))
   ParallelLogger::logDebug(paste0('name: ', settings$name))
   
-  checkIsClass(settings$saveToJson, c('logical'))
-  ParallelLogger::logDebug(paste0('saveToJson: ', settings$saveToJson))
-  
-  checkIsClass(settings$pythonImport, c('character'))
-  ParallelLogger::logDebug(paste0('pythonImport: ', settings$pythonImport))
-
-  if(!is.null(settings$pythonImportSecond)){
-    checkIsClass(settings$pythonImportSecond, c('character'))
-    ParallelLogger::logDebug(paste0('pythonImportSecond: ', settings$pythonImportSecond))
-  } else{
-    ParallelLogger::logDebug(paste0('pythonImportSecond: NULL'))
-  }
+  checkIsClass(settings$pythonModule, c('character'))
+  ParallelLogger::logDebug(paste0('pythonModule: ', settings$pythonModule))
   
-  checkIsClass(settings$pythonClassifier, c('character'))
-  ParallelLogger::logDebug(paste0('pythonClassifier: ', settings$pythonClassifier))
+  checkIsClass(settings$pythonClass, c('character'))
+  ParallelLogger::logDebug(paste0('pythonClass: ', settings$pythonClass))
 }
 
-gridCvPython <- function(
-  matrixData, 
-  labels, 
-  seed, 
-  requiresDenseMatrix, 
-  modelName,
-  pythonImport,
-  pythonImportSecond,
-  pythonClassifier,
-  modelLocation,
-  paramSearch,
-  saveToJson
-  )
-  {
+gridCvPython <- function(matrixData,
+                         labels,
+                         seed,
+                         requiresDenseMatrix,
+                         modelName,
+                         pythonModule,
+                         pythonClass,
+                         modelLocation,
+                         paramSearch,
+                         saveToJson)
+{
+  ParallelLogger::logInfo(paste0("Running CV for ", modelName, " model"))
   
-  ParallelLogger::logInfo(paste0("Rnning CV for ",modelName," model"))
- 
   np <- reticulate::import('numpy')
   os <- reticulate::import('os')
   sys <- reticulate::import('sys')
@@ -250,58 +246,70 @@ gridCvPython <- function(
   scipy <- reticulate::import('scipy')
   joblib <- reticulate::import('joblib')
   
-  firstImport <- reticulate::import(pythonImport)
-  
-  if(!is.null(pythonImportSecond)){
-    classifier <- firstImport[[pythonImportSecond]][[pythonClassifier]]
-  } else{
-    classifier <- firstImport[[pythonClassifier]]
-  }
+  module <- reticulate::import(pythonModule, convert = FALSE)
+  classifier <- module[pythonClass]
   
   ###########################################################################
   
   gridSearchPredictons <- list()
   length(gridSearchPredictons) <- length(paramSearch)
   
-  for(gridId in 1:length(paramSearch)){
-    
+  for (gridId in 1:length(paramSearch)) {
     # initiate prediction
     prediction <- c()
     
     fold <- labels$index
     ParallelLogger::logInfo(paste0('Max fold: ', max(fold)))
     
-    for( i in 1:max(fold)){
-      
-      ParallelLogger::logInfo(paste0('Fold ',i))
+    for (i in 1:max(fold)) {
+      ParallelLogger::logInfo(paste0('Fold ', i))
       trainY <- reticulate::r_to_py(labels$outcomeCount[fold != i])
-      trainX <- reticulate::r_to_py(matrixData[fold != i,])
-      testX <- reticulate::r_to_py(matrixData[fold == i,])
+      trainX <- reticulate::r_to_py(matrixData[fold != i, ])
+      testX <- reticulate::r_to_py(matrixData[fold == i, ])
       
-      if(requiresDenseMatrix){
+      if (requiresDenseMatrix) {
         ParallelLogger::logInfo('Converting sparse martix to dense matrix (CV)')
         trainX <- trainX$toarray()
         testX <- testX$toarray()
       }
       
-      model <- fitPythonModel(classifier, paramSearch[[gridId]], seed, trainX, trainY, np, pythonClassifier)
+      model <-
+        fitPythonModel(classifier,
+                       paramSearch[[gridId]],
+                       seed,
+                       trainX,
+                       trainY,
+                       np,
+                       pythonClass)
       
       ParallelLogger::logInfo("Calculating predictions on left out fold set...")
-      prediction <- rbind(prediction, predictValues(model = model, data = testX, cohort = labels[fold == i,], type = 'binary'))
+      prediction <-
+        rbind(
+          prediction,
+          predictValues(
+            model = model,
+            data = testX,
+            cohort = labels[fold == i, ],
+            type = 'binary'
+          )
+        )
       
     }
     
-    gridSearchPredictons[[gridId]] <- list(
-      prediction = prediction,
-      param = paramSearch[[gridId]]
-    )
+    gridSearchPredictons[[gridId]] <- list(prediction = prediction,
+                                           param = paramSearch[[gridId]])
   }
   
   # get best para (this could be modified to enable any metric instead of AUC, just need metric input in function)
   
-  paramGridSearch <- lapply(gridSearchPredictons, function(x){do.call(computeGridPerformance, x)})  # cvAUCmean, cvAUC, param
+  paramGridSearch <-
+    lapply(gridSearchPredictons, function(x) {
+      do.call(computeGridPerformance, x)
+    })  # cvAUCmean, cvAUC, param
   
-  optimalParamInd <- which.max(unlist(lapply(paramGridSearch, function(x) x$cvPerformance)))
+  optimalParamInd <-
+    which.max(unlist(lapply(paramGridSearch, function(x)
+      x$cvPerformance)))
   
   finalParam <- paramGridSearch[[optimalParamInd]]$param
   
@@ -313,37 +321,53 @@ gridCvPython <- function(
   trainY <- reticulate::r_to_py(labels$outcomeCount)
   trainX <- reticulate::r_to_py(matrixData)
   
-  if(requiresDenseMatrix){
+  if (requiresDenseMatrix) {
     ParallelLogger::logInfo('Converting sparse martix to dense matrix (final model)')
     trainX <- trainX$toarray()
   }
   
-  model <- fitPythonModel(classifier, finalParam , seed, trainX, trainY, np, pythonClassifier)
+  model <-
+    fitPythonModel(classifier,
+                   finalParam ,
+                   seed,
+                   trainX,
+                   trainY,
+                   np,
+                   pythonClass)
   
   ParallelLogger::logInfo("Calculating predictions on all train data...")
-  prediction <- predictValues(model = model, data = trainX, cohort = labels, type = 'binary')
+  prediction <-
+    predictValues(
+      model = model,
+      data = trainX,
+      cohort = labels,
+      type = 'binary'
+    )
   prediction$evaluationType <- 'Train'
   
-  prediction <- rbind(
-    prediction,
-    cvPrediction
-  )
+  prediction <- rbind(prediction,
+                      cvPrediction)
   
   # saving model
-  if(!dir.exists(file.path(modelLocation))){
+  if (!dir.exists(file.path(modelLocation))) {
     dir.create(file.path(modelLocation), recursive = T)
   }
- # joblib$dump(model, os$path$join(modelLocation,"model.pkl"), compress = T) 
-  if(saveToJson){
-    skljson <- reticulate::import('sklearn_json')
-    skljson$to_json(model = model, model_name =  file.path(modelLocation,"model.json"))
+  if (saveToJson) {
+    sklearnToJson(model = model,
+                  path = file.path(modelLocation, "model.json"))
   } else{
-    joblib$dump(model, file.path(modelLocation,"model.pkl"), compress = T) 
+    joblib$dump(model, file.path(modelLocation, "model.pkl"), compress = T)
   }
   
   # feature importance
-  variableImportance <- tryCatch({model$feature_importances_}, error = function(e){ParallelLogger::logInfo(e);return(rep(1,ncol(matrixData)))})
-
+  variableImportance <-
+    tryCatch({
+      reticulate::py_to_r(model$feature_importances_)
+    }, error = function(e) {
+      ParallelLogger::logInfo(e)
+      return(rep(1, ncol(matrixData)))
+    })
+  
   return(
     list(
       prediction = prediction,
@@ -356,79 +380,114 @@ gridCvPython <- function(
 }
 
 
-fitPythonModel <- function(classifier, param, seed, trainX, trainY, np, pythonClassifier){
-  ParallelLogger::logInfo(paste0('data X dim: ', trainX$shape[0], 'x', trainX$shape[1]))
-  ParallelLogger::logInfo(paste0('data Y length: ', np$shape(trainY)[[1]], ' with ',np$sum(trainY), ' outcomes'))
-  
-  timeStart <- Sys.time()
-  
-  # print parameters
-  # convert NULL to NA values
-  paramString <- param
-  for(ind in 1:length(paramString)){
-    if(is.null(paramString[[ind]])){
-      paramString[[ind]] <- 'null'
+fitPythonModel <-
+  function(classifier,
+           param,
+           seed,
+           trainX,
+           trainY,
+           np,
+           pythonClass) {
+    ParallelLogger::logInfo(paste0('data X dim: ', trainX$shape[0], 'x', trainX$shape[1]))
+    ParallelLogger::logInfo(paste0(
+      'data Y length: ',
+      np$shape(trainY)[[1]],
+      ' with ',
+      np$sum(trainY),
+      ' outcomes'
+    ))
+    
+    timeStart <- Sys.time()
+    
+    # print parameters
+    # convert NULL to NA values
+    paramString <- param
+    for (ind in 1:length(paramString)) {
+      if (is.null(paramString[[ind]])) {
+        paramString[[ind]] <- 'null'
+      }
     }
+    ParallelLogger::logInfo(paste(
+      names(param),
+      unlist(paramString),
+      sep = ':',
+      collapse = '    '
+    ))
+    
+    if (!is.null(param)) {
+      model <-
+        do.call(paste0(pythonClass, 'Inputs'),
+                list(classifier = classifier, param = param))
+    } else{
+      model <- classifier()
+    }
+    model <- model$fit(trainX, trainY)
+    timeEnd <- Sys.time()
+    
+    ParallelLogger::logInfo(paste0(
+      "Training model took (mins): ",
+      difftime(timeEnd, timeStart, units = 'mins')
+    ))
+    
+    return(model)
   }
-  ParallelLogger::logInfo(paste(names(param), unlist(paramString), sep = ':', collapse = '    '))
-  
-  if(!is.null(param)){
-    model <- do.call(paste0(pythonClassifier,'Inputs'), list(classifier = classifier, param = param))
-  } else{
-    model <- classifier()
-  }
-  model <- model$fit(trainX, trainY)
-  timeEnd <- Sys.time()
-  
-  ParallelLogger::logInfo(paste0("Training model took (mins): ",difftime(timeEnd, timeStart, units='mins') ))
-  
-  return(model)
-}
 
 
 
-
-computeGridPerformance <- function(prediction, param, performanceFunct = 'computeAuc'){
-  
-  performance <- do.call(
-    what = eval(parse(text = performanceFunct)),
-    args = list(prediction = prediction)
-  )
-  
-  performanceFold <- c()
-  for(i in 1:max(prediction$index)){
-    performanceFold <- c(
-      performanceFold, 
-      do.call(
-        what = eval(parse(text = performanceFunct)),
-        args = list(prediction = prediction[prediction$index == i,])
+#' Computes grid performance with a specified performance function
+#'
+#' @param prediction a dataframe with predictions and outcomeCount per rowId
+#' @param param a list of hyperparameters
+#' @param performanceFunct a string specifying which performance function to use
+#' . Default ``'compute_AUC'``
+#' @return A list with overview of the performance
+#' @export
+computeGridPerformance <-
+  function(prediction, param, performanceFunct = 'computeAuc') {
+    performance <- do.call(what = eval(parse(text = performanceFunct)),
+                           args = list(prediction = prediction))
+    
+    performanceFold <- c()
+    for (i in 1:max(prediction$index)) {
+      performanceFold <- c(performanceFold,
+                           do.call(
+                             what = eval(parse(text = performanceFunct)),
+                             args = list(prediction = prediction[prediction$index == i, ])
+                           ))
+    }
+    
+    paramString <- param
+    for (ind in 1:length(paramString)) {
+      if (is.null(paramString[[ind]])) {
+        paramString[[ind]] <- 'null'
+      }
+    }
+    
+    #hyperSummary <- c(performanceFunct, performance, performanceFold, unlist(paramString))
+    #names(hyperSummary) <- c(
+    #  'Metric',
+    #  'cvPerformance',
+    #  paste0('cvPerformanceFold',1:length(performanceFold)),
+    #  names(param)
+    #)
+    paramValues <- unlist(paramString)
+    names(paramValues) <- names(param)
+    
+    hyperSummary <- as.data.frame(c(
+      data.frame(
+        metric = performanceFunct,
+        fold = c("CV", as.character(1:length(performanceFold))),
+        value = c(performance, performanceFold)
+      ),
+      paramValues
+    ))
+    return(
+      list(
+        metric = performanceFunct,
+        cvPerformance = performance,
+        cvPerformancePerFold = performanceFold,
+        param = param,
+        hyperSummary = hyperSummary
       )
     )
   }
-  
-  paramString <- param
-  for(ind in 1:length(paramString)){
-    if(is.null(paramString[[ind]])){
-      paramString[[ind]] <- 'null'
-    }
-  }
-  
-  hyperSummary <- c(performanceFunct, performance, performanceFold, unlist(paramString))
-  names(hyperSummary) <- c(
-    'Metric', 
-    'cvPerformance', 
-    paste0('cvPerformanceFold',1:length(performanceFold)),
-    names(param)
-  )
-  
-  return(
-    list(
-      metric = performanceFunct,
-      cvPerformance = performance,
-      cvPerformancePerFold = performanceFold,
-      param = param,
-      hyperSummary = hyperSummary
-    )
-  )
-  
-}
diff --git a/R/SklearnClassifierHelpers.R b/R/SklearnClassifierHelpers.R
index 134bfb2e3..0535d833e 100644
--- a/R/SklearnClassifierHelpers.R
+++ b/R/SklearnClassifierHelpers.R
@@ -1,21 +1,32 @@
-listCartesian <- function(allList){
-  
-  sizes <- lapply(allList, function(x) 1:length(x))
-  combinations <- expand.grid(sizes)
-  
-  result <- list()
-  length(result) <- nrow(combinations)
-  
-  for(i in 1:nrow(combinations)){
-    tempList <- list()
-    for(j in 1:ncol(combinations)){
-      tempList <- c(tempList, list(allList[[j]][[combinations[i,j]]]))
-    }
-    names(tempList) <- names(allList)
-    result[[i]] <- tempList
-  }
-  
-  return(result)
-}
+# @file SklearnClassifierHelpers.R
+#
+# Copyright 2022 Observational Health Data Sciences and Informatics
+#
+# This file is part of PatientLevelPrediction
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
+#' Cartesian product
+#' 
+#' Computes the Cartesian product of all the combinations of elements in a list
+#' 
+#' @param allList a list of lists
+#' @return A list with all possible combinations from the input list of lists
+#' @export
+listCartesian <- function(allList) {
+  combinations <- expand.grid(allList)
+  results <- lapply(seq_len(nrow(combinations)),
+                    function(i) lapply(combinations, function(x) x[i][[1]]))
+  return(results)
+}
   
diff --git a/R/SklearnClassifierSettings.R b/R/SklearnClassifierSettings.R
index 6a0934da6..1b6446b70 100644
--- a/R/SklearnClassifierSettings.R
+++ b/R/SklearnClassifierSettings.R
@@ -1,4 +1,4 @@
-# @file PythonClassifier.R
+# @file SklearnClassifierSettings.R
 #
 # Copyright 2021 Observational Health Data Sciences and Informatics
 #
@@ -17,54 +17,51 @@
 # limitations under the License.
 
 #' Create setting for AdaBoost with python DecisionTreeClassifier base estimator
-# @param baseEstimator  (list) The base estimator from which the boosted ensemble is built. Support for sample weighting is required, as well as proper classes_ and n_classes_ attributes. If NULL, then the base estimator is DecisionTreeClassifier initialized with max_depth=1.
 #' @param nEstimators    (list) The maximum number of estimators at which boosting is terminated. In case of perfect fit, the learning procedure is stopped early.
 #' @param learningRate   (list) Weight applied to each classifier at each boosting iteration. A higher learning rate increases the contribution of each classifier. There is a trade-off between the learningRate and nEstimators parameters
 #'                       There is a trade-off between learningRate and nEstimators.
-#' @param algorithm      (list) If ‘SAMME.R’ then use the SAMME.R real boosting algorithm. base_estimator must support calculation of class probabilities. If ‘SAMME’ then use the SAMME discrete boosting algorithm. The SAMME.R algorithm typically converges faster than SAMME, achieving a lower test error with fewer boosting iterations.                      
+#' @param algorithm      (list) If ‘SAMME.R’ then use the SAMME.R real boosting algorithm. base_estimator must support calculation of class probabilities. If ‘SAMME’ then use the SAMME discrete boosting algorithm. The SAMME.R algorithm typically converges faster than SAMME, achieving a lower test error with fewer boosting iterations.
 #' @param seed           A seed for the model
 #'
 #' @examples
 #' \dontrun{
-#' model.adaBoost <- setAdaBoost(nEstimators = list(10,50,200), learningRate = list(1, 0.5, 0.1), 
+#' model.adaBoost <- setAdaBoost(nEstimators = list(10,50,200), learningRate = list(1, 0.5, 0.1),
 #'                               algorithm = list('SAMME.R'), seed = sample(1000000,1)
 #'                               )
 #' }
 #' @export
-setAdaBoost <- function(
-  #baseEstimator = list(NULL),
-  nEstimators = list(10,50, 200), 
-  learningRate = list(1, 0.5, 0.1), 
-  algorithm = list('SAMME.R'),
-  seed = sample(1000000,1)
-){
-  
+setAdaBoost <- function(nEstimators = list(10, 50, 200),
+                        learningRate = list(1, 0.5, 0.1),
+                        algorithm = list('SAMME.R'),
+                        seed = sample(1000000, 1)) {
   checkIsClass(seed[[1]], c("numeric", "integer"))
   checkIsClass(nEstimators, 'list')
   checkIsClass(learningRate, 'list')
   checkIsClass(algorithm, 'list')
   
-  #lapply(1:length(baseEstimator), function(i) checkIsClass(maxDepth[[i]] , c("NULL")))
-  
-  lapply(1:length(nEstimators), function(i) checkIsClass(nEstimators[[i]] , c("integer", "numeric")))
-  lapply(1:length(nEstimators), function(i) checkHigher(nEstimators[[i]] , 0))
+  lapply(1:length(nEstimators), function(i)
+    checkIsClass(nEstimators[[i]] , c("integer", "numeric")))
+  lapply(1:length(nEstimators), function(i)
+    checkHigher(nEstimators[[i]] , 0))
   
-  for(i in 1:length(nEstimators)){
-    if(class(nEstimators[[i]]) %in% c("numeric", "integer")){
+  for (i in 1:length(nEstimators)) {
+    if (inherits(x = nEstimators[[i]], what = c("numeric", "integer"))) {
       nEstimators[[i]] <- as.integer(nEstimators[[i]])
     }
   }
   
-  lapply(1:length(learningRate), function(i) checkIsClass(learningRate[[i]] , c("numeric")))
-  lapply(1:length(learningRate), function(i) checkHigher(learningRate[[i]] , 0))
+  lapply(1:length(learningRate), function(i)
+    checkIsClass(learningRate[[i]] , c("numeric")))
+  lapply(1:length(learningRate), function(i)
+    checkHigher(learningRate[[i]] , 0))
   
-  lapply(1:length(algorithm), function(i) checkIsClass(algorithm[[i]] , c("character")))
+  lapply(1:length(algorithm), function(i)
+    checkIsClass(algorithm[[i]] , c("character")))
   
   # test python is available and the required dependancies are there:
   ##checkPython()
   
   paramGrid <- list(
-    baseEstimator = list(NULL),
     nEstimators = nEstimators,
     learningRate = learningRate,
     algorithm = algorithm,
@@ -74,51 +71,48 @@ setAdaBoost <- function(
   param <- listCartesian(paramGrid)
   
   attr(param, 'settings') <- list(
+    modelType = 'adaBoost',
     seed = seed[[1]],
-    paramNames = names(paramGrid), #use this for logging params
+    paramNames = names(paramGrid),
+    #use this for logging params
     requiresDenseMatrix = F,
-    saveToJson = F,
     name = "AdaBoost",
-    pythonImport = 'sklearn',
-    pythonImportSecond = 'ensemble',
-    pythonClassifier = 'AdaBoostClassifier'
+    pythonModule = "sklearn.ensemble",
+    pythonClass = "AdaBoostClassifier"
   )
   
+  attr(param, 'saveToJson') <- T
   attr(param, 'saveType') <- 'file'
   
-  result <- list(
-    fitFunction = "fitSklearn",
-    param = param
-  )
+  result <- list(fitFunction = "fitSklearn",
+                 param = param)
   class(result) <- "modelSettings"
   
   return(result)
 }
 
 
-AdaBoostClassifierInputs <- function(classifier, param){
-  
+AdaBoostClassifierInputs <- function(classifier, param) {
   model <- classifier(
-    base_estimator = param[[which.max(names(param)=='baseEstimator')]],
-    n_estimators = param[[which.max(names(param)=='nEstimators')]],
-    learning_rate = param[[which.max(names(param)=='learningRate')]],
-    algorithm = param[[which.max(names(param)=='algorithm')]],
-    random_state = param[[which.max(names(param)=='seed')]]
+    n_estimators = param[[which.max(names(param) == 'nEstimators')]],
+    learning_rate = param[[which.max(names(param) == 'learningRate')]],
+    algorithm = param[[which.max(names(param) == 'algorithm')]],
+    random_state = param[[which.max(names(param) == 'seed')]]
   )
   
   return(model)
 }
 
-#' Create setting for the scikit-learn 1.0.1 DecisionTree with python 
+#' Create setting for the scikit-learn 1.0.1 DecisionTree with python
 #' @param criterion The function to measure the quality of a split. Supported criteria are “gini” for the Gini impurity and “entropy” for the information gain.
 #' @param splitter The strategy used to choose the split at each node. Supported strategies are “best” to choose the best split and “random” to choose the best random split.
 #' @param maxDepth    (list) The maximum depth of the tree. If NULL, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples.
 #' @param minSamplesSplit    The minimum number of samples required to split an internal node
 #' @param minSamplesLeaf     The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least minSamplesLeaf training samples in each of the left and right branches. This may have the effect of smoothing the model, especially in regression.
 #' @param minWeightFractionLeaf    The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sampleWeight is not provided.
-#' @param maxFeatures   (list) The number of features to consider when looking for the best split (int/'auto'/NULL)
+#' @param maxFeatures   (list) The number of features to consider when looking for the best split (int/'sqrt'/NULL)
 #' @param maxLeafNodes (list) Grow a tree with max_leaf_nodes in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes. (int/NULL)
-#' @param minImpurityDecrease  Threshold for early stopping in tree growth. A node will split if its impurity is above the threshold, otherwise it is a leaf. 
+#' @param minImpurityDecrease  Threshold for early stopping in tree growth. A node will split if its impurity is above the threshold, otherwise it is a leaf.
 #' @param classWeight         (list) Weights associated with classes 'balance' or NULL
 #' @param seed                The random state seed
 #'
@@ -127,21 +121,20 @@ AdaBoostClassifierInputs <- function(classifier, param){
 #' model.decisionTree <- setDecisionTree(maxDepth=10,minSamplesLeaf=10, seed=NULL )
 #' }
 #' @export
-setDecisionTree <- function(
-  criterion = list('gini'),
-  splitter = list('best'),
-  maxDepth = list(as.integer(4), as.integer(10), NULL),
-  minSamplesSplit = list(2, 10),
-  minSamplesLeaf = list(10, 50),
-  minWeightFractionLeaf = list(0),
-  maxFeatures = list(100,'auto', NULL),
-  maxLeafNodes = list(NULL),
-  minImpurityDecrease = list(10^-7),
-  classWeight = list(NULL, 'balanced'),
-  seed = sample(1000000,1)
-  ){
-  if(!class(seed[[1]])%in%c('numeric', 'integer'))
+setDecisionTree <- function(criterion = list('gini'),
+                            splitter = list('best'),
+                            maxDepth = list(as.integer(4), as.integer(10), NULL),
+                            minSamplesSplit = list(2, 10),
+                            minSamplesLeaf = list(10, 50),
+                            minWeightFractionLeaf = list(0),
+                            maxFeatures = list(100, 'sqrt', NULL),
+                            maxLeafNodes = list(NULL),
+                            minImpurityDecrease = list(10 ^ -7),
+                            classWeight = list(NULL),
+                            seed = sample(1000000, 1)) {
+  if (!inherits(x = seed[[1]], what = c('numeric', 'integer'))) {
     stop('Invalid seed')
+  }
   
   checkIsClass(criterion, 'list')
   checkIsClass(splitter, 'list')
@@ -154,78 +147,115 @@ setDecisionTree <- function(
   checkIsClass(minImpurityDecrease, 'list')
   checkIsClass(classWeight, 'list')
   
-  lapply(1:length(criterion), function(i) checkIsClass(criterion[[i]] , 'character'))
-  lapply(1:length(splitter), function(i) checkIsClass(splitter[[i]] , 'character'))
+  lapply(1:length(criterion), function(i)
+    checkIsClass(criterion[[i]] , 'character'))
+  lapply(1:length(splitter), function(i)
+    checkIsClass(splitter[[i]] , 'character'))
   
   
-  lapply(1:length(criterion), function(i) {if(!criterion[[i]] %in% c('gini', 'entropy')){stop('Incorrect criterion')}})
-
+  lapply(1:length(criterion),
+         function(i) {
+           if (!criterion[[i]] %in% c('gini', 'entropy')) {
+             stop('Incorrect criterion')
+           }
+         })
   
-  lapply(1:length(maxDepth), function(i) checkIsClass(maxDepth[[i]] , c("numeric","integer","NULL")))
-  lapply(1:length(maxDepth), function(i) checkHigher(ifelse(is.null(maxDepth[[i]]),1,maxDepth[[i]]) , 0))
-  for(i in 1:length(maxDepth)){
-    if(class(maxDepth[[i]]) %in% c("numeric", "integer")){
+  
+  lapply(1:length(maxDepth), function(i)
+    checkIsClass(maxDepth[[i]] , c("numeric", "integer", "NULL")))
+  lapply(1:length(maxDepth), function(i)
+    checkHigher(ifelse(is.null(maxDepth[[i]]), 1, maxDepth[[i]]) , 0))
+  for (i in 1:length(maxDepth)) {
+    if (inherits(x = maxDepth[[i]], what =  c("numeric", "integer"))) {
       maxDepth[[i]] <- as.integer(maxDepth[[i]])
     }
   }
   
-  lapply(1:length(minSamplesSplit), function(i) checkIsClass(minSamplesSplit[[i]] , c("numeric", "integer","NULL")))
-  lapply(1:length(minSamplesSplit), function(i) checkHigher(ifelse(is.null(minSamplesSplit[[i]]),1, minSamplesSplit[[i]]) , 0))
+  lapply(1:length(minSamplesSplit),
+         function(i)
+           checkIsClass(minSamplesSplit[[i]] , c("numeric", "integer", "NULL")))
+  lapply(1:length(minSamplesSplit),
+         function(i)
+           checkHigher(ifelse(
+             is.null(minSamplesSplit[[i]]), 1, minSamplesSplit[[i]]
+           ) , 0))
   
   # convert to integer if >= 1
-  for(i in 1:length(minSamplesSplit)){
-    if(minSamplesSplit[[i]] >= 1){
+  for (i in 1:length(minSamplesSplit)) {
+    if (minSamplesSplit[[i]] >= 1) {
       minSamplesSplit[[i]] <- as.integer(minSamplesSplit[[i]])
     }
   }
   
-
-  lapply(1:length(minSamplesLeaf), function(i) checkIsClass(minSamplesLeaf[[i]] , c("numeric", "integer")))
-  lapply(1:length(minSamplesLeaf), function(i) checkHigher(minSamplesLeaf[[i]] , 0))
+  
+  lapply(1:length(minSamplesLeaf),
+         function(i)
+           checkIsClass(minSamplesLeaf[[i]] , c("numeric", "integer")))
+  lapply(1:length(minSamplesLeaf),
+         function(i)
+           checkHigher(minSamplesLeaf[[i]] , 0))
   
   # convert to integer if >= 1
-  for(i in 1:length(minSamplesLeaf)){
-    if(minSamplesLeaf[[i]] >= 1){
+  for (i in 1:length(minSamplesLeaf)) {
+    if (minSamplesLeaf[[i]] >= 1) {
       minSamplesLeaf[[i]] <- as.integer(minSamplesLeaf[[i]])
     }
   }
   
-  
-  lapply(1:length(minWeightFractionLeaf), function(i) checkIsClass(minWeightFractionLeaf[[i]] , c("numeric")))
-  lapply(1:length(minWeightFractionLeaf), function(i) checkHigherEqual(minWeightFractionLeaf[[i]] , 0))
-  
-  lapply(1:length(maxFeatures), function(i) checkIsClass(maxFeatures[[i]] , c("numeric", "integer","character","NULL")))
-  
-  for(i in 1:length(maxFeatures)){
-    if(class(maxFeatures[[i]]) %in% c("numeric", "integer")){
+  lapply(1:length(minWeightFractionLeaf),
+         function(i)
+           checkIsClass(minWeightFractionLeaf[[i]] , c("numeric")))
+  lapply(1:length(minWeightFractionLeaf),
+         function(i)
+           checkHigherEqual(minWeightFractionLeaf[[i]] , 0))
+  
+  lapply(1:length(maxFeatures),
+         function(i)
+           checkIsClass(maxFeatures[[i]] , c(
+             "numeric", "integer", "character", "NULL"
+           )))
+  
+  for (i in 1:length(maxFeatures)) {
+    if (inherits(x = maxFeatures[[i]], what =  c("numeric", "integer"))) {
       maxFeatures[[i]] <- as.integer(maxFeatures[[i]])
     }
   }
   
-  lapply(1:length(maxLeafNodes), function(i) checkIsClass(maxLeafNodes[[i]], c("integer","NULL")))
-  lapply(1:length(maxLeafNodes), function(i) checkHigher(ifelse(is.null(maxLeafNodes[[i]]),1,maxLeafNodes[[i]]) , 0))
-  
-  for(i in 1:length(maxLeafNodes)){
-    if(class(maxLeafNodes[[i]]) %in% c("numeric", "integer")){
+  lapply(1:length(maxLeafNodes),
+         function(i)
+           checkIsClass(maxLeafNodes[[i]], c("integer", "NULL")))
+  lapply(1:length(maxLeafNodes),
+         function(i)
+           checkHigher(ifelse(
+             is.null(maxLeafNodes[[i]]), 1, maxLeafNodes[[i]]
+           ) , 0))
+  
+  for (i in 1:length(maxLeafNodes)) {
+    if (inherits(x = maxLeafNodes[[i]], what =  c("numeric", "integer"))) {
       maxLeafNodes[[i]] <- as.integer(maxLeafNodes[[i]])
     }
   }
   
-  lapply(1:length(minImpurityDecrease), function(i) checkIsClass(minImpurityDecrease[[i]] , c("numeric")))
-  lapply(1:length(minImpurityDecrease), function(i) checkHigherEqual(minImpurityDecrease[[i]], 0))
-
-  lapply(1:length(classWeight), function(i) checkIsClass(classWeight[[i]] , c('character', 'NULL')))
-
+  lapply(1:length(minImpurityDecrease),
+         function(i)
+           checkIsClass(minImpurityDecrease[[i]] , c("numeric")))
+  lapply(1:length(minImpurityDecrease),
+         function(i)
+           checkHigherEqual(minImpurityDecrease[[i]], 0))
+  
+  lapply(1:length(classWeight), function(i)
+    checkIsClass(classWeight[[i]] , c('character', 'NULL')))
+  
   # test python is available and the required dependancies are there:
   ##checkPython()
   
   # scikit-learn 1.0.1 inputs:
-  # criterion='gini', splitter='best', max_depth=None, min_samples_split=2, 
-  # min_samples_leaf=1, min_weight_fraction_leaf=0.0, max_features=None, random_state=None, 
+  # criterion='gini', splitter='best', max_depth=None, min_samples_split=2,
+  # min_samples_leaf=1, min_weight_fraction_leaf=0.0, max_features=None, random_state=None,
   # max_leaf_nodes=None, min_impurity_decrease=0.0, class_weight=None, ccp_alpha=0.0
   
   # must be correct order for python classifier as I can't find a way to do.call a named list
-  # using reticulate 
+  # using reticulate
   paramGrid <- list(
     criterion = criterion,
     splitter = splitter,
@@ -240,66 +270,64 @@ setDecisionTree <- function(
     classWeight = classWeight
   )
   param <- listCartesian(paramGrid)
-
+  
   attr(param, 'settings') <- list(
+    modelType = 'decisionTree',
     seed = seed[[1]],
-    paramNames = names(paramGrid), #use this for logging params
+    paramNames = names(paramGrid),
+    #use this for logging params
     requiresDenseMatrix = F,
-    saveToJson = T,
     name = "Decision Tree",
-    pythonImport = 'sklearn',
-    pythonImportSecond = 'tree',
-    pythonClassifier = 'DecisionTreeClassifier'
+    pythonModule = "sklearn.tree",
+    pythonClass = "DecisionTreeClassifier"
   )
   
+  attr(param, 'saveToJson') <- T
   attr(param, 'saveType') <- 'file'
   
-  result <- list(
-    fitFunction = "fitSklearn",
-    param = param
-  )
+  result <- list(fitFunction = "fitSklearn",
+                 param = param)
   class(result) <- "modelSettings"
   
   return(result)
 }
 
 
-DecisionTreeClassifierInputs <- function(classifier, param){
-  
-model <- classifier(
-  criterion = param[[which.max(names(param)=='criterion')]],
-  splitter = param[[which.max(names(param)=='splitter')]],
-  max_depth = param[[which.max(names(param)=='maxDepth')]],
-  min_samples_split = param[[which.max(names(param)=='minSamplesSplit')]],
-  min_samples_leaf = param[[which.max(names(param)=='minSamplesLeaf')]],
-  min_weight_fraction_leaf = param[[which.max(names(param)=='minWeightFractionLeaf')]],
-  max_features = param[[which.max(names(param)=='maxFeatures')]],
-  random_state = param[[which.max(names(param)=='seed')]],
-  max_leaf_nodes = param[[which.max(names(param)=='maxLeafNodes')]],
-  min_impurity_decrease = param[[which.max(names(param)=='minImpurityDecrease')]],
-  class_weight = param[[which.max(names(param)=='classWeight')]]
-)
-
-return(model)
+DecisionTreeClassifierInputs <- function(classifier, param) {
+  model <- classifier(
+    criterion = param[[which.max(names(param) == 'criterion')]],
+    splitter = param[[which.max(names(param) == 'splitter')]],
+    max_depth = param[[which.max(names(param) == 'maxDepth')]],
+    min_samples_split = param[[which.max(names(param) == 'minSamplesSplit')]],
+    min_samples_leaf = param[[which.max(names(param) == 'minSamplesLeaf')]],
+    min_weight_fraction_leaf = param[[which.max(names(param) == 'minWeightFractionLeaf')]],
+    max_features = param[[which.max(names(param) == 'maxFeatures')]],
+    random_state = param[[which.max(names(param) == 'seed')]],
+    max_leaf_nodes = param[[which.max(names(param) == 'maxLeafNodes')]],
+    min_impurity_decrease = param[[which.max(names(param) == 'minImpurityDecrease')]],
+    class_weight = param[[which.max(names(param) == 'classWeight')]]
+  )
+  
+  return(model)
 }
 
 
-#' Create setting for neural network model with python 
-#' 
+#' Create setting for neural network model with python
+#'
 #' @param hiddenLayerSizes      (list of vectors) The ith element represents the number of neurons in the ith hidden layer.
 #' @param activation (list) Activation function for the hidden layer.
 #' \itemize{
-#'    \item{"identity": no-op activation, useful to implement linear bottleneck, returns f(x) = x}
-#'    \item{"logistic": the logistic sigmoid function, returns f(x) = 1 / (1 + exp(-x)).}
-#'    \item{"tanh": the hyperbolic tan function, returns f(x) = tanh(x).}
-#'    \item{"relu": the rectified linear unit function, returns f(x) = max(0, x)}
+#'    \item "identity": no-op activation, useful to implement linear bottleneck, returns f(x) = x
+#'    \item "logistic": the logistic sigmoid function, returns f(x) = 1 / (1 + exp(-x)).
+#'    \item "tanh": the hyperbolic tan function, returns f(x) = tanh(x).
+#'    \item "relu": the rectified linear unit function, returns f(x) = max(0, x)
 #'  }
 #' @param solver     (list) The solver for weight optimization. (‘lbfgs’, ‘sgd’, ‘adam’)
 #' @param alpha      (list) L2 penalty (regularization term) parameter.
 #' @param batchSize  (list) Size of minibatches for stochastic optimizers. If the solver is ‘lbfgs’, the classifier will not use minibatch. When set to “auto”, batchSize=min(200, n_samples).
-#' @param learningRate (list) Only used when solver='sgd' Learning rate schedule for weight updates.{‘constant’, ‘invscaling’, ‘adaptive’}, default=’constant’
+#' @param learningRate (list) Only used when solver='sgd' Learning rate schedule for weight updates. ‘constant’, ‘invscaling’, ‘adaptive’, default=’constant’
 #' @param learningRateInit  (list) Only used when solver=’sgd’ or ‘adam’. The initial learning rate used. It controls the step-size in updating the weights.
-#' @param powerT     (list) Only used when solver=’sgd’.  The exponent for inverse scaling learning rate. It is used in updating effective learning rate when the learning_rate is set to ‘invscaling’. 
+#' @param powerT     (list) Only used when solver=’sgd’.  The exponent for inverse scaling learning rate. It is used in updating effective learning rate when the learning_rate is set to ‘invscaling’.
 #' @param maxIter  (list)  Maximum number of iterations. The solver iterates until convergence (determined by ‘tol’) or this number of iterations. For stochastic solvers (‘sgd’, ‘adam’), note that this determines the number of epochs (how many times each data point will be used), not the number of gradient steps.
 #' @param shuffle (list) boolean: Whether to shuffle samples in each iteration. Only used when solver=’sgd’ or ‘adam’.
 #' @param tol        (list) Tolerance for the optimization. When the loss or score is not improving by at least tol for nIterNoChange consecutive iterations, unless learning_rate is set to ‘adaptive’, convergence is considered to be reached and training stops.
@@ -312,40 +340,38 @@ return(model)
 #' @param beta2    (list) Exponential decay rate for estimates of second moment vector in adam, should be in 0 to 1.
 #' @param epsilon  (list) Value for numerical stability in adam.
 #' @param nIterNoChange     (list) Maximum number of epochs to not meet tol improvement. Only effective when solver=’sgd’ or ‘adam’.
-#' @param seed       A seed for the model 
+#' @param seed       A seed for the model
 #'
 #' @examples
 #' \dontrun{
 #' model.mlp <- setMLP()
 #' }
 #' @export
-setMLP <- function(
-  hiddenLayerSizes = list(c(100), c(20,4)), #must be integers
-  activation = list('relu'),
-  solver = list('adam'),
-  alpha = list(0.3,0.01,0.0001,0.000001), 
-  batchSize = list('auto'),
-  learningRate = list('constant'),
-  learningRateInit = list(0.001),
-  powerT = list(0.5),
-  maxIter = list(200, 100), 
-  shuffle = list(TRUE),
-  tol = list(0.0001),
-  warmStart = list(TRUE),
-  momentum = list(0.9),
-  nesterovsMomentum = list(TRUE),
-  earlyStopping = list(FALSE),
-  validationFraction = list(0.1),
-  beta1 = list(0.9), 
-  beta2 = list(0.999), 
-  epsilon = list(1,0.1,0.00000001), 
-  nIterNoChange = list(10),
-  seed = sample(100000,1)
-  ){
-  
-  checkIsClass(seed, c('numeric','integer'))
+setMLP <- function(hiddenLayerSizes = list(c(100), c(20)),
+                   #must be integers
+                   activation = list('relu'),
+                   solver = list('adam'),
+                   alpha = list(0.3, 0.01, 0.0001, 0.000001),
+                   batchSize = list('auto'),
+                   learningRate = list('constant'),
+                   learningRateInit = list(0.001),
+                   powerT = list(0.5),
+                   maxIter = list(200, 100),
+                   shuffle = list(TRUE),
+                   tol = list(0.0001),
+                   warmStart = list(TRUE),
+                   momentum = list(0.9),
+                   nesterovsMomentum = list(TRUE),
+                   earlyStopping = list(FALSE),
+                   validationFraction = list(0.1),
+                   beta1 = list(0.9),
+                   beta2 = list(0.999),
+                   epsilon = list(0.00000001),
+                   nIterNoChange = list(10),
+                   seed = sample(100000, 1)) {
+  checkIsClass(seed, c('numeric', 'integer'))
   checkIsClass(hiddenLayerSizes, c('list'))
-  checkIsClass(activation, c('list')) 
+  checkIsClass(activation, c('list'))
   checkIsClass(solver, c('list'))
   checkIsClass(alpha, c('list'))
   checkIsClass(batchSize, c('list'))
@@ -353,7 +379,7 @@ setMLP <- function(
   checkIsClass(learningRateInit, c('list'))
   checkIsClass(powerT, c('list'))
   checkIsClass(maxIter, c('list'))
-  checkIsClass(shuffle, c('list')) 
+  checkIsClass(shuffle, c('list'))
   checkIsClass(tol, c('list'))
   checkIsClass(warmStart, c('list'))
   checkIsClass(momentum, c('list'))
@@ -366,40 +392,40 @@ setMLP <- function(
   checkIsClass(nIterNoChange, c('list'))
   
   
-  for(i in 1:length(hiddenLayerSizes)){
-      hiddenLayerSizes[[i]] <- as.integer(hiddenLayerSizes[[i]])
+  for (i in 1:length(hiddenLayerSizes)) {
+    hiddenLayerSizes[[i]] <- as.integer(hiddenLayerSizes[[i]])
   }
   
   
-  for(i in 1:length(batchSize)){
-    if(class(batchSize[[i]]) %in% c("numeric", "integer")){
+  for (i in 1:length(batchSize)) {
+    if (inherits(x = batchSize[[i]], what =  c("numeric", "integer"))) {
       batchSize[[i]] <- as.integer(batchSize[[i]])
     }
   }
   
-  for(i in 1:length(maxIter)){
-    if(class(maxIter[[i]]) %in% c("numeric", "integer")){
+  for (i in 1:length(maxIter)) {
+    if (inherits(x = maxIter[[i]], what =  c("numeric", "integer"))) {
       maxIter[[i]] <- as.integer(maxIter[[i]])
     }
   }
   
-  for(i in 1:length(nIterNoChange)){
-    if(class(nIterNoChange[[i]]) %in% c("numeric", "integer")){
+  for (i in 1:length(nIterNoChange)) {
+    if (inherits(x = nIterNoChange[[i]], what =  c("numeric", "integer"))) {
       nIterNoChange[[i]] <- as.integer(nIterNoChange[[i]])
     }
   }
   
   # add lapply for values...
   paramGrid <- list(
-    hiddenLayerSizes = hiddenLayerSizes, 
+    hiddenLayerSizes = hiddenLayerSizes,
     activation = activation,
     solver = solver,
-    alpha = alpha, 
+    alpha = alpha,
     batchSize = batchSize,
     learningRate = learningRate,
     learningRateInit = learningRateInit,
     powerT = powerT,
-    maxIter = maxIter, 
+    maxIter = maxIter,
     shuffle = shuffle,
     seed = list(as.integer(seed[[1]])),
     tol = tol,
@@ -409,61 +435,59 @@ setMLP <- function(
     nesterovsMomentum = nesterovsMomentum,
     earlyStopping = earlyStopping,
     validationFraction = validationFraction,
-    beta1 = beta1, 
-    beta2 = beta2 , 
-    epsilon = epsilon, 
+    beta1 = beta1,
+    beta2 = beta2 ,
+    epsilon = epsilon,
     nIterNoChange = nIterNoChange
   )
-    
+  
   param <- listCartesian(paramGrid)
   
   attr(param, 'settings') <- list(
+    modelType = 'mlp',
     seed = seed[[1]],
-    paramNames = names(paramGrid), #use this for logging params
+    paramNames = names(paramGrid),
+    #use this for logging params
     requiresDenseMatrix = F,
-    saveToJson = F, # current bug in sklearn-json
     name = "Neural Network",
-    pythonImport = 'sklearn',
-    pythonImportSecond = 'neural_network',
-    pythonClassifier = 'MLPClassifier'
+    pythonModule = "sklearn.neural_network",
+    pythonClass = "MLPClassifier"
   )
   
+  attr(param, 'saveToJson') <- T
   attr(param, 'saveType') <- 'file'
   
-  result <- list(
-    fitFunction = "fitSklearn",
-    param = param
-  )
+  result <- list(fitFunction = "fitSklearn",
+                 param = param)
   class(result) <- "modelSettings"
   
   return(result)
 }
 
-MLPClassifierInputs <- function(classifier, param){
-
+MLPClassifierInputs <- function(classifier, param) {
   model <- classifier(
-    hidden_layer_sizes = param[[which.max(names(param)=='hiddenLayerSizes')]],
-    activation = param[[which.max(names(param)=='activation')]],
-    solver = param[[which.max(names(param)=='solver')]],
-    alpha = param[[which.max(names(param)=='alpha')]],
-    batch_size = param[[which.max(names(param)=='batchSize')]],
-    learning_rate = param[[which.max(names(param)=='learningRate')]],
-    learning_rate_init = param[[which.max(names(param)=='learningRateInit')]],
-    power_t = param[[which.max(names(param)=='powerT')]],
-    max_iter = param[[which.max(names(param)=='maxIter')]],
-    shuffle = param[[which.max(names(param)=='shuffle')]],
-    random_state = param[[which.max(names(param)=='seed')]],
-    tol = param[[which.max(names(param)=='tol')]],
+    hidden_layer_sizes = param[[which.max(names(param) == 'hiddenLayerSizes')]],
+    activation = param[[which.max(names(param) == 'activation')]],
+    solver = param[[which.max(names(param) == 'solver')]],
+    alpha = param[[which.max(names(param) == 'alpha')]],
+    batch_size = param[[which.max(names(param) == 'batchSize')]],
+    learning_rate = param[[which.max(names(param) == 'learningRate')]],
+    learning_rate_init = param[[which.max(names(param) == 'learningRateInit')]],
+    power_t = param[[which.max(names(param) == 'powerT')]],
+    max_iter = param[[which.max(names(param) == 'maxIter')]],
+    shuffle = param[[which.max(names(param) == 'shuffle')]],
+    random_state = param[[which.max(names(param) == 'seed')]],
+    tol = param[[which.max(names(param) == 'tol')]],
     verbose = F,
-    warm_start = param[[which.max(names(param)=='warmStart')]],
-    momentum = param[[which.max(names(param)=='momentum')]],
-    nesterovs_momentum = param[[which.max(names(param)=='nesterovsMomentum')]],
-    early_stopping = param[[which.max(names(param)=='earlyStopping')]],
-    validation_fraction = param[[which.max(names(param)=='validationFraction')]],
-    beta_1 = param[[which.max(names(param)=='beta1')]],
-    beta_2 = param[[which.max(names(param)=='beta2')]],
-    epsilon = param[[which.max(names(param)=='epsilon')]],
-    n_iter_no_change = param[[which.max(names(param)=='nIterNoChange')]]
+    warm_start = param[[which.max(names(param) == 'warmStart')]],
+    momentum = param[[which.max(names(param) == 'momentum')]],
+    nesterovs_momentum = param[[which.max(names(param) == 'nesterovsMomentum')]],
+    early_stopping = param[[which.max(names(param) == 'earlyStopping')]],
+    validation_fraction = param[[which.max(names(param) == 'validationFraction')]],
+    beta_1 = param[[which.max(names(param) == 'beta1')]],
+    beta_2 = param[[which.max(names(param) == 'beta2')]],
+    epsilon = param[[which.max(names(param) == 'epsilon')]],
+    n_iter_no_change = param[[which.max(names(param) == 'nIterNoChange')]]
   )
   
   return(model)
@@ -471,44 +495,41 @@ MLPClassifierInputs <- function(classifier, param){
 
 
 
-#' Create setting for naive bayes model with python 
+#' Create setting for naive bayes model with python
 #'
 #' @examples
 #' \dontrun{
 #' model.nb <- setNaiveBayes()
 #' }
 #' @export
-setNaiveBayes <- function(){
-  
+setNaiveBayes <- function() {
   # test python is available and the required dependancies are there:
   ##checkPython()
   
   param <- list(none = 'true')
   
   attr(param, 'settings') <- list(
+    modelType = 'naiveBayes',
     seed = as.integer(0),
-    paramNames = c(), #use this for logging params
+    paramNames = c(),
+    #use this for logging params
     requiresDenseMatrix = T,
-    saveToJson = T,
     name = "Naive Bayes",
-    pythonImport = 'sklearn',
-    pythonImportSecond = 'naive_bayes',
-    pythonClassifier = 'GaussianNB'
+    pythonModule = "sklearn.naive_bayes",
+    pythonClass = "GaussianNB"
   )
   
+  attr(param, 'saveToJson') <- T
   attr(param, 'saveType') <- 'file'
   
-  result <- list(
-    fitFunction = "fitSklearn",
-    param = param
-  )
+  result <- list(fitFunction = "fitSklearn",
+                 param = param)
   class(result) <- "modelSettings"
   
   return(result)
 }
 
-GaussianNBInputs <- function(classifier, param){
-  
+GaussianNBInputs <- function(classifier, param) {
   model <- classifier()
   
   return(model)
@@ -517,57 +538,53 @@ GaussianNBInputs <- function(classifier, param){
 
 #' Create setting for random forest model with python (very fast)
 #'
-#' @param ntrees    (list) The number of trees to build 
+#' @param ntrees    (list) The number of trees to build
 #' @param criterion (list) The function to measure the quality of a split. Supported criteria are “gini” for the Gini impurity and “entropy” for the information gain. Note: this parameter is tree-specific.
 #' @param maxDepth   (list) The maximum depth of the tree. If NULL, then nodes are expanded until all leaves are pure or until all leaves contain less than minSamplesSplit samples.
 #' @param minSamplesSplit (list) The minimum number of samples required to split an internal node
 #' @param minSamplesLeaf (list) The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least minSamplesLeaf training samples in each of the left and right branches. This may have the effect of smoothing the model, especially in regression.
 #' @param minWeightFractionLeaf (list) The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sampleWeight is not provided.
-#' @param mtries (list) The number of features to consider when looking for the best split: 
+#' @param mtries (list) The number of features to consider when looking for the best split:
 #' \itemize{
-#' \item{int}{then consider max_features features at each split.}
-#' \item{float}{then max_features is a fraction and round(max_features * n_features) features are considered at each split}
-#' \item{'auto'}{then max_features=sqrt(n_features)}
-#' \item{'sqrt'}{then max_features=sqrt(n_features) (same as “auto”)}
-#' \item{'log2'}{then max_features=log2(n_features).}
-#' \item{NULL}{then max_features=n_features}
+#' \item int then consider max_features features at each split.
+#' \item float then max_features is a fraction and round(max_features * n_features) features are considered at each split
+#' \item 'sqrt' then max_features=sqrt(n_features)
+#' \item 'log2' then max_features=log2(n_features)
+#' \item NULL then max_features=n_features
 #' }
 #' @param maxLeafNodes    (list) Grow trees with max_leaf_nodes in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes.
 #' @param minImpurityDecrease (list) A node will be split if this split induces a decrease of the impurity greater than or equal to this value.
 #' @param bootstrap (list) Whether bootstrap samples are used when building trees. If False, the whole dataset is used to build each tree.
 #' @param maxSamples (list) If bootstrap is True, the number of samples to draw from X to train each base estimator.
 #' @param oobScore (list) Whether to use out-of-bag samples to estimate the generalization score. Only available if bootstrap=True.
-#' @param classWeight (list) Weights associated with classes. If not given, all classes are supposed to have weight one. {NULL, “balanced”, “balanced_subsample”}
+#' @param classWeight (list) Weights associated with classes. If not given, all classes are supposed to have weight one. NULL, “balanced”, “balanced_subsample”
 #' @param nJobs The number of jobs to run in parallel.
 #' @param seed  A seed when training the final model
 #'
 #' @examples
 #' \dontrun{
-#' model.rf <- setRandomForest(mtries=list('auto',5,20),  ntrees=c(10,100), 
+#' model.rf <- setRandomForest(mtries=list('auto',5,20),  ntrees=c(10,100),
 #'                            maxDepth=c(5,20))
-#' }                           
+#' }
 #' @export
-setRandomForest <- function(
-  ntrees =  list(100,500),
-  criterion = list('gini'),
-  maxDepth = list(4,10,17),
-  minSamplesSplit = list(2,5),
-  minSamplesLeaf = list(1,10),
-  minWeightFractionLeaf = list(0),
-  mtries = list('auto', 'log2'),
-  maxLeafNodes = list(NULL),
-  minImpurityDecrease = list(0),
-  bootstrap = list(TRUE),
-  maxSamples = list(NULL, 0.9),
-  oobScore = list(FALSE),
-  nJobs = list(NULL),
-  classWeight = list('balanced_subsample', NULL),
-  seed = sample(100000,1)
-  ){
-  
-  checkIsClass(seed, c('numeric','integer'))
+setRandomForest <- function(ntrees =  list(100, 500),
+                            criterion = list('gini'),
+                            maxDepth = list(4, 10, 17),
+                            minSamplesSplit = list(2, 5),
+                            minSamplesLeaf = list(1, 10),
+                            minWeightFractionLeaf = list(0),
+                            mtries = list('sqrt', 'log2'),
+                            maxLeafNodes = list(NULL),
+                            minImpurityDecrease = list(0),
+                            bootstrap = list(TRUE),
+                            maxSamples = list(NULL, 0.9),
+                            oobScore = list(FALSE),
+                            nJobs = list(NULL),
+                            classWeight = list(NULL),
+                            seed = sample(100000, 1)) {
+  checkIsClass(seed, c('numeric', 'integer'))
   checkIsClass(ntrees, c('list'))
-  checkIsClass(criterion, c('list')) 
+  checkIsClass(criterion, c('list'))
   checkIsClass(maxDepth, c('list'))
   checkIsClass(minSamplesSplit, c('list'))
   checkIsClass(minSamplesLeaf, c('list'))
@@ -575,51 +592,51 @@ setRandomForest <- function(
   checkIsClass(mtries, c('list'))
   checkIsClass(maxLeafNodes, c('list'))
   checkIsClass(minImpurityDecrease, c('list'))
-  checkIsClass(bootstrap, c('list')) 
+  checkIsClass(bootstrap, c('list'))
   checkIsClass(maxSamples, c('list'))
   checkIsClass(oobScore, c('list'))
   checkIsClass(nJobs, c('list'))
   checkIsClass(classWeight, c('list'))
   
   # convert to integer when needed
-  for(i in 1:length(ntrees)){
-    if(class(ntrees[[i]]) %in% c("numeric", "integer")){
+  for (i in 1:length(ntrees)) {
+    if (inherits(x = ntrees[[i]], what = c("numeric", "integer"))) {
       ntrees[[i]] <- as.integer(ntrees[[i]])
     }
   }
-  for(i in 1:length(maxDepth)){
-    if(class(maxDepth[[i]]) %in% c("numeric", "integer")){
+  for (i in 1:length(maxDepth)) {
+    if (inherits(x = maxDepth[[i]], what =  c("numeric", "integer"))) {
       maxDepth[[i]] <- as.integer(maxDepth[[i]])
     }
   }
   
-  for(i in 1:length(minSamplesSplit)){
-    if(minSamplesSplit[[i]]>=1){
+  for (i in 1:length(minSamplesSplit)) {
+    if (minSamplesSplit[[i]] >= 1) {
       minSamplesSplit[[i]] <- as.integer(minSamplesSplit[[i]])
     }
   }
   
-  for(i in 1:length(minSamplesLeaf)){
-    if(minSamplesLeaf[[i]]>=1){
+  for (i in 1:length(minSamplesLeaf)) {
+    if (minSamplesLeaf[[i]] >= 1) {
       minSamplesLeaf[[i]] <- as.integer(minSamplesLeaf[[i]])
     }
   }
   
-  for(i in 1:length(maxLeafNodes)){
-    if(class(maxLeafNodes[[i]]) %in% c("numeric", "integer")){
+  for (i in 1:length(maxLeafNodes)) {
+    if (inherits(x = maxLeafNodes[[i]], what =  c("numeric", "integer"))) {
       maxLeafNodes[[i]] <- as.integer(maxLeafNodes[[i]])
     }
   }
   
-  for(i in 1:length(nJobs)){
-    if(class(nJobs[[i]]) %in% c("numeric", "integer")){
+  for (i in 1:length(nJobs)) {
+    if (inherits(x = nJobs[[i]], what =  c("numeric", "integer"))) {
       nJobs[[i]] <- as.integer(nJobs[[i]])
     }
   }
   
-  for(i in 1:length(maxSamples)){
-    if(class(maxSamples[[i]]) %in% c("numeric", "integer")){
-      if(maxSamples[[i]] >= 1){
+  for (i in 1:length(maxSamples)) {
+    if (inherits(x = maxSamples[[i]], what =  c("numeric", "integer"))) {
+      if (maxSamples[[i]] >= 1) {
         maxSamples[[i]] <- as.integer(maxSamples[[i]])
       }
     }
@@ -646,48 +663,46 @@ setRandomForest <- function(
   param <- listCartesian(paramGrid)
   
   attr(param, 'settings') <- list(
+    modelType = 'randomForest',
     seed = seed[[1]],
-    paramNames = names(paramGrid), #use this for logging params
+    paramNames = names(paramGrid),
+    #use this for logging params
     requiresDenseMatrix = F,
-    saveToJson = T,
     name = "Random forest",
-    pythonImport = 'sklearn',
-    pythonImportSecond = 'ensemble',
-    pythonClassifier = 'RandomForestClassifier'
-  ) 
+    pythonModule = "sklearn.ensemble",
+    pythonClass = "RandomForestClassifier"
+  )
   
+  attr(param, 'saveToJson') <- T
   attr(param, 'saveType') <- 'file'
   
-  result <- list(
-    fitFunction = "fitSklearn",
-    param = param
-  )
+  result <- list(fitFunction = "fitSklearn",
+                 param = param)
   class(result) <- "modelSettings"
   
   return(result)
 }
 
 
-RandomForestClassifierInputs <- function(classifier, param){
-  
+RandomForestClassifierInputs <- function(classifier, param) {
   model <- classifier(
-    n_estimators = param[[which.max(names(param)=='ntrees')]],
-    criterion = param[[which.max(names(param)=='criterion')]],
-    max_depth = param[[which.max(names(param)=='maxDepth')]],
-    min_samples_split = param[[which.max(names(param)=='minSamplesSplit')]],
-    min_samples_leaf = param[[which.max(names(param)=='minSamplesLeaf')]],
-    min_weight_fraction_leaf = param[[which.max(names(param)=='minWeightFractionLeaf')]],
-    max_features = param[[which.max(names(param)=='mtries')]],
-    max_leaf_nodes = param[[which.max(names(param)=='maxLeafNodes')]],
-    min_impurity_decrease = param[[which.max(names(param)=='minImpurityDecrease')]],
-    bootstrap = param[[which.max(names(param)=='bootstrap')]],
-    max_samples = param[[which.max(names(param)=='maxSamples')]],
-    oob_score = param[[which.max(names(param)=='oobScore')]],
-    n_jobs = param[[which.max(names(param)=='nJobs')]],
-    random_state = param[[which.max(names(param)=='seed')]],
-    verbose = 0,
+    n_estimators = param[[which.max(names(param) == 'ntrees')]],
+    criterion = param[[which.max(names(param) == 'criterion')]],
+    max_depth = param[[which.max(names(param) == 'maxDepth')]],
+    min_samples_split = param[[which.max(names(param) == 'minSamplesSplit')]],
+    min_samples_leaf = param[[which.max(names(param) == 'minSamplesLeaf')]],
+    min_weight_fraction_leaf = param[[which.max(names(param) == 'minWeightFractionLeaf')]],
+    max_features = param[[which.max(names(param) == 'mtries')]],
+    max_leaf_nodes = param[[which.max(names(param) == 'maxLeafNodes')]],
+    min_impurity_decrease = param[[which.max(names(param) == 'minImpurityDecrease')]],
+    bootstrap = param[[which.max(names(param) == 'bootstrap')]],
+    max_samples = param[[which.max(names(param) == 'maxSamples')]],
+    oob_score = param[[which.max(names(param) == 'oobScore')]],
+    n_jobs = param[[which.max(names(param) == 'nJobs')]],
+    random_state = param[[which.max(names(param) == 'seed')]],
+    verbose = 0L,
     warm_start = F,
-    class_weight = param[[which.max(names(param)=='classWeight')]]
+    class_weight = param[[which.max(names(param) == 'classWeight')]]
   )
   
   return(model)
@@ -696,10 +711,10 @@ RandomForestClassifierInputs <- function(classifier, param){
 
 
 #' Create setting for the python sklearn SVM (SVC function)
-#' @param C             (list) Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive. The penalty is a squared l2 penalty.      
+#' @param C             (list) Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive. The penalty is a squared l2 penalty.
 #' @param kernel        (list) Specifies the kernel type to be used in the algorithm. one of ‘linear’, ‘poly’, ‘rbf’, ‘sigmoid’, ‘precomputed’. If none is given ‘rbf’ will be used.
 #' @param degree        (list) degree of kernel function is significant only in poly, rbf, sigmoid
-#' @param gamma         (list) kernel coefficient for rbf and poly, by default 1/n_features will be taken. {‘scale’, ‘auto’} or float, default=’scale’
+#' @param gamma         (list) kernel coefficient for rbf and poly, by default 1/n_features will be taken. ‘scale’, ‘auto’ or float, default=’scale’
 #' @param coef0         (list) independent term in kernel function. It is only significant in poly/sigmoid.
 #' @param shrinking     (list) whether to use the shrinking heuristic.
 #' @param tol           (list) Tolerance for stopping criterion.
@@ -712,24 +727,20 @@ RandomForestClassifierInputs <- function(classifier, param){
 #' model.svm <- setSVM(kernel='rbf', seed = NULL)
 #' }
 #' @export
-setSVM <- function(
-  C = list(1,0.9,2,0.1), 
-  kernel = list('rbf'),
-  degree = list(1,3,5), 
-  gamma = list('scale', 1e-04, 3e-05, 0.001, 0.01, 0.25),
-  coef0 = list(0.0),
-  shrinking = list(TRUE), 
-  tol = list(0.001),
-  classWeight = list('balanced', NULL), 
-  cacheSize  = 500,
-  seed = sample(100000,1)
-  ){
-  
-  
-  checkIsClass(seed, c('numeric','integer'))
-  checkIsClass(cacheSize, c('numeric','integer'))
+setSVM <- function(C = list(1, 0.9, 2, 0.1),
+                   kernel = list('rbf'),
+                   degree = list(1, 3, 5),
+                   gamma = list('scale', 1e-04, 3e-05, 0.001, 0.01, 0.25),
+                   coef0 = list(0.0),
+                   shrinking = list(TRUE),
+                   tol = list(0.001),
+                   classWeight = list(NULL),
+                   cacheSize  = 500,
+                   seed = sample(100000, 1)) {
+  checkIsClass(seed, c('numeric', 'integer'))
+  checkIsClass(cacheSize, c('numeric', 'integer'))
   checkIsClass(C, c('list'))
-  checkIsClass(kernel, c('list')) 
+  checkIsClass(kernel, c('list'))
   checkIsClass(degree, c('list'))
   checkIsClass(gamma, c('list'))
   checkIsClass(coef0, c('list'))
@@ -737,13 +748,12 @@ setSVM <- function(
   checkIsClass(tol, c('list'))
   checkIsClass(classWeight, c('list'))
   
-  for(i in 1:length(degree)){
-    if(class(degree[[i]]) %in% c("numeric", "integer")){
+  for (i in 1:length(degree)) {
+    if (inherits(x = degree[[i]], what =  c("numeric", "integer"))) {
       degree[[i]] <- as.integer(degree[[i]])
     }
   }
   
-  
   paramGrid = list(
     C = C,
     kernel = kernel,
@@ -758,48 +768,46 @@ setSVM <- function(
   )
   
   param <- listCartesian(paramGrid)
-
+  
   
   attr(param, 'settings') <- list(
+    modelType = 'svm',
     seed = seed[[1]],
-    paramNames = names(paramGrid), #use this for logging params
+    paramNames = names(paramGrid),
+    #use this for logging params
     requiresDenseMatrix = F,
-    saveToJson = T,
     name = "Support Vector Machine",
-    pythonImport = 'sklearn',
-    pythonImportSecond = 'svm',
-    pythonClassifier = 'SVC'
-  ) 
+    pythonModule = "sklearn.svm",
+    pythonClass = "SVC"
+  )
   
+  attr(param, 'saveToJson') <- T
   attr(param, 'saveType') <- 'file'
- 
-  result <- list(
-    fitFunction = "fitSklearn",
-    param = param
-  )
+  
+  result <- list(fitFunction = "fitSklearn",
+                 param = param)
   class(result) <- "modelSettings"
   
   return(result)
 }
 
-SVCInputs <- function(classifier, param){
-  
+SVCInputs <- function(classifier, param) {
   model <- classifier(
-    C  = param[[which.max(names(param)=='C')]],
-    kernel = param[[which.max(names(param)=='kernel')]],
-    degree  = param[[which.max(names(param)=='degree')]],
-    gamma  = param[[which.max(names(param)=='gamma')]],
-    coef0 = param[[which.max(names(param)=='coef0')]],
-    shrinking = param[[which.max(names(param)=='shrinking')]],
+    C  = param[[which.max(names(param) == 'C')]],
+    kernel = param[[which.max(names(param) == 'kernel')]],
+    degree  = param[[which.max(names(param) == 'degree')]],
+    gamma  = param[[which.max(names(param) == 'gamma')]],
+    coef0 = param[[which.max(names(param) == 'coef0')]],
+    shrinking = param[[which.max(names(param) == 'shrinking')]],
     probability = T,
-    tol = param[[which.max(names(param)=='tol')]],
-    cache_size = param[[which.max(names(param)=='cacheSize')]],
-    class_weight = param[[which.max(names(param)=='classWeight')]],
+    tol = param[[which.max(names(param) == 'tol')]],
+    cache_size = param[[which.max(names(param) == 'cacheSize')]],
+    class_weight = param[[which.max(names(param) == 'classWeight')]],
     verbose = F,
     max_iter = as.integer(-1),
     decision_function_shape = 'ovr',
     break_ties = F,
-    random_state = param[[which.max(names(param)=='seed')]]
+    random_state = param[[which.max(names(param) == 'seed')]]
   )
   
   return(model)
diff --git a/R/SklearnToJson.R b/R/SklearnToJson.R
new file mode 100644
index 000000000..2a2771338
--- /dev/null
+++ b/R/SklearnToJson.R
@@ -0,0 +1,443 @@
+# @file SklearnToJson.R
+#
+# Copyright 2023 Observational Health Data Sciences and Informatics
+#
+# This file is part of PatientLevelPrediction
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitatons under the License.
+#
+#' Saves sklearn python model object to json in path
+#' @param     model a fitted sklearn python model object
+#' @param     path  path to the saved model file
+#' @export
+sklearnToJson <- function(model, path) {
+  py <- reticulate::import_builtins(convert=FALSE)
+  json <- reticulate::import("json", convert=FALSE)
+  if (inherits(model, "sklearn.tree._classes.DecisionTreeClassifier")) {
+    serializedModel <- serializeDecisionTree(model)
+  } else if (inherits(model, "sklearn.ensemble._forest.RandomForestClassifier")) {
+    serializedModel <- serializeRandomForest(model)
+  } else if (inherits(model, "sklearn.ensemble._weight_boosting.AdaBoostClassifier")) {
+    serializedModel <- serializeAdaboost(model)
+  } else if (inherits(model, "sklearn.naive_bayes.GaussianNB")) {
+    serializedModel <- serializeNaiveBayes(model)
+  } else if (inherits(model, "sklearn.neural_network._multilayer_perceptron.MLPClassifier")) {
+    serializedModel <- serializeMLP(model) 
+  } else if (inherits(model, "sklearn.svm._classes.SVC" )) {
+    serializedModel <- serializeSVM(model)
+  } else {
+    stop("Unsupported model")
+  }
+  
+  with(py$open(path, "w"), as=file, {
+    json$dump(serializedModel, fp=file)
+  })
+  return(invisible())
+}
+
+#' Loads sklearn python model from json 
+#' @param     path  path to the model json file
+#' @export
+sklearnFromJson <- function(path) {
+  py <- reticulate::import_builtins(convert=FALSE)
+  json <- reticulate::import("json", convert=FALSE)
+  with(py$open(path, "r"), as=file, {
+    model <- json$load(fp=file)
+  })
+  if (reticulate::py_bool(model["meta"] == "decision-tree")) {
+    model <- deSerializeDecisionTree(model)
+  } else if (reticulate::py_bool(model["meta"] == "rf")) {
+    model <- deSerializeRandomForest(model)
+  } else if (reticulate::py_bool(model["meta"] == "adaboost")) {
+    model <- deSerializeAdaboost(model)
+  } else if (reticulate::py_bool(model["meta"] == "naive-bayes")) {
+    model <- deSerializeNaiveBayes(model)
+  } else if (reticulate::py_bool(model["meta"] == "mlp")) {
+    model <- deSerializeMlp(model)
+  }  else if (reticulate::py_bool(model["meta"] == "svm")) {
+    model <- deSerializeSVM(model)
+  } else {
+    stop("Unsupported model")
+  }
+  return(model)
+}
+
+serializeTree <- function(tree) {
+  serializedTree <- tree$`__getstate__`()
+  dtypes <- serializedTree["nodes"]$dtype
+  
+  serializedTree["nodes"] <- serializedTree["nodes"]$tolist()
+  serializedTree["values"] <- serializedTree["values"]$tolist()
+  
+  return(list(serializedTree, dtypes)) 
+}
+
+deSerializeTree <- function(tree_dict, nFeatures, nClasses, nOutputs) {
+  # TODO the below only works for tree_dict loaded from json, if not it
+  for (i in 0:(length(tree_dict["nodes"])-1)) {
+    reticulate::py_set_item(tree_dict["nodes"], i,
+                            reticulate::tuple(reticulate::py_to_r(tree_dict["nodes"][i])))
+  }
+  
+  names <- list("left_child", "right_child", "feature", "threshold", "impurity", "n_node_samples", "weighted_n_node_samples")
+  if (length(tree_dict["nodes"][0])==8) {
+    # model used sklearn>=1.3 which added a parameter
+    names[[8]] <- "missing_go_to_left"
+  }
+  
+  sklearn <- reticulate::import("sklearn")
+  np <- reticulate::import("numpy", convert = FALSE)
+  
+  tree_dict["nodes"] <- np$array(tree_dict["nodes"], 
+                                 dtype=np$dtype(reticulate::dict(
+                                   names = names,
+                                   formats = tree_dict["nodes_dtype"]
+                                   )))
+  tree_dict["values"] <- np$array(tree_dict["values"])
+  
+  Tree <- sklearn$tree$`_tree`$Tree(nFeatures, 
+                                    np$array(reticulate::tuple(nClasses), 
+                                             dtype=np$intp),
+                               nOutputs)
+  
+  Tree$`__setstate__`(tree_dict)
+  
+  return(Tree)
+}
+
+serializeDecisionTree <- function(model) {
+   tree <- serializeTree(model$tree_)
+   dtypes <- tree[[2]]
+   tree <- tree[[1]]
+   py <- reticulate::import_builtins(convert=FALSE)
+   serialized_model <- reticulate::dict(
+     "meta" = "decision-tree",
+     "feature_importances_" = model$feature_importances_$tolist(),
+     "max_features_" = model$max_features_,
+     "n_classes_" = py$int(model$n_classes_),
+     "n_features_in_" = model$n_features_in_,
+     "n_outputs_" = model$n_outputs_,
+     "tree_" = tree,
+     "classes_" = model$classes_$tolist(),
+     "params" = model$get_params()
+   )
+   
+   tree_dtypes <- list()
+   for (i in 0:(length(dtypes)-1)) {
+     tree_dtypes <- c(tree_dtypes, dtypes[[i]]$str)
+   }
+   
+   serialized_model["tree_"]["nodes_dtype"] <- tree_dtypes
+   return(serialized_model)
+}
+
+deSerializeDecisionTree <- function(model_dict) {
+  np <- reticulate::import("numpy", convert=FALSE)
+  sklearn <- reticulate::import("sklearn", convert=FALSE)
+  deserialized_model <- do.call(sklearn$tree$DecisionTreeClassifier,
+                                reticulate::py_to_r(model_dict["params"]))
+  
+  deserialized_model$classes_ <- np$array(model_dict["classes_"])
+  deserialized_model$max_features_ <- model_dict["max_features_"]
+  deserialized_model$n_classes_ <- model_dict["n_classes_"]
+  deserialized_model$n_features_in <- model_dict["n_features_in_"]
+  deserialized_model$n_outputs_ <- model_dict["n_outputs_"]
+  
+  tree <- deSerializeTree(model_dict["tree_"],
+                          model_dict["n_features_in_"],
+                          model_dict["n_classes_"],
+                          model_dict["n_outputs_"])
+  deserialized_model$tree_ <- tree
+  
+  return(deserialized_model)
+}
+
+serializeRandomForest <- function(model) {
+  estimators <- list()
+  for (i in 1:length(model$estimators_)) {
+    estimators <- c(estimators, serializeDecisionTree(model$estimators_[i - 1]))
+  }
+  
+  serialized_model <- reticulate::dict(
+    "meta" = "rf",
+    "max_depth" = model$max_depth,
+    "min_samples_split" = model$min_samples_split,
+    "min_samples_leaf" = model$min_samples_leaf,
+    "min_weight_fraction_leaf" = model$min_weight_fraction_leaf,
+    "max_features" = model$max_features,
+    "max_leaf_nodes" = model$max_leaf_nodes,
+    "min_impurity_decrease" = model$min_impurity_decrease,
+    "min_impurity_split" = model$min_samples_split,
+    "n_features_in_" = model$n_features_in_,
+    "n_outputs_" = model$n_outputs_,
+    "classes_" = model$classes_$tolist(),
+    "estimators_" = reticulate::r_to_py(estimators),
+    "params" = model$get_params(),
+    "n_classes_" = model$n_classes_)
+  
+  if (reticulate::py_bool(model$`__dict__`["oob_score_"] != reticulate::py_none())) { 
+    serialized_model["oob_score_"] <- model$oob_score_ 
+    serialized_model["oob_decision_function_"] <- model$oob_decision_function_$tolist()
+  }
+  
+  return(serialized_model)
+}
+  
+deSerializeRandomForest <- function(model_dict) {
+  np <- reticulate::import("numpy", convert=FALSE)
+  sklearn <- reticulate::import("sklearn", convert=FALSE)
+  model <- do.call(sklearn$ensemble$RandomForestClassifier,
+                   reticulate::py_to_r(model_dict["params"]))
+  
+  estimators <- list()
+  for (i in 1:length(model_dict$estimators_)) {
+    estimators <- c(estimators, deSerializeDecisionTree(model_dict["estimators_"][i - 1]))
+  }
+  
+  model$estimators_ <- np$array(estimators)
+    
+  model$classes_ <- np$array(model_dict["classes_"])
+  model$n_features_in_ <- model_dict["n_features_in_"]
+  model$n_outputs_ <- model_dict["n_outputs_"]  
+  model$max_depth <- model_dict["max_depth"]
+  model$min_samples_split <- model_dict["min_samples_split"]
+  model$min_samples_leaf <- model_dict["min_samples_leaf"]
+  model$min_weight_fraction_leaf <- model_dict["min_weight_fraction_leaf"]
+  model$max_features <- model_dict["max_features"]
+  model$max_leaf_nodes <- model_dict["max_leaf_nodes"]
+  model$min_impurity_decrease <- model_dict["min_impurity_decrease"]
+  model$min_impurity_split <- model_dict["min_impurity_split"]
+  model$n_classes_ <- model_dict["n_classes_"]
+  
+  if (reticulate::py_bool(model_dict$oob_score_ != reticulate::py_none())){
+    model$oob_score_ <- model_dict["oob_score_"]
+    model$oob_decision_function_ <-  model_dict["oob_decision_function_"]
+  }
+  return(model)    
+}
+
+serializeAdaboost <- function(model) {
+  estimators <- list()
+  for (i in 1:length(model$estimators_)) {
+    estimators <- c(estimators, serializeDecisionTree(model$estimators_[i - 1]))
+  }
+  serialized_model <- reticulate::dict(
+    "meta" = "adaboost",
+    "estimators_" = reticulate::r_to_py(estimators),
+    "n_features_in_" = model$n_features_in_,
+    "n_classes_" = model$n_classes_,
+    "params" = model$get_params(),
+    "classes_" = model$classes_$tolist(),
+    "estimator_weights_" = model$estimator_weights_$tolist())
+  
+  return(serialized_model)
+}
+
+deSerializeAdaboost <- function(model_dict) {
+  np <- reticulate::import("numpy", convert=FALSE)
+  sklearn <- reticulate::import("sklearn", convert=FALSE)
+  model <- do.call(sklearn$ensemble$AdaBoostClassifier,
+                   reticulate::py_to_r(model_dict["params"]))
+  estimators <- list()
+  for (i in 1:length(model_dict$estimators_)) {
+    estimators <- c(estimators, deSerializeDecisionTree(model_dict["estimators_"][i - 1]))
+  }
+  
+  model$estimators_ <- np$array(estimators)
+  model$classes_ <- np$array(model_dict["classes_"])
+  model$n_features_in_ <- model_dict["n_features_in_"]
+  model$n_classes_ <- model_dict["n_classes_"]
+  model$estimator_weights_ <- np$array(model_dict["estimator_weights_"])
+  
+  return(model)
+}
+
+serializeNaiveBayes <- function(model) {
+  serialized_model = reticulate::dict(
+    "meta" = "naive-bayes",
+    "classes_" = model$classes_$tolist(),
+    "class_count_" = model$class_count_$tolist(),
+    "class_prior_" = model$class_prior_$tolist(),
+    "theta_" = model$theta_$tolist(),
+    "epsilon_" = model$epsilon_,
+    "params" = model$get_params(),
+    "var_" = model$var_$tolist()
+  )
+  return(serialized_model)
+}
+
+deSerializeNaiveBayes <- function(model_dict) {
+  sklearn <- reticulate::import("sklearn", convert=FALSE)
+  np <- reticulate::import("numpy", convert=FALSE)
+  model <- do.call(sklearn$naive_bayes$GaussianNB,
+                   reticulate::py_to_r(model_dict["params"]))
+  
+  model$classes_ <- np$array(model_dict["classes_"])
+  model$class_count_ <- np$array(model_dict["class_count_"])
+  model$class_prior_ <- np$array(model_dict["class_prior_"])
+  model$theta_ <- np$array(model_dict["theta_"])
+  model$epsilon_ <- model_dict["epsilon_"]
+  model$var_ <- np$array(model_dict["var_"])
+  
+  return(model)
+}
+
+serializeMLP <- function(model) {
+  # TODO Check if length(intercepts_) is ever different from length(coefs_)
+  for (i in 0:(length(model$coefs_) - 1)) {
+    reticulate::py_set_item(model$coefs_, i, 
+                            model$coefs_[i]$tolist())
+    reticulate::py_set_item(model$intercepts_, i, 
+                            model$intercepts_[i]$tolist())
+  }
+  serialized_model <- reticulate::dict(
+    "meta" = "mlp",
+    "coefs_" = model$coefs_,
+    "loss_" = model$loss_,
+    "intercepts_" = model$intercepts_,
+    "n_iter_" = model$n_iter_,
+    "n_layers_" = model$n_layers_,
+    "n_outputs_" = model$n_outputs_,
+    "out_activation_" = model$out_activation_,
+    "params" = model$get_params(),
+    "classes_" = model$classes_$tolist()
+  )
+  return(serialized_model)
+}
+
+deSerializeMlp <- function(model_dict) {
+  sklearn <- reticulate::import("sklearn", convert=FALSE)
+  np <- reticulate::import("numpy", convert=FALSE)
+  
+  model <- do.call(sklearn$neural_network$MLPClassifier,
+                   reticulate::py_to_r(model_dict["params"]))
+  for (i in 0:(length(model_dict["coefs_"]) - 1)) {
+    reticulate::py_set_item(model_dict["coefs_"], i,
+                            np$array(model_dict["coefs_"][i]))
+    reticulate::py_set_item(model_dict["intercepts_"], i,
+                            np$array(model_dict["intercepts_"][i]))
+    
+  }
+  model$coefs_ = model_dict["coefs_"]
+  model$loss_ = model_dict["loss_"]
+  model$intercepts_ = model_dict["intercepts_"]
+  model$n_iter_ = model_dict["n_iter_"]
+  model$n_layers_ = model_dict["n_layers_"]
+  model$n_outputs_ = model_dict["n_outputs_"]
+  model$out_activation_ = model_dict["out_activation_"]
+  model$classes_ = np$array(model_dict["classes_"])
+  
+  return(model)
+}
+
+serializeSVM <- function(model) {
+  serialized_model = reticulate::dict(
+    "meta" = "svm",
+    "class_weight_" = model$class_weight_$tolist(),
+    "classes_" = model$classes_$tolist(),
+    "support_" = model$support_$tolist(),
+    "n_support_" = model$n_support_$tolist(),
+    "intercept_" = model$intercept_$tolist(),
+    "probA_" = model$probA_$tolist(),
+    "probB_" = model$probB_$tolist(),
+    "_intercept_" = model$`_intercept_`$tolist(),
+    "shape_fit_" = model$shape_fit_,
+    "_gamma" = model$`_gamma`,
+    "params" = model$get_params()
+  )
+  if (inherits(model$support_vectors_, "numpy.ndarray")) {
+    serialized_model["support_vectors_"] <- model$support_vectors_$tolist()
+  } else {
+    serialized_model["support_vectors_"] <- serializeCsrMatrix(model$support_vectors_)
+  }
+  
+  if (inherits(model$dual_coef_, "numpy.ndarray")) {
+    serialized_model["dual_coef_"] <- model$dual_coef_$tolist()
+  } else {
+    serialized_model["dual_coef_"] <- serializeCsrMatrix(model$dual_coef_)
+  }
+  
+  if (inherits(model$`_dual_coef_`, "numpy.ndarray")) {
+    serialized_model["_dual_coef_"] <- model$`_dual_coef_`$tolist()
+  } else {
+    serialized_model["_dual_coef_"] <- serializeCsrMatrix(model$`_dual_coef_`)
+  }
+ return(serialized_model)
+}
+
+deSerializeSVM <- function(model_dict) {
+  sklearn <- reticulate::import("sklearn", convert=FALSE) 
+  np <- reticulate::import("numpy", convert=FALSE)
+  model <- do.call(sklearn$svm$SVC, 
+                   reticulate::py_to_r(model_dict["params"]))
+  model$shape_fit_ <- model_dict$shape_fit_
+  model$`_gamma`<- model_dict["_gamma"]
+  model$class_weight_ <- np$array(model_dict$class_weight_)$astype(np$float64)
+  model$classes_  <- np$array(model_dict["classes_"])
+  model$support_ <- np$array(model_dict["support_"])$astype(np$int32)
+  model$`_n_support` <- np$array(model_dict["n_support_"])$astype(np$int32)
+  model$intercept_ <- np$array(model_dict["intercept_"])$astype(np$float64)
+  model$`_probA` <- np$array(model_dict["probA_"])$astype(np$float64)
+  model$`_probB` <- np$array(model_dict["probB_"])$astype(np$float64)
+  model$`_intercept_` <- np$array(model_dict["_intercept_"])$astype(np$float64)
+  
+  if (reticulate::py_bool((model_dict$support_vectors_["meta"] != reticulate::py_none())) & 
+      (reticulate::py_bool(model_dict$support_vectors_["meta"] == "csr"))) {
+    model$support_vectors_ <- deSerializeCsrMatrix(model_dict$support_vectors_)
+    model$`_sparse` <- TRUE
+  } else {
+    model$support_vectors_ <- np$array(model_dict$support_vectors_)$astype(np$float64)
+    model$`_sparse` <- FALSE
+  }
+  if (reticulate::py_bool((model_dict$dual_coef_["meta"] != reticulate::py_none())) & 
+      (reticulate::py_bool(model_dict$dual_coef_["meta"] == "csr"))) {
+    model$dual_coef_ <- deSerializeCsrMatrix(model_dict$dual_coef_)
+  } else {
+    model$dual_coef_ <- np$array(model_dict$dual_coef_)$astype(np$float64)
+  }
+  
+  if (reticulate::py_bool((model_dict$`_dual_coef_`["meta"] != reticulate::py_none())) & 
+      (reticulate::py_bool(model_dict$`_dual_coef_`["meta"] == "csr"))) {
+    model$`_dual_coef_` <- deSerializeCsrMatrix(model_dict$`dual_coef_`)
+  } else {
+    model$`_dual_coef_` <- np$array(model_dict$`_dual_coef_`)$astype(np$float64)
+  }
+  return(model)
+}    
+
+serializeCsrMatrix <- function(csr_matrix) {
+  serialized_csr_matrix = reticulate::dict(
+    "meta" = "csr",
+    "indices" = csr_matrix$indices$tolist(),
+    "indptr" = csr_matrix$indptr$tolist(),
+    "_shape"= csr_matrix$`_shape`)
+  serialized_csr_matrix["data"] <- csr_matrix$data$tolist()
+  return(serialized_csr_matrix)
+}
+
+deSerializeCsrMatrix <- function(csr_dict, 
+                                 data_type=np$float64, 
+                                 indices_type=np$int32, 
+                                 indptr_type=np$int32) {
+  sp <- reticulate::import("scipy", convert=FALSE)
+  np <- reticulate::import("numpy", convert=FALSE)
+  csr_matrix <- sp$sparse$csr_matrix(
+    reticulate::tuple(list(np$array(csr_dict["data"])$astype(data_type),
+                           np$array(csr_dict["indices"])$astype(indices_type),
+                           np$array(csr_dict["indptr"])$astype(indptr_type))), 
+    shape=csr_dict["shape"]
+  )
+  return(csr_matrix)
+}
+
+ 
\ No newline at end of file
diff --git a/R/ThresholdSummary.R b/R/ThresholdSummary.R
index e5203f0d2..b36d86014 100644
--- a/R/ThresholdSummary.R
+++ b/R/ThresholdSummary.R
@@ -81,6 +81,14 @@ getThresholdSummary_binary <- function(prediction, evalColumn, ...){
     # sort prediction
     predictionOfInterest <- predictionOfInterest[order(-predictionOfInterest$value),]
     
+    # because of numerical precision issues (I think), in very rare cases the preferenceScore 
+    # is not monotonically decreasing after this sort (it should follow the predictions)
+    # as a fix I remove the troublesome row from influencing the thresholdSummary
+    if (!all(predictionOfInterest$preferenceScore == cummin(predictionOfInterest$preferenceScore))) {
+      troubleRow <- (which((predictionOfInterest$preferenceScore == cummin(predictionOfInterest$preferenceScore))==FALSE))
+      predictionOfInterest <- predictionOfInterest[-troubleRow,]
+      }
+    
     # create indexes
     if(length(predictionOfInterest$preferenceScore)>100){
       indexesOfInt <- c(
@@ -166,6 +174,7 @@ getThresholdSummary_binary <- function(prediction, evalColumn, ...){
       )
     )
     
+    
   }
   
   result <- as.data.frame(result)
@@ -196,17 +205,19 @@ getThresholdSummary_survival <- function(prediction, evalColumn, timepoint, ...)
     )
     
     nbSummary <- tryCatch(
-      {
+      { xstart <- max(min(preddat$p),0.001); 
+        xstop <- min(max(preddat$p),0.99);
         stdca(
           data = preddat, 
           outcome = "y", 
           ttoutcome = "t", 
           timepoint = timepoint,  
           predictors = "p", 
-          xstart = max(min(preddat$p),0.001), #0.001, 
-          xstop = min(max(preddat$p),0.99), 
-          xby = 0.001, 
-          smooth=F
+          xstart = xstart, 
+          xstop = xstop, 
+          xby = (xstop - xstart)/100,
+          smooth = FALSE,
+          graph = FALSE
         )
       },
       error = function(e){ParallelLogger::logError(e); return(NULL)}
@@ -249,7 +260,7 @@ stdca <- function (data, outcome, ttoutcome, timepoint, predictors, xstart = 0.0
       1), outcome]) > 0) & cmprsk == FALSE) {
     stop("outcome must be coded as 0 and 1")
   }
-  if (class(data) != "data.frame") {
+  if (!inherits(x = data, what = "data.frame")) {
     stop("Input data must be class data.frame")
   }
   if (xstart < 0 | xstart > 1) {
@@ -453,6 +464,37 @@ stdca <- function (data, outcome, ttoutcome, timepoint, predictors, xstart = 0.0
 }
 
 
+checkToByTwoTableInputs <- function(TP,FP,FN,TN){
+  # check classes
+  if(!inherits(x = TP, what = c('integer','numeric'))){
+    stop('Incorrect TP class')
+  }
+  if(!inherits(x = FP, what = c('integer','numeric'))){
+    stop('Incorrect FP class')
+  }
+  if(!inherits(x = TN, what = c('integer','numeric'))){
+    stop('Incorrect TN class')
+  }
+  if(!inherits(x = FN, what = c('integer','numeric'))){
+    stop('Incorrect FN class')
+  }
+  
+  # check positive values
+  if(sum(TP<0)>0){
+    stop('TP < 0')
+  }
+  if(sum(FP<0)>0){
+    stop('FP < 0')
+  }
+  if(sum(TN<0)>0){
+    stop('TN < 0')
+  }
+  if(sum(FN<0)>0){
+    stop('FN < 0')
+  }
+  
+  return(invisible(TRUE))
+}
 # making all this single for easy unit testing
 #' Calculate the f1Score
 #'
@@ -468,14 +510,13 @@ stdca <- function (data, outcome, ttoutcome, timepoint, predictors, xstart = 0.0
 #' f1Score value
 #'
 f1Score <- function(TP,TN,FN,FP){
-  if(sum(TP<0)>0) stop('TP < 0')
-  if(sum(FP<0)>0) stop('FP < 0')
-  if(sum(TN<0)>0) stop('TN < 0')
-  if(sum(FN<0)>0) stop('FN < 0')
-  if(class(TP)!='numeric') stop('Incorrect TP class')
-  if(class(FP)!='numeric') stop('Incorrect FP class')
-  if(class(TN)!='numeric') stop('Incorrect TN class')
-  if(class(FN)!='numeric') stop('Incorrect FN class')
+  checkToByTwoTableInputs(
+    TP = TP,
+    FP = FP,
+    FN = FN,
+    TN = TN
+    )
+
   return(2*(TP/(TP+FP))*(TP/(TP+FN))/((TP/(TP+FP))+(TP/(TP+FN))))
 }
 #' Calculate the accuracy
@@ -492,15 +533,15 @@ f1Score <- function(TP,TN,FN,FP){
 #' accuracy value
 #'
 accuracy <- function(TP,TN,FN,FP){
-  if(sum(TP<0)>0) stop('TP < 0')
-  if(sum(FP<0)>0) stop('FP < 0')
-  if(sum(TN<0)>0) stop('TN < 0')
-  if(sum(FN<0)>0) stop('FN < 0')
-  if(class(TP)!='numeric') stop('Incorrect TP class')
-  if(class(FP)!='numeric') stop('Incorrect FP class')
-  if(class(TN)!='numeric') stop('Incorrect TN class')
-  if(class(FN)!='numeric') stop('Incorrect FN class')
-  (TP+TN)/(TP+TN+FP+FN)}
+  checkToByTwoTableInputs(
+    TP = TP,
+    FP = FP,
+    FN = FN,
+    TN = TN
+  )
+  
+  return((TP+TN)/(TP+TN+FP+FN))
+}
 
 #' Calculate the sensitivity
 #'
@@ -516,15 +557,15 @@ accuracy <- function(TP,TN,FN,FP){
 #' sensitivity value
 #'
 sensitivity <- function(TP,TN,FN,FP){
-  if(sum(TP<0)>0) stop('TP < 0')
-  if(sum(FP<0)>0) stop('FP < 0')
-  if(sum(TN<0)>0) stop('TN < 0')
-  if(sum(FN<0)>0) stop('FN < 0')
-  if(class(TP)!='numeric') stop('Incorrect TP class')
-  if(class(FP)!='numeric') stop('Incorrect FP class')
-  if(class(TN)!='numeric') stop('Incorrect TN class')
-  if(class(FN)!='numeric') stop('Incorrect FN class')
-  TP/(TP+FN)}
+  checkToByTwoTableInputs(
+    TP = TP,
+    FP = FP,
+    FN = FN,
+    TN = TN
+  )
+
+  return(TP/(TP+FN))
+}
 
 #' Calculate the falseNegativeRate
 #'
@@ -540,15 +581,15 @@ sensitivity <- function(TP,TN,FN,FP){
 #' falseNegativeRate  value
 #'
 falseNegativeRate <- function(TP,TN,FN,FP){
-  if(sum(TP<0)>0) stop('TP < 0')
-  if(sum(FP<0)>0) stop('FP < 0')
-  if(sum(TN<0)>0) stop('TN < 0')
-  if(sum(FN<0)>0) stop('FN < 0')
-  if(class(TP)!='numeric') stop('Incorrect TP class')
-  if(class(FP)!='numeric') stop('Incorrect FP class')
-  if(class(TN)!='numeric') stop('Incorrect TN class')
-  if(class(FN)!='numeric') stop('Incorrect FN class')
-  FN/(TP+FN)}
+  checkToByTwoTableInputs(
+    TP = TP,
+    FP = FP,
+    FN = FN,
+    TN = TN
+  )
+  
+  return(FN/(TP+FN))
+}
 
 #' Calculate the falsePositiveRate
 #'
@@ -564,15 +605,15 @@ falseNegativeRate <- function(TP,TN,FN,FP){
 #' falsePositiveRate  value
 #'
 falsePositiveRate <- function(TP,TN,FN,FP){
-  if(sum(TP<0)>0) stop('TP < 0')
-  if(sum(FP<0)>0) stop('FP < 0')
-  if(sum(TN<0)>0) stop('TN < 0')
-  if(sum(FN<0)>0) stop('FN < 0')
-  if(class(TP)!='numeric') stop('Incorrect TP class')
-  if(class(FP)!='numeric') stop('Incorrect FP class')
-  if(class(TN)!='numeric') stop('Incorrect TN class')
-  if(class(FN)!='numeric') stop('Incorrect FN class')
-  FP/(FP+TN)}
+  checkToByTwoTableInputs(
+    TP = TP,
+    FP = FP,
+    FN = FN,
+    TN = TN
+  )
+
+  return(FP/(FP+TN))
+}
 
 #' Calculate the specificity
 #'
@@ -588,15 +629,16 @@ falsePositiveRate <- function(TP,TN,FN,FP){
 #' specificity value
 #'
 specificity <- function(TP,TN,FN,FP){
-  if(sum(TP<0)>0) stop('TP < 0')
-  if(sum(FP<0)>0) stop('FP < 0')
-  if(sum(TN<0)>0) stop('TN < 0')
-  if(sum(FN<0)>0) stop('FN < 0')
-  if(class(TP)!='numeric') stop('Incorrect TP class')
-  if(class(FP)!='numeric') stop('Incorrect FP class')
-  if(class(TN)!='numeric') stop('Incorrect TN class')
-  if(class(FN)!='numeric') stop('Incorrect FN class')
-  TN/(FP+TN)}
+  
+  checkToByTwoTableInputs(
+    TP = TP,
+    FP = FP,
+    FN = FN,
+    TN = TN
+  )
+ 
+  return(TN/(FP+TN))
+}
 
 #' Calculate the positivePredictiveValue
 #'
@@ -612,15 +654,16 @@ specificity <- function(TP,TN,FN,FP){
 #' positivePredictiveValue value
 #'
 positivePredictiveValue <- function(TP,TN,FN,FP){
-  if(sum(TP<0)>0) stop('TP < 0')
-  if(sum(FP<0)>0) stop('FP < 0')
-  if(sum(TN<0)>0) stop('TN < 0')
-  if(sum(FN<0)>0) stop('FN < 0')
-  if(class(TP)!='numeric') stop('Incorrect TP class')
-  if(class(FP)!='numeric') stop('Incorrect FP class')
-  if(class(TN)!='numeric') stop('Incorrect TN class')
-  if(class(FN)!='numeric') stop('Incorrect FN class')
-  TP/(TP+FP)}
+  
+  checkToByTwoTableInputs(
+    TP = TP,
+    FP = FP,
+    FN = FN,
+    TN = TN
+  )
+  
+  return(TP/(TP+FP))
+}
 
 #' Calculate the falseDiscoveryRate
 #'
@@ -636,15 +679,16 @@ positivePredictiveValue <- function(TP,TN,FN,FP){
 #' falseDiscoveryRate value
 #'
 falseDiscoveryRate <- function(TP,TN,FN,FP){
-  if(sum(TP<0)>0) stop('TP < 0')
-  if(sum(FP<0)>0) stop('FP < 0')
-  if(sum(TN<0)>0) stop('TN < 0')
-  if(sum(FN<0)>0) stop('FN < 0')
-  if(class(TP)!='numeric') stop('Incorrect TP class')
-  if(class(FP)!='numeric') stop('Incorrect FP class')
-  if(class(TN)!='numeric') stop('Incorrect TN class')
-  if(class(FN)!='numeric') stop('Incorrect FN class')
-  FP/(TP+FP)}
+  
+  checkToByTwoTableInputs(
+    TP = TP,
+    FP = FP,
+    FN = FN,
+    TN = TN
+  )
+  
+  return(FP/(TP+FP))
+  }
 
 #' Calculate the negativePredictiveValue
 #'
@@ -660,15 +704,16 @@ falseDiscoveryRate <- function(TP,TN,FN,FP){
 #' negativePredictiveValue value
 #'
 negativePredictiveValue <- function(TP,TN,FN,FP){
-  if(sum(TP<0)>0) stop('TP < 0')
-  if(sum(FP<0)>0) stop('FP < 0')
-  if(sum(TN<0)>0) stop('TN < 0')
-  if(sum(FN<0)>0) stop('FN < 0')
-  if(class(TP)!='numeric') stop('Incorrect TP class')
-  if(class(FP)!='numeric') stop('Incorrect FP class')
-  if(class(TN)!='numeric') stop('Incorrect TN class')
-  if(class(FN)!='numeric') stop('Incorrect FN class')
-  TN/(FN+TN)}
+  
+  checkToByTwoTableInputs(
+    TP = TP,
+    FP = FP,
+    FN = FN,
+    TN = TN
+  )
+  
+  return(TN/(FN+TN))
+}
 
 #' Calculate the falseOmissionRate
 #'
@@ -684,15 +729,16 @@ negativePredictiveValue <- function(TP,TN,FN,FP){
 #' falseOmissionRate value
 #'
 falseOmissionRate <- function(TP,TN,FN,FP){
-  if(sum(TP<0)>0) stop('TP < 0')
-  if(sum(FP<0)>0) stop('FP < 0')
-  if(sum(TN<0)>0) stop('TN < 0')
-  if(sum(FN<0)>0) stop('FN < 0')
-  if(class(TP)!='numeric') stop('Incorrect TP class')
-  if(class(FP)!='numeric') stop('Incorrect FP class')
-  if(class(TN)!='numeric') stop('Incorrect TN class')
-  if(class(FN)!='numeric') stop('Incorrect FN class')
-  FN/(FN+TN)}
+  
+  checkToByTwoTableInputs(
+    TP = TP,
+    FP = FP,
+    FN = FN,
+    TN = TN
+  )
+  
+  return(FN/(FN+TN))
+}
 
 #' Calculate the positiveLikelihoodRatio
 #'
@@ -708,15 +754,16 @@ falseOmissionRate <- function(TP,TN,FN,FP){
 #' positiveLikelihoodRatio value
 #' 
 positiveLikelihoodRatio <- function(TP,TN,FN,FP){
-  if(sum(TP<0)>0) stop('TP < 0')
-  if(sum(FP<0)>0) stop('FP < 0')
-  if(sum(TN<0)>0) stop('TN < 0')
-  if(sum(FN<0)>0) stop('FN < 0')
-  if(class(TP)!='numeric') stop('Incorrect TP class')
-  if(class(FP)!='numeric') stop('Incorrect FP class')
-  if(class(TN)!='numeric') stop('Incorrect TN class')
-  if(class(FN)!='numeric') stop('Incorrect FN class')
-  (TP/(TP+FN))/(FP/(FP+TN))}
+  
+  checkToByTwoTableInputs(
+    TP = TP,
+    FP = FP,
+    FN = FN,
+    TN = TN
+  )
+  
+  return((TP/(TP+FN))/(FP/(FP+TN)))
+}
 
 #' Calculate the negativeLikelihoodRatio
 #'
@@ -732,15 +779,16 @@ positiveLikelihoodRatio <- function(TP,TN,FN,FP){
 #' negativeLikelihoodRatio value
 #'
 negativeLikelihoodRatio <- function(TP,TN,FN,FP){
-  if(sum(TP<0)>0) stop('TP < 0')
-  if(sum(FP<0)>0) stop('FP < 0')
-  if(sum(TN<0)>0) stop('TN < 0')
-  if(sum(FN<0)>0) stop('FN < 0')
-  if(class(TP)!='numeric') stop('Incorrect TP class')
-  if(class(FP)!='numeric') stop('Incorrect FP class')
-  if(class(TN)!='numeric') stop('Incorrect TN class')
-  if(class(FN)!='numeric') stop('Incorrect FN class')
-  (FN/(TP+FN))/(TN/(FP+TN))}
+  
+  checkToByTwoTableInputs(
+    TP = TP,
+    FP = FP,
+    FN = FN,
+    TN = TN
+  )
+  
+  return((FN/(TP+FN))/(TN/(FP+TN)))
+}
 
 
 #' Calculate the diagnostic odds ratio
@@ -757,12 +805,13 @@ negativeLikelihoodRatio <- function(TP,TN,FN,FP){
 #' diagnosticOddsRatio value
 #'
 diagnosticOddsRatio <- function(TP,TN,FN,FP){
-  if(sum(TP<0)>0) stop('TP < 0')
-  if(sum(FP<0)>0) stop('FP < 0')
-  if(sum(TN<0)>0) stop('TN < 0')
-  if(sum(FN<0)>0) stop('FN < 0')
-  if(class(TP)!='numeric') stop('Incorrect TP class')
-  if(class(FP)!='numeric') stop('Incorrect FP class')
-  if(class(TN)!='numeric') stop('Incorrect TN class')
-  if(class(FN)!='numeric') stop('Incorrect FN class')
-  ((TP/(TP+FN))/(FP/(FP+TN)))/((FN/(TP+FN))/(TN/(FP+TN)))}
+  
+  checkToByTwoTableInputs(
+    TP = TP,
+    FP = FP,
+    FN = FN,
+    TN = TN
+  )
+  
+  return(((TP/(TP+FN))/(FP/(FP+TN)))/((FN/(TP+FN))/(TN/(FP+TN))))
+}
diff --git a/R/ViewShinyPlp.R b/R/ViewShinyPlp.R
index df613ceca..65d60b8b0 100644
--- a/R/ViewShinyPlp.R
+++ b/R/ViewShinyPlp.R
@@ -7,11 +7,28 @@
 #' 
 #' @export
 viewMultiplePlp <- function(analysesLocation){
-  viewPlps(result = analysesLocation, 
-           validation=NULL, 
-           useDatabase = F,
-           usePlpObject = F,
-           useFileSystem = T)
+  
+  if(!file.exists(file.path(analysesLocation, 'sqlite', 'databaseFile.sqlite'))){
+    stop('No database found')
+  }
+  
+  connectionDetailSettings <- list(
+    dbms = 'sqlite',
+    server = file.path(analysesLocation, 'sqlite', 'databaseFile.sqlite')
+  )
+  
+  databaseSettings <- list(
+    connectionDetailSettings = connectionDetailSettings, 
+    schema = 'main',
+    tablePrefix = '',
+    dbms = 'sqlite',
+    server = file.path(analysesLocation, 'sqlite', 'databaseFile.sqlite'),
+    user = NULL, 
+    password = NULL,
+    port = NULL
+  )
+  
+  viewPlps(databaseSettings)
 }
 
 #' viewPlp - Interactively view the performance and model settings
@@ -22,17 +39,37 @@ viewMultiplePlp <- function(analysesLocation){
 #' Either the result of runPlp and view the plots
 #' @param runPlp             The output of runPlp() (an object of class 'runPlp')
 #' @param validatePlp  The output of externalValidatePlp (on object of class 'validatePlp')
+#' @param diagnosePlp  The output of diagnosePlp()
 #' @return
 #' Opens a shiny app for interactively viewing the results
 #'
 #' @export
+viewPlp <- function(runPlp, validatePlp = NULL, diagnosePlp = NULL) {
+  
+  server <- insertRunPlpToSqlite(
+    runPlp = runPlp, 
+    externalValidatePlp = validatePlp,
+    diagnosePlp = diagnosePlp
+    )
+  
+  connectionDetailSettings <- list(
+    dbms = 'sqlite',
+    server = server
+  )
+  
+  databaseSettings <- list(
+    connectionDetailSettings = connectionDetailSettings, 
+    schema = 'main',
+    tablePrefix = '',
+    dbms = 'sqlite',
+    server = server,
+    user = NULL, 
+    password = NULL,
+    port = NULL
+  )
+  
+  viewPlps(databaseSettings)
 
-viewPlp <- function(runPlp, validatePlp = NULL) {
-  viewPlps(result = runPlp, 
-           validation=validatePlp, 
-           useDatabase = F,
-           usePlpObject = T,
-           useFileSystem = F)
 }
 
 
@@ -50,26 +87,37 @@ viewPlp <- function(runPlp, validatePlp = NULL) {
 #' @param myTableAppend A string appended to the results tables (optional)
 #' 
 #' @export
-viewDatabaseResultPlp <- function(mySchema, myServer, myUser, myPassword, myDbms, myPort = NULL, myTableAppend){
-  
-  ensure_installed('pool')
-  ensure_installed('DBI')
-  
-  Sys.setenv("shinydbSchema" = mySchema)
-  Sys.setenv("shinydbServer" = myServer)
-  Sys.setenv("shinydbUser" = myUser)
-  Sys.setenv("shinydbPw" = myPassword)
-  Sys.setenv("shinydbDbms" = myDbms)
-  if(!is.null(myPort)){
-    Sys.setenv("shinydbPort" = myPort)
-  }
-  Sys.setenv("shinydbTableAppend" = myTableAppend)
+viewDatabaseResultPlp <- function(
+  mySchema, 
+  myServer, 
+  myUser, 
+  myPassword, 
+  myDbms, 
+  myPort = NULL, 
+  myTableAppend
+  ){
+  
+  connectionDetailSettings <- list(
+    dbms = myDbms,
+    server =  myServer, 
+    user = myUser, 
+    password = myPassword, 
+    port = myPort
+  )
+  
+  databaseSettings <- list(
+    connectionDetailSettings = connectionDetailSettings, 
+    schema = mySchema,
+    tablePrefix = myTableAppend,
+    dbms = myDbms,
+    server = myServer,
+    user = myUser, 
+    password = myPassword,
+    port = myPort
+  )
+  
+  viewPlps(databaseSettings)
   
-  viewPlps(result = NULL, 
-           validation=NULL, 
-           useDatabase = T,
-           usePlpObject = F,
-           useFileSystem = F)
 }
 
 
@@ -77,49 +125,56 @@ viewDatabaseResultPlp <- function(mySchema, myServer, myUser, myPassword, myDbms
 # code for multiple and single together
 # one shiny app 
 
-viewPlps <- function(result, 
-                     validation=NULL, 
-                     useDatabase = NULL, 
-                     usePlpObject = NULL,
-                     useFileSystem = NULL){
-  ensure_installed("shiny")
-  ensure_installed("shinydashboard")
-  ensure_installed("shinycssloaders")
-  ensure_installed("DT")
-  ensure_installed("htmlwidgets")
-  ensure_installed("shinyWidgets")
-  ensure_installed("plotly")
- 
-  appDir <- system.file("shiny", "PLPViewer", package = "PatientLevelPrediction")
-  shinySettings <- list(result = result, 
-                        validation = validation, 
-                        useDatabase = useDatabase,
-                        usePlpObject = usePlpObject,
-                        useFileSystem = useFileSystem)
-  .GlobalEnv$shinySettings <- shinySettings
-  on.exit(rm(shinySettings, envir = .GlobalEnv))
-  shiny::runApp(appDir) 
-}
-
-
-
-#' Launch the Diagnostics Explorer Shiny app
-#'
-#' @param dataFolder       A folder where the exported zip files with the results are stored.  
-#'                         Zip files containing results from multiple databases can be placed in the same
-#'                         folder.
-#' @param launch.browser   Should the app be launched in your default browser, or in a Shiny window.
-#'                         Note: copying to clipboard will not work in a Shiny window.
-#'
-#' @details
-#' Launches a Shiny app that allows the user to explore the diagnostics
-#'
-#' @export
-launchDiagnosticsExplorer <- function(dataFolder, launch.browser = FALSE) {
-  ensure_installed("DT")
-  appDir <- system.file("shiny", "DiagnosticsExplorer", package = "PatientLevelPrediction")
-  shinySettings <- list(dataFolder = dataFolder)
-  .GlobalEnv$shinySettings <- shinySettings
-  on.exit(rm(shinySettings, envir = .GlobalEnv))
-  shiny::runApp(appDir)
-}
+viewPlps <- function(databaseSettings){
+  ensure_installed("ShinyAppBuilder") 
+  ensure_installed("ResultModelManager")
+  
+  connectionDetails <- do.call(
+    DatabaseConnector::createConnectionDetails, 
+    databaseSettings$connectionDetailSettings
+    )
+  connection <- ResultModelManager::ConnectionHandler$new(connectionDetails)
+  databaseSettings$connectionDetailSettings <- NULL
+  
+  shinyAppVersion <- strsplit(x = as.character(utils::packageVersion('ShinyAppBuilder')), split = '\\.')[[1]]
+  
+  if((shinyAppVersion[1] <= 1 & shinyAppVersion[2] < 2)){
+    # Old code to be backwards compatable
+    config <- ParallelLogger::loadSettingsFromJson(
+      fileName = system.file(
+        'shinyConfig.json', 
+        package = "PatientLevelPrediction"
+      )
+    )
+    # set database settings into system variables
+  Sys.setenv("resultDatabaseDetails_prediction" = as.character(ParallelLogger::convertSettingsToJson(databaseSettings)))
+  ShinyAppBuilder::viewShiny(
+    config = config, 
+    connection = connection
+    )
+  } else{
+    ohdsiModulesVersion <- strsplit(x = as.character(utils::packageVersion('OhdsiShinyModules')), split = '\\.')[[1]]
+    if(paste0(ohdsiModulesVersion[1], ".", ohdsiModulesVersion[2])>= 1.2){
+      config <- ParallelLogger::loadSettingsFromJson(
+        fileName = system.file(
+          'shinyConfigUpdate.json', 
+          package = "PatientLevelPrediction"
+        )
+      )
+      databaseSettings$plpTablePrefix = databaseSettings$tablePrefix
+      databaseSettings$cgTablePrefix = databaseSettings$tablePrefix
+      databaseSettings$databaseTable = 'database_meta_table'
+      databaseSettings$databaseTablePrefix = databaseSettings$tablePrefix
+    ShinyAppBuilder::viewShiny(
+      config = config, 
+      connection = connection, 
+      resultDatabaseSettings = databaseSettings
+        )
+    } else{
+      ParallelLogger::logWarn('Need to update package OhdsiShinyModules')
+    }
+    
+  }
+  
+  
+}
\ No newline at end of file
diff --git a/R/uploadPlpDbResults.R b/R/uploadPlpDbResults.R
deleted file mode 100644
index 175e10848..000000000
--- a/R/uploadPlpDbResults.R
+++ /dev/null
@@ -1,2841 +0,0 @@
-# @file UploadPlpDbResults.R
-#
-# Copyright 2021 Observational Health Data Sciences and Informatics
-#
-# This file is part of PatientLevelPrediction
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#' Create the results tables to store PatientLevelPrediction models and results into a database
-#' @description
-#' This function executes a large set of SQL statements to create tables that can store models and results
-#'
-#' @details
-#' This function can be used to create (or delete) PatientLevelPrediction result tables
-#'
-#' @param conn                         A connection to a database created by using the
-#'                                     function \code{connect} in the
-#'                                     \code{DatabaseConnector} package.
-#' @param resultSchema                 The name of the database schema that the result tables will be created.
-#' @param targetDialect                The database management system being used
-#' @param deleteExistingTables         If true any existing tables matching the PatientLevelPrediction result tables names will be deleted
-#' @param createTables                 If true the PatientLevelPrediction result tables will be created
-#' @param stringAppendToTables         A string that appends to the PatientLevelPrediction result tables
-#' @param tempEmulationSchema          The temp schema used when the database management system is oracle
-#' @param testFile                     (used for testing) The location of an sql file with the table creation code
-#'
-#' @return
-#' Returns NULL but creates the required tables into the specified database schema.
-#' 
-#' @export
-createPlpResultTables <- function(conn, 
-                                  resultSchema, 
-                                  targetDialect = 'postgresql',
-                                  deleteExistingTables = T, 
-                                  createTables = T,
-                                  stringAppendToTables = '',
-                                  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"),
-                                  testFile = NULL){
-  
-  
-  if(deleteExistingTables){
-    ParallelLogger::logInfo('Deleting existing tables')
-    
-    tables <- c(
-      "CALIBRATION_SUMMARY", 
-      "COVARIATE_SUMMARY", 
-      "DEMOGRAPHIC_SUMMARY",
-      "EVALUATION_STATISTICS", 
-      "PREDICTION_DISTRIBUTION", 
-      "THRESHOLD_SUMMARY",
-      
-      "ATTRITION", #new 
-      
-      "DIAGNOSTICS", #new 
-      "RECALIBRATIONS", #new 
-      
-      "RESULTS", 
-      
-      "STUDY_MODELS",  
-      
-      "MODELS", 
-      
-      "MODEL_DESIGNS",  
-
-      "MODEL_SETTINGS", 
-      "COVARIATE_SETTINGS",
-      "POPULATION_SETTINGS", 
-      "FEATURE_ENGINEERING_SETTINGS", 
-      "SPLIT_SETTINGS", 
-      "PLP_DATA_SETTINGS", #new
-      "SAMPLE_SETTINGS", 
-      "TIDY_COVARIATES_SETTINGS", #new 
-      "TARS", 
-    
-      "STUDIES", 
-      "COHORTS",
-      "DATABASE_DETAILS", 
-      "RESEARCHERS" 
-      )
-    
-    if(stringAppendToTables != ''){
-      tables <- paste0(toupper(gsub('_','',gsub(' ','', stringAppendToTables))), '_', tables)
-    }
-    
-    alltables <- DatabaseConnector::getTableNames(connection = conn, 
-                                                  databaseSchema = resultSchema)
-    
-    for(tb in tables){
-      if(tb %in%alltables){
-        sql <- 'TRUNCATE TABLE @my_schema.@table'
-        sql <- SqlRender::render(sql, 
-                                 my_schema = resultSchema, 
-                                 table=tb)
-        sql <- SqlRender::translate(sql, targetDialect = targetDialect, 
-                                    tempEmulationSchema = tempEmulationSchema)
-        DatabaseConnector::executeSql(conn, sql)
-        
-        sql <- 'DROP TABLE @my_schema.@table'
-        sql <- SqlRender::render(sql, 
-                                 my_schema = resultSchema, 
-                                 table=tb)
-        sql <- SqlRender::translate(sql, targetDialect = targetDialect,
-                                    tempEmulationSchema = tempEmulationSchema)
-        DatabaseConnector::executeSql(conn, sql)
-      }
-      
-    }
-    
-  }
-  
-  if(createTables){
-    ParallelLogger::logInfo('Creating PLP results tables')
-    
-    if(stringAppendToTables != ''){
-      stringAppendToTables <- paste0(toupper(gsub('_','',gsub(' ','', stringAppendToTables))), '_')
-    }
-    
-    if(is.null(testFile)){
-      renderedSql <- SqlRender::loadRenderTranslateSql(sqlFilename = "PlpResultTables.sql",
-                                                       packageName = "PatientLevelPrediction",
-                                                       dbms = targetDialect,
-                                                       tempEmulationSchema = tempEmulationSchema,
-                                                       my_schema = resultSchema,
-                                                       string_to_append = stringAppendToTables
-      )
-    } else {
-      sql <- readChar(testFile, file.info(testFile)$size) 
-      renderedSql <- SqlRender::render(sql = sql[1],
-                                       my_schema = resultSchema,
-                                       string_to_append = stringAppendToTables)
-      renderedSql <- SqlRender::translate(sql = renderedSql,
-                                          targetDialect = targetDialect,
-                                          tempEmulationSchema = tempEmulationSchema)
-      
-    }
-    
-    DatabaseConnector::executeSql(conn, renderedSql)
-  }
-  
-}
-
-# could add cohortDatabaseSchema and cohortTable as inputs below, plus database table
-
-#' Populate the PatientLevelPrediction results tables
-#' @description
-#' This function formats and uploads results that have been generated via an ATLAS prediction package into a database
-#'
-#' @details
-#' This function can be used upload PatientLevelPrediction results into a database
-#'
-#' @param conn                         A connection to a database created by using the
-#'                                     function \code{connect} in the
-#'                                     \code{DatabaseConnector} package.
-#' @param resultSchema                 (string) The name of the database schema with the result tables.
-#' @param stringAppendToTables         (string) A string that appends to the PatientLevelPrediction result tables
-#' @param targetDialect                (string) The database management system being used
-#' @param tempEmulationSchema          (string) The temp schema used when the database management system is oracle
-#' @param packageName                  (string) The name of the ATLAS R package used to generate the results (this is used to extract cohort jsons)
-#' @param studyJsonList                (list) A list of lists per cohort with the cohort_name, cohort_id and cohort_json 
-#' @param studyName                    (string) A reference study name 
-#' @param studyDescription             (string) A description of the study
-#' @param researcherName               (string) Name of the researcher who developed the study
-#' @param researcherEmail              (string) Email of the researcher who developed the study
-#' @param researcherOrg                (string) Organisation of the researcher who developed the study
-#' @param databaseName                 (string) name of the database used to develop the model/s
-#' @param databaseAcronym              (string) acronym of the database used to develop the model/s
-#' @param databaseVersion              (int) Version of the database used to develop the model/s
-#' @param databaseDescription          (string) Description of the database used to develop the model/s
-#' @param databaseType                 (string) Type of the database used to develop the model/s (e.g., claims)
-#' @param valDatabases                 (list) A named list with details of the external validation databases.  Needs to contain: name, description, version, type.
-#' @param resultLocation               (string) location of directory where the main package results were saved
-#' @param resultPattern                (string) A string to match to select models of interest  
-#' @param validationLocation           (string) location of directory where the validation package results were saved
-#' @param addInternalValidation        (boolean) Whether the internval validation results should be uploaded
-#' @param addExternalValidation        (boolean) Whether the externval validation results should be uploaded
-#' @param gsubVal                      (string) Remove patterns from the result name
-#' @param removePattern                (string) Restrict to result names with this pattern
-#'    
-#' @return
-#' Returns NULL but uploads all the results in resultLocation to the PatientLevelPrediction result tables in resultSchema
-#' 
-#' @export
-populatePlpResultTables <- function(conn, 
-                                    resultSchema, 
-                                    stringAppendToTables = '',
-                                    targetDialect = 'postgresql',
-                                    tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"),
-                                    packageName,
-                                    studyJsonList,
-                                    studyName = '',
-                                    studyDescription = '',
-                                    researcherName = '',
-                                    researcherEmail = '',
-                                    researcherOrg = '',
-                                    databaseName = NULL,
-                                    databaseAcronym = NULL,
-                                    databaseVersion = 1,
-                                    databaseDescription = NULL,
-                                    databaseType = NULL,
-                                    valDatabases = list(ccae = list(name = 'CCAE', 
-                                                                    description = '',
-                                                                    version = 1,
-                                                                    type = 'US Claims')),
-                                    resultLocation = NULL,
-                                    resultPattern = '',
-                                    validationLocation = file.path(resultLocation,'Validation'),
-                                    addInternalValidation = T,
-                                    addExternalValidation = T,
-                                    gsubVal = NULL,
-                                    removePattern = NULL
-){
-
-  ensure_installed("jsonlite")
-  
-  # input checks
-  ##TODO
-  if(base::missing(packageName)){
-    if(base::missing(studyJsonList)){
-      stop('Need either packageName or studyJsonList')
-    }else{
-      if(is.null(studyJsonList)){
-        stop('studyJsonList needs to be non-null')
-      }
-      cohortType <- 'list'
-      jsonInput <- studyJsonList
-    }
-  } else{
-    if(is.null(packageName)){
-      stop('packageName needs to be non-null')
-    }
-    cohortType <- 'package'
-    jsonInput <- packageName
-  }
-  
-  if(stringAppendToTables != ''){
-    stringAppendToTables <- paste0(toupper(gsub('_','',gsub(' ','', stringAppendToTables))), '_')
-  }
-  
-  studyId <- addStudy(conn = conn, 
-                      resultSchema = resultSchema, 
-                      targetDialect = targetDialect,
-                      studyName = studyName,
-                      studyDescription = studyDescription,
-                      stringAppendToTables = stringAppendToTables,
-                      tempEmulationSchema = tempEmulationSchema)
-  
-  ParallelLogger::logInfo(paste0('studyId: ', studyId))
-  
-  researcherId <- addResearcher(conn = conn, 
-                                resultSchema = resultSchema, 
-                                targetDialect = targetDialect,
-                                researcherName = researcherName, 
-                                researcherEmail = researcherEmail,
-                                researcherOrg = researcherOrg, 
-                                stringAppendToTables = stringAppendToTables,
-                                tempEmulationSchema = tempEmulationSchema)
-  
-  ParallelLogger::logInfo(paste0('researcherId: ', researcherId))
-  
-  dbId <- addDatabase(conn = conn, 
-                      resultSchema = resultSchema, 
-                      targetDialect = targetDialect,
-                      databaseName = databaseName,
-                      databaseAcronym = databaseAcronym,
-                      databaseVersion = databaseVersion,
-                      databaseDescription = databaseDescription,
-                      databaseType = databaseType, 
-                      stringAppendToTables = stringAppendToTables,
-                      tempEmulationSchema = tempEmulationSchema)
-  ParallelLogger::logInfo(paste0('dbId: ', dbId))
-  
-  mdls <- dir(resultLocation, pattern = resultPattern)
-  removeMdls <- union(grep('.csv', mdls),grep('.txt', mdls))
-  if(length(removeMdls)>0){
-    mdls <- mdls[-removeMdls] 
-  }
-  
-  # remove pattern
-  if(!is.null(removePattern)){
-    mdls <- mdls[-grep(removePattern, mdls)]
-  }
-  
-  for(modelRes in mdls){
-    ParallelLogger::logInfo(paste0('Adding results for model @ ', modelRes))
-    
-    # TODO edit csv here
-    mdl <- tryCatch(
-      {PatientLevelPrediction::loadPlpResult(file.path(resultLocation, modelRes, 'plpResult'))}, 
-                    error = function(e){ParallelLogger::logInfo(e);return(NULL)}
-      )
-    
-    if(!is.null(mdl)){
-      
-      # add TAR
-      tarId <- addTar(conn = conn, 
-                      resultSchema = resultSchema, 
-                      targetDialect = targetDialect,
-                      startDay = mdl$model$settings$populationSettings$riskWindowStart, 
-                      startAnchor = mdl$model$settings$populationSettings$startAnchor,
-                      endDay = mdl$model$settings$populationSettings$riskWindowEnd,  
-                      endAnchor = mdl$model$settings$populationSettings$endAnchor, 
-                      stringAppendToTables = stringAppendToTables,
-                      tempEmulationSchema = tempEmulationSchema
-        )
-      ParallelLogger::logInfo(paste0('tarId: ', tarId))
-      
-      tId <- addCohort(conn = conn, 
-                       resultSchema = resultSchema, 
-                       targetDialect = targetDialect,
-                       jsonInput = jsonInput, type = cohortType,
-                       cohortId = mdl$model$trainDetails$cohortId, 
-                       stringAppendToTables = stringAppendToTables,
-                       tempEmulationSchema = tempEmulationSchema)
-      ParallelLogger::logInfo(paste0('tId: ', tId))
-      
-      oId <- addCohort(conn = conn, 
-                       resultSchema = resultSchema, 
-                       targetDialect = targetDialect,
-                       jsonInput = jsonInput, type = cohortType,
-                       cohortId = mdl$model$trainDetails$outcomeId, 
-                       stringAppendToTables = stringAppendToTables,
-                       tempEmulationSchema = tempEmulationSchema)
-      ParallelLogger::logInfo(paste0('oId: ', oId))
-      
-      popSetId <- addPopulationSetting(conn = conn, 
-                                       resultSchema = resultSchema, 
-                                       targetDialect = targetDialect,
-                                       json = mdl$model$settings$populationSettings, 
-                                       stringAppendToTables = stringAppendToTables,
-                                       tempEmulationSchema = tempEmulationSchema)
-      ParallelLogger::logInfo(paste0('popSetId: ', popSetId))
-      
-      covSetId <- addCovariateSetting(conn = conn, 
-                                      resultSchema = resultSchema, 
-                                      targetDialect = targetDialect,
-                                      json = mdl$model$settings$covariateSettings, 
-                                      stringAppendToTables = stringAppendToTables,
-                                      tempEmulationSchema = tempEmulationSchema)
-      ParallelLogger::logInfo(paste0('covSetId: ', covSetId))
-      
-      modSetId <- addModelSetting(conn = conn, 
-                                  resultSchema = resultSchema, 
-                                  targetDialect = targetDialect,
-                                  modelType = mdl$model$settings$modelSettings$model,
-                                  json = mdl$model$settings$modelSettings, 
-                                  stringAppendToTables = stringAppendToTables,
-                                  tempEmulationSchema = tempEmulationSchema)
-      ParallelLogger::logInfo(paste0('modSetId: ', modSetId))
-      
-      # NEW: add plp_data_settings
-      plpDataSetId <- addPlpDataSetting(
-        conn = conn, 
-        resultSchema = resultSchema, 
-        targetDialect = targetDialect,
-        json = mdl$model$settings$plpDataSettings, 
-        stringAppendToTables = stringAppendToTables,
-        tempEmulationSchema = tempEmulationSchema
-        )
-      ParallelLogger::logInfo(paste0('plpDataSetId: ', plpDataSetId))
-      
-      # NEW: add FE_settings
-      FESetId <- addFESetting(
-        conn = conn, 
-        resultSchema = resultSchema, 
-        targetDialect = targetDialect,
-        json = mdl$model$settings$featureEngineering, 
-        stringAppendToTables = stringAppendToTables,
-        tempEmulationSchema = tempEmulationSchema
-        )
-      ParallelLogger::logInfo(paste0('FESetId: ', FESetId))
-      
-      # NEW: add sample_settings
-      sampleSetId <- addSampleSetting(
-        conn = conn, 
-        resultSchema = resultSchema, 
-        targetDialect = targetDialect,
-        json = mdl$model$settings$sampleSettings, 
-        stringAppendToTables = stringAppendToTables,
-        tempEmulationSchema = tempEmulationSchema
-        )
-      ParallelLogger::logInfo(paste0('sampleSetId: ', sampleSetId)
-      )
-      
-      # NEW: add tidy_covariate_settings
-      tidySetId <- addTidySetting(
-        conn = conn, 
-        resultSchema = resultSchema, 
-        targetDialect = targetDialect,
-        json = mdl$model$settings$tidyCovariates, 
-        stringAppendToTables = stringAppendToTables,
-        tempEmulationSchema = tempEmulationSchema
-      )
-      ParallelLogger::logInfo(paste0('tidySetId: ', tidySetId))
-
-      
-      # this is now split setting - update this function
-      splitId <- addSplitSettings(
-        conn = conn, 
-        resultSchema = resultSchema, 
-        targetDialect = targetDialect,
-        json = mdl$model$settings$splitSettings,
-        stringAppendToTables = stringAppendToTables,
-        tempEmulationSchema = tempEmulationSchema
-      )
-      ParallelLogger::logInfo(paste0('splitId: ', splitId))
-      
-      # create this function
-      modelDesignId <- addModelDesign( # need to create
-        conn = conn, 
-        resultSchema = resultSchema, 
-        targetDialect = targetDialect,
-        targetId = tId,
-        outcomeId = oId,
-        tarId = tarId,
-        plpDataSettingId = plpDataSetId,
-        populationSettingId = popSetId,
-        modelSettingId = modSetId,
-        covariateSettingId = covSetId,
-        sampleSettingId = sampleSetId,
-        splitSettingId = splitId, # changed from trainingId
-        featureEngineeringSettingId = FESetId,
-        tidyCovariatesSettingId = tidySetId,
-        researcherId = researcherId,
-        stringAppendToTables = stringAppendToTables,
-        tempEmulationSchema = tempEmulationSchema
-      )
-      ParallelLogger::logInfo(paste0('modelDesignId: ', modelDesignId))
-      
-      # create this function
-      modelId <- addModel(
-        conn = conn, 
-        resultSchema = resultSchema, 
-        targetDialect = targetDialect,
-        analysisId = mdl$model$trainDetails$analysisId, # trainDetails
-        modelDesignId = modelDesignId,
-        researcherId = researcherId,
-        databaseId = dbId,
-        hyperParamSearch = mdl$model$trainDetails$hyperParamSearch, #mdl$trainDetails$hyperParamSearch
-        plpModelFile = " ",
-        executionDateTime = format(mdl$executionSummary$ExecutionDateTime, format="%Y-%m-%d"), #mdl$trainDetails$trainingDate
-        trainingTime = mdl$model$trainDetails$trainingTime, #mdl$trainDetails$trainingTime
-        intercept = ifelse(is.list(mdl$model), mdl$model$model$coefficients[1], 0),
-        requireDenseMatrix = mdl$model$settings$requireDenseMatrix,
-        stringAppendToTables = stringAppendToTables,
-        tempEmulationSchema = tempEmulationSchema
-      )
-      ParallelLogger::logInfo(paste0('modelId: ', modelId))
-      
-      # add modelId and studyId
-      addStudiesModel(conn = conn, 
-                      resultSchema = resultSchema, 
-                      targetDialect = targetDialect,
-                      studyId = studyId,
-                      modelId = modelId,
-                      stringAppendToTables = stringAppendToTables,
-                      tempEmulationSchema = tempEmulationSchema)
-      
-      # add internalValication
-      if(addInternalValidation){
-        
-        ParallelLogger::logInfo('Adding internal validation results')
-        
-        ##if exists 
-        if(!is.null(mdl)){
-          # add attrition here...
-          resultId <- addResult(conn = conn, 
-                                resultSchema = resultSchema, 
-                                targetDialect = targetDialect,
-                                modelId = modelId,
-                                researcherId = researcherId,
-                                databaseId = dbId,
-                                targetId = tId,
-                                outcomeId = oId,
-                                tarId = tarId,
-            restrictPlpDataSettingId = plpDataSetId,
-                                populationSettingId = popSetId,
-                                executionDateTime = format(mdl$executionSummary$ExecutionDateTime, format="%Y-%m-%d"),
-                                plpVersion = mdl$executionSummary$PackageVersion$packageVersion, 
-                                stringAppendToTables = stringAppendToTables,
-                                tempEmulationSchema = tempEmulationSchema)
-          ParallelLogger::logInfo(paste0('resultId: ', resultId))
-          
-          # add attriition
-          if(!is.null(mdl$model$trainDetails$attrition)){
-            addAttrition(
-              conn = conn, resultSchema = resultSchema, targetDialect = targetDialect,
-              resultId = resultId,
-              attrition = mdl$model$trainDetails$attrition,
-              overWriteIfExists = T, 
-              stringAppendToTables = stringAppendToTables,
-              tempEmulationSchema = tempEmulationSchema
-              )
-          }
-          
-          # add eval
-          if(!is.null(mdl$performanceEvaluation)){
-            addEvaluation(conn = conn, resultSchema = resultSchema, targetDialect = targetDialect,
-                          resultId = resultId,
-                          performanceEvaluation = mdl$performanceEvaluation,
-                          overWriteIfExists = T, 
-                          stringAppendToTables = stringAppendToTables,
-                          tempEmulationSchema = tempEmulationSchema)
-          }
-          if(!is.null(mdl$covariateSummary)){
-            addCovariateSummary(conn = conn, resultSchema = resultSchema, targetDialect = targetDialect,
-                                resultId = resultId,
-                                covariateSummary = mdl$covariateSummary,
-                                restrictToIncluded = T,
-                                overWriteIfExists = T, 
-                                stringAppendToTables = stringAppendToTables,
-                                tempEmulationSchema = tempEmulationSchema)
-          }
-          
-        }
-      }
-      
-      # add validation results for this model
-      if(addExternalValidation){
-        
-        ParallelLogger::logInfo('Adding external validation results')
-        
-        
-        if(is.null(validationLocation)){
-          validationLocation <- file.path(resultLocation, 'validation')
-        }
-        valDbs <- dir(validationLocation)
-        
-        # restrict to the databases with info
-        valDbs <- valDbs[valDbs%in%names(valDatabases)]
-        
-        if(length(valDbs)>0){
-          
-          valDbs <- valDbs[!valDbs %in% c('plplog.txt')]
-          
-          for(valDb in valDbs){
-            
-            #get valDbId
-            valDbId <- addDatabase(conn = conn, 
-                                   resultSchema = resultSchema, 
-                                   targetDialect = targetDialect,
-                                   databaseName = valDatabases[[valDb]]$name,
-                                   databaseAcronym = valDb,
-                                   databaseVersion = valDatabases[[valDb]]$version,
-                                   databaseDescription = valDatabases[[valDb]]$description,
-                                   databaseType =valDatabases[[valDb]]$type, 
-                                   stringAppendToTables = stringAppendToTables,
-                                   tempEmulationSchema = tempEmulationSchema)
-            
-            validationResults <- as.character(dir(file.path(validationLocation, valDb)))
-            validationResults <- validationResults[validationResults != 'CohortCounts.csv']
-            
-            valMods <- data.frame(validationResults = validationResults)
-            
-            if(!is.null(gsubVal)){
-              valModsEdit <- valMods$validationResults
-              for(i in 1:nrow(gsubVal)){
-                valModsEdit <- gsub(x = valModsEdit, pattern = gsubVal[i,1], replacement = gsubVal[i,2])
-              }
-              valMods$validationResultsEdit <- valModsEdit
-            }else{
-              valMods$validationResultsEdit <- valMods$validationResults
-            }
-            
-            # remove pattern
-            if(!is.null(removePattern)){
-              if(length(grep(removePattern, valMods$validationResultsEdit))>0){
-                valMods <- valMods[-grep(removePattern, valMods$validationResultsEdit),]
-              }
-            }
-            
-            # restrict to analysis
-            ParallelLogger::logInfo(paste0('restricting to ', modelRes))
-            valMods <- valMods[grep(modelRes, valMods$validationResultsEdit),]
-            
-            if(nrow(valMods)>0){
-              
-              # load each result
-              for(valInd in 1:nrow(valMods)){
-                
-              #resultName <- dir(file.path(validationLocation, valDb, valMods$validationResults[valInd]))
-              #resultName <- resultName[grep('.rds',resultName)]
-                ParallelLogger::logInfo(paste0('Loading validation at:', file.path(validationLocation, valDb, valMods$validationResults[valInd], 'validationResult' )))
-                vmdl <- tryCatch(
-                  {PatientLevelPrediction::loadPlpResult(file.path(validationLocation, valDb, valMods$validationResults[valInd], 'validationResult' ))},
-                  error = function(e){ParallelLogger::logInfo(e); return(NULL)}
-                )
-
-                
-                if(!is.null(vmdl)){
-                  tId <- addCohort(conn = conn, 
-                                   resultSchema = resultSchema, 
-                                   targetDialect = targetDialect,
-                                   jsonInput = jsonInput, type = cohortType,
-                                   #cohortId = vmdl$model$validationDetails$cohortId,
-                    cohortId = ifelse(
-                      !is.null(vmdl$model$validationDetails$cohortId),
-                      vmdl$model$validationDetails$cohortId,
-                      vmdl$prediction$cohortId[1]
-                      ), 
-                                   stringAppendToTables = stringAppendToTables,
-                                   tempEmulationSchema = tempEmulationSchema)
-                  oId <- addCohort(conn = conn, 
-                                   resultSchema = resultSchema, 
-                                   targetDialect = targetDialect,
-                                   jsonInput = jsonInput, type = cohortType,
-                                   cohortId = vmdl$model$validationDetails$outcomeId, 
-                                   stringAppendToTables = stringAppendToTables,
-                                   tempEmulationSchema = tempEmulationSchema)
-                  
-                  # get tarId (added)
-                  tarId <- addTar(
-                    conn = conn, 
-                    resultSchema = resultSchema, 
-                    targetDialect = targetDialect,
-                    startDay = vmdl$model$validationDetails$populationSettings$riskWindowStart, 
-                    startAnchor = vmdl$model$validationDetails$populationSettings$startAnchor,
-                    endDay = vmdl$model$validationDetails$populationSettings$riskWindowEnd,  
-                    endAnchor = vmdl$model$validationDetails$populationSettings$endAnchor, 
-                    stringAppendToTables = stringAppendToTables,
-                    tempEmulationSchema = tempEmulationSchema
-                  )
-                  
-                  # popSetId (added)
-                  popSetId <- addPopulationSetting(
-                    conn = conn, 
-                    resultSchema = resultSchema, 
-                    targetDialect = targetDialect,
-                    json = vmdl$model$validationDetails$populationSettings, 
-                    stringAppendToTables = stringAppendToTables,
-                    tempEmulationSchema = tempEmulationSchema
-                    )
-                  
-
-                  plpDataSetId <- addPlpDataSetting(
-                    conn = conn, 
-                    resultSchema = resultSchema, 
-                    targetDialect = targetDialect, 
-                    stringAppendToTables = stringAppendToTables, 
-                    tempEmulationSchema = tempEmulationSchema, 
-                    json = ifelse(
-                      !is.null(vmdl$model$validationDetails$plpDataSettings),
-                      vmdl$model$validationDetails$plpDataSettings,
-                      vmdl$model$settings$plpDataSettings
-                      ) 
-                    )
-                  
-                  # add result
-                  resultId <- addResult(conn = conn, 
-                                        resultSchema = resultSchema, 
-                                        targetDialect = targetDialect,
-                                        modelId = modelId,
-                                        researcherId = researcherId,
-                                        databaseId = valDbId,
-                                        targetId = tId,
-                                        outcomeId = oId,
-                                        tarId = tarId,
-                    restrictPlpDataSettingId = plpDataSetId,
-                                        populationSettingId = popSetId,
-                                        executionDateTime = format(vmdl$executionSummary$ExecutionDateTime, format="%Y-%m-%d"),
-                                        plpVersion = vmdl$executionSummary$PackageVersion$packageVersion, 
-                                        stringAppendToTables = stringAppendToTables,
-                                        tempEmulationSchema = tempEmulationSchema)
-                  
-                  
-                  
-                  # add attrition
-                  if(!is.null(vmdl$model$validationDetails$attrition)){
-                    addAttrition(
-                      conn = conn, resultSchema = resultSchema, targetDialect = targetDialect,
-                      resultId = resultId,
-                      attrition = vmdl$model$validationDetails$attrition,
-                      overWriteIfExists = T, 
-                      stringAppendToTables = stringAppendToTables,
-                      tempEmulationSchema = tempEmulationSchema
-                    )
-                  }
-                  
-                  # add performance 
-                  #=============
-                  if(!is.null(vmdl$performanceEvaluation)){
-                    addEvaluation(conn = conn, resultSchema = resultSchema, targetDialect = targetDialect,
-                                  resultId = resultId,
-                                  performanceEvaluation = vmdl$performanceEvaluation,
-                                  overWriteIfExists = T, 
-                                  stringAppendToTables = stringAppendToTables,
-                                  tempEmulationSchema = tempEmulationSchema)
-                  }
-                  
-                  if(!is.null(vmdl$covariateSummary)){
-                    addCovariateSummary(conn = conn, resultSchema = resultSchema, targetDialect = targetDialect,
-                                        resultId = resultId,
-                                        covariateSummary = vmdl$covariateSummary,
-                                        restrictToIncluded = T,
-                                        overWriteIfExists = T, 
-                                        stringAppendToTables = stringAppendToTables,
-                                        tempEmulationSchema = tempEmulationSchema)
-                  }
-                  
-                }
-                
-                #+++++++++++++
-                
-                
-              } # end val per database
-            } # end if val exists
-            
-          } # val database
-        }
-        
-      } #externalVal
-      
-      
-    } #model not null 
-    
-  } # per model
-  
-} #end funct
-
-
-#======================
-# HELPER FUNCTIONS
-#======================
-enc <- function(x){
-  return(paste0("'", x, "'"))
-}
-
-cleanNum <- function(x){
-  types <- unlist(lapply(1:ncol(x), function(i) class(x[,i])))
-  
-  ids <- which(types%in% c("numeric", "integer" ))
-  
-  for(id in ids){
-    okVals <- is.finite(x[,id])
-    
-    if(sum(okVals)!=length(okVals)){
-      x[!okVals,id] <- NA
-    }
-    
-  }
-  return(x)  
-}
-
-checkTable <- function(conn,
-                       resultSchema, 
-                       stringAppendToTables = '',
-                       targetDialect,
-                       tableName,
-                       columnNames, 
-                       values,
-                       tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  vals <- paste0(paste0(columnNames," = ", values), collapse = " and ")
-  
-  sql <- "SELECT * from @my_schema.@string_to_append@table where @input_vals;"
-  sql <- SqlRender::render(sql, 
-                           my_schema = resultSchema,
-                           table = tableName,
-                           input_vals = vals,
-                           string_to_append = stringAppendToTables)
-  sql <- SqlRender::translate(sql, targetDialect = targetDialect,
-                              tempEmulationSchema = tempEmulationSchema)
-  result <- DatabaseConnector::querySql(conn, sql, snakeCaseToCamelCase = T)
-  
-  return(result)
-}
-
-
-checkJson <- function(conn,
-                      resultSchema, 
-                      stringAppendToTables = '',
-                      targetDialect,
-                      tableName,
-                      jsonColumnName,
-                      id,
-                      json,
-                      tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  sql <- "SELECT * from @my_schema.@string_to_append@table;"
-  sql <- SqlRender::render(sql, 
-                           my_schema = resultSchema,
-                           table = tableName,
-                           string_to_append = stringAppendToTables)
-  sql <- SqlRender::translate(sql, targetDialect = targetDialect,
-                              tempEmulationSchema = tempEmulationSchema)
-  result <- DatabaseConnector::querySql(conn, sql, snakeCaseToCamelCase = T)
-  
-  resultId <- NULL
-  if(nrow(result)>0){
-    colId <- result[,jsonColumnName] == json
-    if(sum(colId)>0){
-      resultId <- result[colId,id][1]
-    }
-  }
-  
-  return(resultId)
-}
-
-# gets the column names in camelCase of a table
-getColumnNames <- function(conn, resultSchema, targetDialect, tableName, stringAppendToTables = '',
-                           tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  sql <- "select top 1 * from @my_schema.@string_to_append@table;"
-  sql <- SqlRender::render(sql, 
-                           my_schema = resultSchema,
-                           table = tableName,
-                           string_to_append = stringAppendToTables)
-  sql <- SqlRender::translate(sql, targetDialect = targetDialect,
-                              tempEmulationSchema = tempEmulationSchema)
-  result <- DatabaseConnector::querySql(connection = conn, sql = sql, snakeCaseToCamelCase = T)
-  
-  return(colnames(result))
-}
-
-# True/False check whether results exist in table 
-checkResultExists <- function(conn, resultSchema, targetDialect,
-                              snakeCaseToCamelCase,
-                              tableName,
-                              resultId,
-                              tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  sql <- "select * from @my_schema.@table where result_id = @result_id;"
-  sql <- SqlRender::render(sql, 
-                           my_schema = resultSchema,
-                           table = tableName,
-                           result_id = resultId)
-  sql <- SqlRender::translate(sql, targetDialect = targetDialect,
-                              tempEmulationSchema = tempEmulationSchema)
-  result <- DatabaseConnector::querySql(connection = conn, sql = sql, snakeCaseToCamelCase = T)
-  return(nrow(result)>0)
-}
-
-
-#======================
-# end helpers
-addStudy <- function(conn, resultSchema, targetDialect,
-                     studyName, studyDescription,
-                     stringAppendToTables = '',
-                     tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  result <- checkTable(conn = conn, 
-                       resultSchema = resultSchema, 
-                       stringAppendToTables = stringAppendToTables,
-                       targetDialect = targetDialect, 
-                       tableName = 'studies',
-                       columnNames = c('study_name', 'study_description'), 
-                       values = c(paste0("'",studyName,"'"), 
-                                  paste0("'",studyDescription,"'")
-                       ),
-                       tempEmulationSchema = tempEmulationSchema
-  )
-  
-  if(nrow(result)>0){
-    ParallelLogger::logInfo('Study already exists')
-  }
-  else{
-    ParallelLogger::logInfo(paste0('Adding new study: ', studyName ))
-    
-    # add my detail
-    sql <- "INSERT INTO @my_schema.@string_to_appendstudies(study_name, study_description) 
-          VALUES ('@name','@desc');"
-    sql <- SqlRender::render(sql, 
-                             my_schema = resultSchema,
-                             name = studyName,
-                             desc = studyDescription,
-                             string_to_append = stringAppendToTables
-    )
-    
-    sql <- SqlRender::translate(sql, targetDialect = targetDialect, 
-                                tempEmulationSchema = tempEmulationSchema)
-    
-    DatabaseConnector::executeSql(conn, sql)
-    
-    result <- checkTable(conn = conn, 
-                         resultSchema = resultSchema, 
-                         stringAppendToTables = stringAppendToTables,
-                         targetDialect = targetDialect, 
-                         tableName = 'studies',
-                         columnNames = c('study_name', 'study_description'), 
-                         values = c(paste0("'",studyName,"'"), 
-                                    paste0("'",studyDescription,"'")
-                         ),
-                         tempEmulationSchema = tempEmulationSchema
-    )
-    
-  }
-  
-  return(result$studyId[1])
-  
-}
-
-addStudiesModel <- function(conn, 
-                            resultSchema, 
-                            targetDialect,
-                            studyId,
-                            modelId,
-                            stringAppendToTables,
-                            tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  
-  result <- checkTable(conn = conn, 
-                       resultSchema = resultSchema, 
-                       stringAppendToTables = stringAppendToTables,
-                       targetDialect = targetDialect, 
-                       tableName = 'study_models',
-                       columnNames = c('study_id', 'model_id'), 
-                       values = c(studyId, modelId),
-                       tempEmulationSchema = tempEmulationSchema
-  )
-  
-  if(nrow(result)>0){
-    ParallelLogger::logInfo('Study and model already linked')
-  }
-  else{
-    ParallelLogger::logInfo(paste0('Adding link between study: ', studyId, ' and model: ', modelId ))
-    
-    # add my detail
-    sql <- "INSERT INTO @my_schema.@string_to_appendstudy_models(study_id, model_id) 
-          VALUES ('@studyid','@modelid');"
-    sql <- SqlRender::render(sql, 
-                             my_schema = resultSchema,
-                             studyid = studyId,
-                             modelid = modelId,
-                             string_to_append = stringAppendToTables
-    )
-    
-    sql <- SqlRender::translate(sql = sql, targetDialect = targetDialect,
-                                tempEmulationSchema = tempEmulationSchema)
-    
-    DatabaseConnector::executeSql(conn, sql)
-    
-    result <- checkTable(conn = conn, 
-                         resultSchema = resultSchema, 
-                         stringAppendToTables = stringAppendToTables,
-                         targetDialect = targetDialect, 
-                         tableName = 'study_models',
-                         columnNames = c('study_id', 'model_id'), 
-                         values = c(studyId, modelId),
-                         tempEmulationSchema = tempEmulationSchema
-    )
-    
-  }
-  
-  return(invisible(result$studyId[1]))
-  
-}
-
-
-addResearcher <- function(conn, resultSchema, targetDialect,
-                          stringAppendToTables = '',
-                          researcherName, 
-                          researcherEmail,
-                          researcherOrg,
-                          tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  result <- checkTable(conn = conn, 
-                       resultSchema = resultSchema, 
-                       stringAppendToTables = stringAppendToTables,
-                       targetDialect = targetDialect, 
-                       tableName = 'researchers',
-                       columnNames = c('researcher_name', 'researcher_email', 'researcher_affiliation'), 
-                       values = c(paste0("'",researcherName,"'"), 
-                                  paste0("'",researcherEmail,"'"),
-                                  paste0("'",researcherOrg,"'")),
-                       tempEmulationSchema = tempEmulationSchema
-  )
-  
-  if(nrow(result)>0){
-    ParallelLogger::logInfo('Researcher already exists')
-  }
-  else{
-    ParallelLogger::logInfo(paste0('Adding Researcher: ', researcherName ))
-    
-    # add my detail
-    sql <- "INSERT INTO @my_schema.@string_to_appendresearchers(researcher_name, researcher_email, researcher_affiliation) 
-          VALUES ('@name','@email', '@org');"
-    sql <- SqlRender::render(sql, 
-                             my_schema = resultSchema,
-                             name = researcherName,
-                             email = researcherEmail,
-                             org = researcherOrg,
-                             string_to_append = stringAppendToTables
-    )
-    
-    sql <- SqlRender::translate(sql = sql, targetDialect = targetDialect,
-                                tempEmulationSchema = tempEmulationSchema)
-    
-    DatabaseConnector::executeSql(conn, sql)
-    
-    result <- checkTable(conn = conn, 
-                         resultSchema = resultSchema, 
-                         stringAppendToTables = stringAppendToTables,
-                         targetDialect = targetDialect, 
-                         tableName = 'researchers',
-                         columnNames = c('researcher_name', 'researcher_email', 'researcher_affiliation'), 
-                         values = c(paste0("'",researcherName,"'"), 
-                                    paste0("'",researcherEmail,"'"),
-                                    paste0("'",researcherOrg,"'")),
-                         tempEmulationSchema = tempEmulationSchema
-    )
-    
-  }
-  
-  return(result$researcherId[1])
-  
-}
-
-
-addDatabase <- function(conn, resultSchema, targetDialect,
-                        stringAppendToTables = '',
-                        databaseName,
-                        databaseAcronym,
-                        databaseVersion = 1,
-                        databaseDescription,
-                        databaseType,
-                        tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  result <- checkTable(conn = conn, 
-                       resultSchema = resultSchema, 
-                       stringAppendToTables = stringAppendToTables,
-                       targetDialect = targetDialect, 
-                       tableName = 'database_details',
-                       columnNames = c('database_name', 'database_acronym',
-                                       'database_version',
-                                       'database_description', 'database_type'), 
-                       values = c(paste0("'",databaseName,"'"), 
-                                  paste0("'",databaseAcronym,"'"),
-                                  databaseVersion,
-                                  paste0("'",databaseDescription,"'"),
-                                  paste0("'",databaseType,"'")),
-                       tempEmulationSchema = tempEmulationSchema
-  )
-  
-  if(nrow(result)>0){
-    ParallelLogger::logInfo(paste0('Database ', databaseName ,' already exists'))
-  } else {
-    
-    sql <- "INSERT INTO @my_schema.@string_to_appenddatabase_details(database_name, database_acronym,
-                                  database_version,
-                                  database_description, database_type) 
-          VALUES ('@dbName','@db', @version, '@desc', '@type');"
-    sql <- SqlRender::render(sql, 
-                             my_schema = resultSchema,
-                             dbName = databaseName, 
-                             db = databaseAcronym,
-                             version = databaseVersion,
-                             desc = databaseDescription,
-                             type = databaseType,
-                             string_to_append = stringAppendToTables)
-    sql <- SqlRender::translate(sql, targetDialect = targetDialect,
-                                tempEmulationSchema = tempEmulationSchema)
-    DatabaseConnector::executeSql(conn, sql)
-    
-    result <- checkTable(conn = conn, 
-                         resultSchema = resultSchema, 
-                         stringAppendToTables = stringAppendToTables,
-                         targetDialect = targetDialect, 
-                         tableName = 'database_details',
-                         columnNames = c('database_name', 'database_acronym', 'database_version',
-                                         'database_description', 'database_type'), 
-                         values = c(paste0("'",databaseName,"'"), 
-                                    paste0("'",databaseAcronym,"'"),
-                                    databaseVersion,
-                                    paste0("'",databaseDescription,"'"),
-                                    paste0("'",databaseType,"'")),
-                         tempEmulationSchema = tempEmulationSchema
-    )
-    
-  }
-  
-  return(result$databaseId[1])
-  
-}
-
-
-addTar <- function(conn, resultSchema, targetDialect,
-                   stringAppendToTables = '',
-                   startDay, 
-                   startAnchor,
-                   endDay,  
-                   endAnchor,
-                   tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  result <- checkTable(conn = conn, 
-                       resultSchema = resultSchema, 
-                       stringAppendToTables = stringAppendToTables,
-                       targetDialect = targetDialect, 
-                       tableName = 'tars',
-                       columnNames = c('tar_start_day', 'tar_start_anchor',
-                                       'tar_end_day', 'tar_end_anchor'), 
-                       values = c(startDay, 
-                                  paste0("'",startAnchor,"'"),
-                                  endDay,
-                                  paste0("'",endAnchor,"'")),
-                       tempEmulationSchema = tempEmulationSchema
-  )
-  
-  if(nrow(result)==0){
-    
-    ParallelLogger::logInfo('Adding TAR')
-    # tars - id 1
-    sql <- "INSERT INTO @my_schema.@string_to_appendtars(tar_start_day, tar_start_anchor,
-                           tar_end_day, tar_end_anchor) 
-          VALUES (@tar_start_day, @tar_start_anchor, @tar_end_day, @tar_end_anchor);"
-    sql <- SqlRender::render(sql, 
-                             my_schema = resultSchema,
-                             tar_start_day = startDay,
-                             tar_start_anchor = paste0("'",startAnchor,"'"),
-                             tar_end_day = endDay,
-                             tar_end_anchor = paste0("'",endAnchor,"'"),
-                             string_to_append = stringAppendToTables)
-    
-    sql <- SqlRender::translate(sql, targetDialect = targetDialect, 
-                                tempEmulationSchema = tempEmulationSchema)
-    
-    DatabaseConnector::executeSql(conn, sql)
-    
-    #getId of new
-    result <- checkTable(conn = conn, 
-                         resultSchema = resultSchema, 
-                         stringAppendToTables = stringAppendToTables,
-                         targetDialect = targetDialect, 
-                         tableName = 'tars',
-                         columnNames = c('tar_start_day', 'tar_start_anchor',
-                                         'tar_end_day', 'tar_end_anchor'), 
-                         values = c(startDay, 
-                                    paste0("'",startAnchor,"'"),
-                                    endDay,
-                                    paste0("'",endAnchor,"'")),
-                         tempEmulationSchema = tempEmulationSchema
-    )
-    
-  } else {
-    ParallelLogger::logInfo('TAR exists')
-  }
-  
-  
-  return(result$tarId[1])
-  
-}
-
-
-
-
-
-getCohortFromList <- function(jsonList, cohortId){
-  
-  #cohort_name, cohort_id and cohort_json
-  ParallelLogger::logInfo(paste0('Adding cohorts from input list'))
-  id <- which(unlist(lapply(jsonList, function(x){x$cohort_id == cohortId})))[1]
-  
-  json <- jsonList[[id]]$cohort_json
-  
-  details <- data.frame(
-    cohortName = jsonList[[id]]$cohort_name,
-    cohortId = jsonList[[id]]$cohort_id,
-    webApiCohortId = jsonList[[id]]$cohort_id
-  )
-  
-  return(list(json = json,
-              cohortTocreate = details))
-}
-
-# this can be simplified now we use cohort id as the json file name:
-getCohortFromPackage <- function(packageName, cohortId){
-  
-  ParallelLogger::logInfo(paste0('Adding cohorts from ', packageName))
-  # check packageName
-  if(!dir.exists(system.file(package = packageName))){
-    stop('Package path not found - set pckPath input to the location of the study package you executed')
-  } else {
-    
-    ParallelLogger::logInfo(paste0('Extracting cohort ',cohortId,' json from ', packageName))
-    # check required files:
-    cohortToCreateLoc <- system.file('Cohorts.csv', # updated for new skeleton
-                                     package = packageName)
-    
-    if(!file.exists(cohortToCreateLoc)){
-      stop('No Cohorts.csv in package')
-    }
-    
-    if(!dir.exists(file.path(system.file(package = packageName), 'cohorts'))){
-      stop('No cohorts in package')
-    }
-  }
-  
-  
-  # add the cohorts and store the map atlas_id, cohort_id, cohort_name
-  cohortsToCreate <- utils::read.csv(cohortToCreateLoc)
-  cohortTocreate <- cohortsToCreate[cohortsToCreate$cohortId == cohortId,]
-  
-  jsonFileName <- file.path(system.file(package = packageName), 'cohorts', paste0(cohortTocreate$cohortId, '.json'))
-  json <- readChar(jsonFileName, file.info(jsonFileName)$size)
-  
-  
-  return(list(json = json,
-              cohortTocreate = cohortTocreate))
-}
-
-
-# adds json from package unless json is specified
-addCohort <- function(conn, resultSchema, targetDialect,
-                      stringAppendToTables = '',
-                      jsonInput, type = 'package',
-                      cohortId,
-                      tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  if(type == 'package'){
-    object <- getCohortFromPackage(packageName = jsonInput, cohortId)
-  } else{
-    object <- getCohortFromList(jsonList = jsonInput, cohortId)
-  }
-  
-  json <- object$json
-  cohortTocreate <- object$cohortTocreate
-  
-    # make sure the json has been converted 
-    if(class(json)!='character'){
-      ParallelLogger::logInfo('converting json to character')
-      json <- jsonlite::serializeJSON(json, digits = 23)
-    }
-    
-    # reduce the size to save
-    json <-  substr(json, 1, 4000) # TESTING - FIX THIS [TODO]
-    
-    #check whether cohort already in table:
-    result <- checkTable(conn = conn, 
-                         resultSchema = resultSchema, 
-                         stringAppendToTables = stringAppendToTables,
-                         targetDialect = targetDialect, 
-                         tableName = 'cohorts',
-                         columnNames = c('cohort_name', 'atlas_id'), 
-                         values = c(paste0("'",cohortTocreate$cohortName[1],"'"), cohortTocreate$cohortId[1]),
-                         tempEmulationSchema = tempEmulationSchema
-    )
-    
-    addNew <- F
-    if(nrow(result)>0){
-      addNew <- json %in% result$cohortJson
-      ParallelLogger::logInfo(paste0('json in jsons:', addNew))
-    }
-    
-    if(addNew){
-      ParallelLogger::logInfo(paste0('Cohort ',cohortTocreate$cohortName,' exists in result database with id', result$cohortId))
-    } else{
-      ParallelLogger::logInfo(paste0('Adding cohort ',cohortTocreate$cohortName[1]))
-      
-      data <- data.frame(cohortName = cohortTocreate$cohortName, 
-                         atlasId = cohortTocreate$cohortId,
-                         cohortJson = json)
-      DatabaseConnector::insertTable(connection = conn, 
-                                     databaseSchema = resultSchema, 
-                                     tableName = paste0(stringAppendToTables, 'cohorts'),
-                                     data = data,
-                                     dropTableIfExists = F, 
-                                     createTable = F, 
-                                     tempTable = F, 
-                                     progressBar = T,
-                                     camelCaseToSnakeCase = T, 
-                                     tempEmulationSchema = tempEmulationSchema
-      )
-      
-      # now check and get id
-      result <- checkTable(conn = conn, 
-                           resultSchema = resultSchema, 
-                           stringAppendToTables = stringAppendToTables,
-                           targetDialect = targetDialect, 
-                           tableName = 'cohorts',
-                           columnNames = c('cohort_name', 'atlas_id'), 
-                           values = c(paste0("'",cohortTocreate$cohortName,"'"), cohortTocreate$cohortId),
-                           tempEmulationSchema = tempEmulationSchema
-      )
-      
-      jsonInd <- result$cohortJson %in% json
-      result <- result[jsonInd,]
-      
-    }
-  
-  return(result$cohortId[1])
-}
-
-
-addPopulationSetting <- function(conn, resultSchema, targetDialect,
-                                 stringAppendToTables = '',
-                                 json,
-                                 tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  # process json to make it ordered...
-  # make sure the json has been converted 
-  if(class(json)!='character'){
-    json <- as.character(jsonlite::serializeJSON(json, digits = 23))
-  }
-  
-  jsonId <- checkJson(conn = conn,
-                      resultSchema = resultSchema, 
-                      stringAppendToTables = stringAppendToTables,
-                      targetDialect = targetDialect, 
-                      tableName = 'population_settings',
-                      jsonColumnName = 'populationSettingsJson',
-                      id = 'populationSettingId',
-                      json = json, 
-                      tempEmulationSchema = tempEmulationSchema)
-  
-  if(is.null(jsonId)){
-    ParallelLogger::logInfo('Adding new population settings')
-    
-    data <- data.frame(populationSettingsJson = json)
-    DatabaseConnector::insertTable(connection = conn, 
-                                   databaseSchema = resultSchema, 
-                                   tableName = paste0(stringAppendToTables, 'population_settings'),
-                                   data = data, 
-                                   dropTableIfExists = F, 
-                                   createTable = F, 
-                                   tempTable = F, 
-                                   progressBar = T,
-                                   camelCaseToSnakeCase = T,
-                                   tempEmulationSchema = tempEmulationSchema
-    )
-    
-    #getId of new
-    jsonId <- checkJson(conn = conn,
-                        resultSchema = resultSchema, 
-                        stringAppendToTables = stringAppendToTables,
-                        targetDialect = targetDialect, 
-                        tableName = 'population_settings',
-                        jsonColumnName = 'populationSettingsJson',
-                        id = 'populationSettingId',
-                        json = json,
-                        tempEmulationSchema = tempEmulationSchema)
-    
-  } else{
-    ParallelLogger::logInfo('Population settings exists')
-  }
-  
-  return(jsonId)
-}
-
-
-addCovariateSetting <- function(conn, resultSchema, targetDialect,
-                                stringAppendToTables = '',
-                                json,
-                                tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  # process json to make it ordered...
-  # make sure the json has been converted 
-  if(class(json)!='character'){
-    # this code created character that is too long for redshfit
-    #json <- as.character(jsonlite::serializeJSON(json, digits = 23))
-    # add attributes
-    if(class(json) == 'covariateSettings'){
-      json <- list(json)
-    }
-    json <- lapply(json, addAttributes)
-    #convert
-    json <- jsonlite::toJSON(
-      x = json, 
-      pretty = T, 
-      digits = 23, 
-      auto_unbox=TRUE, 
-      null = "null"
-    )
-    json <- as.character(json) # now convert to character
-    print(nchar(json))
-  }
-  
-  jsonId <- checkJson(conn = conn,
-                      resultSchema = resultSchema, 
-                      stringAppendToTables = stringAppendToTables,
-                      targetDialect = targetDialect, 
-                      tableName = 'covariate_settings',
-                      jsonColumnName = 'covariateSettingsJson',
-                      id = 'covariateSettingId',
-                      json = json,
-                      tempEmulationSchema = tempEmulationSchema)
-  
-  if(is.null(jsonId)){
-    
-    ParallelLogger::logInfo('Adding new covariate settings')
-    
-    data <- data.frame(covariateSettingsJson = json)
-    DatabaseConnector::insertTable(connection = conn, 
-                                   databaseSchema = resultSchema, 
-                                   tableName = paste0(stringAppendToTables, 'covariate_settings'),
-                                   data = data, 
-                                   dropTableIfExists = F, 
-                                   createTable = F, 
-                                   tempTable = F, 
-                                   progressBar = T,
-                                   camelCaseToSnakeCase = T,
-                                   tempEmulationSchema = tempEmulationSchema
-    )
-    
-    #getId of new
-    jsonId <- checkJson(conn = conn,
-                        resultSchema = resultSchema, 
-                        stringAppendToTables = stringAppendToTables,
-                        targetDialect = targetDialect, 
-                        tableName = 'covariate_settings',
-                        jsonColumnName = 'covariateSettingsJson',
-                        id = 'covariateSettingId',
-                        json = json,
-                        tempEmulationSchema = tempEmulationSchema)
-    
-  } else{
-    ParallelLogger::logInfo('Covariate setting exists')
-  }
-  
-  return(jsonId)
-}
-
-
-addModelSetting <- function(conn, resultSchema, targetDialect,
-                            stringAppendToTables = '',
-                            modelType, json,
-                            tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  # process json to make it ordered...
-  # make sure the json has been converted 
-  if(class(json)!='character'){
-    json <- as.character(jsonlite::serializeJSON(json, digits = 23))
-  }
-  
-  jsonId <- checkJson(conn = conn,
-                      resultSchema = resultSchema, 
-                      stringAppendToTables = stringAppendToTables,
-                      targetDialect = targetDialect, 
-                      tableName = 'model_settings',
-                      jsonColumnName = 'modelSettingsJson',
-                      id = 'modelSettingId',
-                      json = json,
-                      tempEmulationSchema = tempEmulationSchema)
-  
-  if(is.null(jsonId)){
-    
-    ParallelLogger::logInfo('Adding new model settings')
-    
-    data <- data.frame(modelType = modelType,
-                       modelSettingsJson = json)
-    DatabaseConnector::insertTable(connection = conn, 
-                                   databaseSchema = resultSchema, 
-                                   tableName = paste0(stringAppendToTables, 'model_settings'),
-                                   data = data, 
-                                   dropTableIfExists = F, 
-                                   createTable = F, 
-                                   tempTable = F, 
-                                   progressBar = T,
-                                   camelCaseToSnakeCase = T,
-                                   tempEmulationSchema = tempEmulationSchema)
-    
-    #getId of new
-    jsonId <- checkJson(conn = conn,
-                        resultSchema = resultSchema, 
-                        stringAppendToTables = stringAppendToTables,
-                        targetDialect = targetDialect, 
-                        tableName = 'model_settings',
-                        jsonColumnName = 'modelSettingsJson',
-                        id = 'modelSettingId',
-                        json = json,
-                        tempEmulationSchema = tempEmulationSchema)
-    
-  } else{
-    ParallelLogger::logInfo('Model setting exists')
-  }
-  
-  return(jsonId)
-}
-
-addTidySetting <- function(
-  conn, 
-  resultSchema, 
-  targetDialect,
-  stringAppendToTables = '',
-  json,
-  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  if(class(json)!='character'){
-    
-    #modify to make smaller but keep key part
-    json$deletedInfrequentCovariateIds <- c()
-    json$normFactors <- json$normFactors %>% dplyr::filter(.data$maxValue !=1)
-    
-    json <- as.character(jsonlite::serializeJSON(json, digits = 23))
-  }
-  
-  jsonId <- checkJson(conn = conn,
-    resultSchema = resultSchema, 
-    stringAppendToTables = stringAppendToTables,
-    targetDialect = targetDialect, 
-    tableName = 'tidy_covariates_settings',
-    jsonColumnName = 'tidyCovariatesSettingsJson',
-    id = 'tidyCovariatesSettingId',
-    json = json,
-    tempEmulationSchema = tempEmulationSchema)
-  
-  if(is.null(jsonId)){
-    
-    ParallelLogger::logInfo('Adding new tidy covariates settings')
-    
-    data <- data.frame(
-      tidyCovariatesSettingsJson = json
-    )
-    
-    DatabaseConnector::insertTable(
-      connection = conn, 
-      databaseSchema = resultSchema, 
-      tableName = paste0(stringAppendToTables, 'tidy_covariates_settings'),
-      data = data, 
-      dropTableIfExists = F, 
-      createTable = F, 
-      tempTable = F, 
-      progressBar = T,
-      camelCaseToSnakeCase = T,
-      tempEmulationSchema = tempEmulationSchema
-    )
-    
-    #getId of new
-    jsonId <- checkJson(
-      conn = conn,
-      resultSchema = resultSchema, 
-      stringAppendToTables = stringAppendToTables,
-      targetDialect = targetDialect, 
-      tableName = 'tidy_covariates_settings',
-      jsonColumnName = 'tidyCovariatesSettingsJson',
-      id = 'tidyCovariatesSettingId',
-      json = json,
-      tempEmulationSchema = tempEmulationSchema
-      )
-    
-  } else{
-    ParallelLogger::logInfo('tidy covariates setting exists')
-  }
-  
-  return(jsonId)
-  
-}
-
-addSampleSetting <- function(
-  conn, 
-  resultSchema, 
-  targetDialect,
-  stringAppendToTables = '',
-  json,
-  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  if(class(json)!='character'){
-    json <- as.character(jsonlite::serializeJSON(json, digits = 23))
-  }
-  
-  jsonId <- checkJson(
-    conn = conn,
-    resultSchema = resultSchema, 
-    stringAppendToTables = stringAppendToTables,
-    targetDialect = targetDialect, 
-    tableName = 'sample_settings',
-    jsonColumnName = 'sampleSettingsJson',
-    id = 'sampleSettingId',
-    json = json,
-    tempEmulationSchema = tempEmulationSchema
-    )
-  
-  if(is.null(jsonId)){
-    
-    ParallelLogger::logInfo('Adding new sample settings')
-    
-    data <- data.frame(
-      sampleSettingsJson = json
-    )
-    
-    DatabaseConnector::insertTable(
-      connection = conn, 
-      databaseSchema = resultSchema, 
-      tableName = paste0(stringAppendToTables, 'sample_settings'),
-      data = data, 
-      dropTableIfExists = F, 
-      createTable = F, 
-      tempTable = F, 
-      progressBar = T,
-      camelCaseToSnakeCase = T,
-      tempEmulationSchema = tempEmulationSchema
-    )
-    
-    #getId of new
-    jsonId <- checkJson(
-      conn = conn,
-      resultSchema = resultSchema, 
-      stringAppendToTables = stringAppendToTables,
-      targetDialect = targetDialect, 
-      tableName = 'sample_settings',
-      jsonColumnName = 'sampleSettingsJson',
-      id = 'sampleSettingId',
-      json = json,
-      tempEmulationSchema = tempEmulationSchema
-      )
-    
-  } else{
-    ParallelLogger::logInfo('sample setting exists')
-  }
-  
-  return(jsonId)
-  
-}
-
-addPlpDataSetting <- function(
-  conn, 
-  resultSchema, 
-  targetDialect,
-  stringAppendToTables = '',
-  json,
-  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  if(class(json)!='character'){
-    json <- as.character(jsonlite::serializeJSON(json, digits = 23))
-  }
-  
-  jsonId <- checkJson(conn = conn,
-    resultSchema = resultSchema, 
-    stringAppendToTables = stringAppendToTables,
-    targetDialect = targetDialect, 
-    tableName = 'plp_data_settings',
-    jsonColumnName = 'plpDataSettingsJson',
-    id = 'plpDataSettingId',
-    json = json,
-    tempEmulationSchema = tempEmulationSchema)
-  
-  if(is.null(jsonId)){
-    
-    ParallelLogger::logInfo('Adding new plp data settings')
-    
-    data <- data.frame(
-      plpDataSettingsJson = json
-    )
-    
-    DatabaseConnector::insertTable(
-      connection = conn, 
-      databaseSchema = resultSchema, 
-      tableName = paste0(stringAppendToTables, 'plp_data_settings'),
-      data = data, 
-      dropTableIfExists = F, 
-      createTable = F, 
-      tempTable = F, 
-      progressBar = T,
-      camelCaseToSnakeCase = T,
-      tempEmulationSchema = tempEmulationSchema
-    )
-    
-    #getId of new
-    jsonId <- checkJson(
-      conn = conn,
-      resultSchema = resultSchema, 
-      stringAppendToTables = stringAppendToTables,
-      targetDialect = targetDialect, 
-      tableName = 'plp_data_settings',
-      jsonColumnName = 'plpDataSettingsJson',
-      id = 'plpDataSettingId',
-      json = json,
-      tempEmulationSchema = tempEmulationSchema
-      )
-    
-  } else{
-    ParallelLogger::logInfo('Split setting exists')
-  }
-  
-  return(jsonId)
-  
-}
-
-addFESetting <- function(
-  conn, 
-  resultSchema, 
-  targetDialect,
-  stringAppendToTables = '',
-  json,
-  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  if(class(json)!='character'){
-    json <- as.character(jsonlite::serializeJSON(json, digits = 23))
-  }
-  
-  jsonId <- checkJson(
-    conn = conn,
-    resultSchema = resultSchema, 
-    stringAppendToTables = stringAppendToTables,
-    targetDialect = targetDialect, 
-    tableName = 'feature_engineering_settings',
-    jsonColumnName = 'featureEngineeringSettingsJson',
-    id = 'featureEngineeringSettingId',
-    json = json,
-    tempEmulationSchema = tempEmulationSchema
-    )
-  
-  if(is.null(jsonId)){
-    
-    ParallelLogger::logInfo('Adding new feature_engineering settings')
-    
-    data <- data.frame(
-      featureEngineeringSettingsJson = json
-    )
-    
-    DatabaseConnector::insertTable(
-      connection = conn, 
-      databaseSchema = resultSchema, 
-      tableName = paste0(stringAppendToTables, 'feature_engineering_settings'),
-      data = data, 
-      dropTableIfExists = F, 
-      createTable = F, 
-      tempTable = F, 
-      progressBar = T,
-      camelCaseToSnakeCase = T,
-      tempEmulationSchema = tempEmulationSchema
-    )
-    
-    #getId of new
-    jsonId <- checkJson(
-      conn = conn,
-      resultSchema = resultSchema, 
-      stringAppendToTables = stringAppendToTables,
-      targetDialect = targetDialect, 
-      tableName = 'feature_engineering_settings',
-      jsonColumnName = 'featureEngineeringSettingsJson',
-      id = 'featureEngineeringSettingId',
-      json = json,
-      tempEmulationSchema = tempEmulationSchema
-      )
-    
-  } else{
-    ParallelLogger::logInfo('feature engineering setting exists')
-  }
-  
-  return(jsonId)
-  
-}
-
-addSplitSettings <- function(
-  conn, 
-  resultSchema, 
-  targetDialect,
-  stringAppendToTables = '',
-  json,
-  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  if(class(json)!='character'){
-    json <- as.character(jsonlite::serializeJSON(json, digits = 23))
-  }
-  
-  jsonId <- checkJson(
-    conn = conn,
-    resultSchema = resultSchema, 
-    stringAppendToTables = stringAppendToTables,
-    targetDialect = targetDialect, 
-    tableName = 'split_settings',
-    jsonColumnName = 'splitSettingsJson',
-    id = 'splitSettingId',
-    json = json,
-    tempEmulationSchema = tempEmulationSchema
-    )
-  
-  if(is.null(jsonId)){
-    
-    ParallelLogger::logInfo('Adding new split settings')
-    
-    data <- data.frame(
-      splitSettingsJson = json
-      )
-    
-    DatabaseConnector::insertTable(
-      connection = conn, 
-      databaseSchema = resultSchema, 
-      tableName = paste0(stringAppendToTables, 'split_settings'),
-      data = data, 
-      dropTableIfExists = F, 
-      createTable = F, 
-      tempTable = F, 
-      progressBar = T,
-      camelCaseToSnakeCase = T,
-      tempEmulationSchema = tempEmulationSchema
-      )
-    
-    #getId of new
-    jsonId <- checkJson(
-      conn = conn,
-      resultSchema = resultSchema, 
-      stringAppendToTables = stringAppendToTables,
-      targetDialect = targetDialect, 
-      tableName = 'split_settings',
-      jsonColumnName = 'splitSettingsJson',
-      id = 'splitSettingId',
-      json = json,
-      tempEmulationSchema = tempEmulationSchema
-      )
-    
-  } else{
-    ParallelLogger::logInfo('Split setting exists')
-  }
-  
-  return(jsonId)
-  
-}
-
-
-addModelDesign <- function(
-  conn, 
-  resultSchema, targetDialect,
-  stringAppendToTables = stringAppendToTables,
-  targetId,
-  outcomeId,
-  tarId,
-  plpDataSettingId,
-  populationSettingId,
-  modelSettingId,
-  covariateSettingId,
-  sampleSettingId,
-  splitSettingId,
-  featureEngineeringSettingId,
-  tidyCovariatesSettingId,
-  researcherId,
-  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  if(is.null(targetId)){
-    stop('targetId is null')
-  }
-  if(is.null(outcomeId)){
-    stop('outcomeId is null')
-  }
-  if(is.null(tarId)){
-    stop('tarId is null')
-  }
-  
-  if(is.null(plpDataSettingId)){
-    stop('plpDataSettingId is null')
-  }
-  if(is.null(populationSettingId)){
-    stop('populationSettingId is null')
-  }
-  if(is.null(modelSettingId)){
-    stop('modelSettingId is null')
-  }
-  if(is.null(covariateSettingId)){
-    stop('covariateSettingId is null')
-  }
-  if(is.null(sampleSettingId)){
-    stop('sampleSettingId is null')
-  }
-  if(is.null(splitSettingId)){
-    stop('splitSettingId is null')
-  }
-  if(is.null(featureEngineeringSettingId)){
-    stop('featureEngineeringSettingId is null')
-  }
-  if(is.null(tidyCovariatesSettingId)){
-    stop('tidyCovariatesSettingId is null')
-  }
-  
-  if(is.null(researcherId)){
-    stop('researcherId is null')
-  }
-  
-  # process json to make it ordered...
-  # TODO
-  
-  result <- checkTable(
-    conn = conn, 
-    resultSchema = resultSchema, 
-    stringAppendToTables = stringAppendToTables,
-    targetDialect = targetDialect, 
-    tableName = 'model_designs',
-    columnNames = c(
-      'target_id',
-      'outcome_id',
-      'tar_id',
-      'plp_data_setting_id',
-      'population_setting_id',
-      'model_setting_id',
-      'covariate_setting_id',
-      'sample_setting_id',
-      'split_setting_id',
-      'feature_engineering_setting_id',
-      'tidy_covariates_setting_id',
-      'researcher_id'
-      ), 
-    values = c(
-      targetId,
-      outcomeId,
-      tarId,
-      plpDataSettingId,
-      populationSettingId,
-      modelSettingId,
-      covariateSettingId,
-      sampleSettingId,
-      splitSettingId,
-      featureEngineeringSettingId,
-      tidyCovariatesSettingId,
-      researcherId
-      ),
-    tempEmulationSchema = tempEmulationSchema
-  )
-  
-  if(nrow(result)==0){
-    # model
-    sql <- "INSERT INTO @my_schema.@string_to_appendmodel_designs(
-    target_id,
-    outcome_id,
-    tar_id,
-    plp_data_setting_id,
-    population_setting_id,
-    model_setting_id,
-    covariate_setting_id,
-    sample_setting_id,
-    split_setting_id,
-    feature_engineering_setting_id,
-    tidy_covariates_setting_id,
-    researcher_id
-    ) VALUES 
-  ( 
-  @target_id, 
-  @outcome_id, 
-  @tar_id, 
-  @plp_data_setting_id,
-  @population_setting_id, 
-  @model_setting_id, 
-  @covariate_setting_id, 
-  @sample_setting_id, 
-  @split_setting_id, 
-  @feature_engineering_setting_id, 
-  @tidy_covariates_setting_id,
-  @researcher_id
-    )"
-    sql <- SqlRender::render(
-      sql, 
-      my_schema = resultSchema,
-      target_id = targetId,
-      outcome_id = outcomeId,
-      tar_id = tarId,
-      plp_data_setting_id= plpDataSettingId,
-      population_setting_id = populationSettingId,
-      model_setting_id = modelSettingId,
-      covariate_setting_id = covariateSettingId,
-      sample_setting_id = sampleSettingId,
-      split_setting_id = splitSettingId,
-      feature_engineering_setting_id = featureEngineeringSettingId,
-      tidy_covariates_setting_id = tidyCovariatesSettingId,
-      researcher_id = researcherId,
-      string_to_append = stringAppendToTables
-    )
-    sql <- SqlRender::translate(sql, targetDialect = targetDialect,
-                                tempEmulationSchema = tempEmulationSchema)
-    DatabaseConnector::executeSql(conn, sql)
-    
-    #getId of new
-    result <- checkTable(conn = conn, 
-                         resultSchema = resultSchema, 
-                         stringAppendToTables = stringAppendToTables,
-                         targetDialect = targetDialect, 
-                         tableName = 'model_designs',
-                         columnNames = c(
-                                         'target_id',
-                                         'outcome_id',
-                                         'tar_id',
-                           'plp_data_setting_id',
-                           'population_setting_id',
-                           'model_setting_id',
-                           'covariate_setting_id',
-                           'sample_setting_id',
-                           'split_setting_id',
-                           'feature_engineering_setting_id',
-                           'tidy_covariates_setting_id',
-                            'researcher_id'), 
-                         values = c(targetId,
-                                    outcomeId,
-                                    tarId,
-                           plpDataSettingId,
-                           populationSettingId,
-                           modelSettingId,
-                           covariateSettingId,
-                           sampleSettingId,
-                           splitSettingId,
-                           featureEngineeringSettingId,
-                           tidyCovariatesSettingId,
-                           researcherId),
-                         tempEmulationSchema = tempEmulationSchema
-    )
-    
-  } 
-  
-  return(result$modelDesignId[1])
-}
-
-addModel <- function(
-  conn, 
-  resultSchema, 
-  targetDialect,
-  stringAppendToTables = stringAppendToTables,
-  analysisId,
-  modelDesignId,
-  researcherId,
-  databaseId,
-  hyperParamSearch,
-  plpModelFile,
-  executionDateTime,
-  trainingTime,
-  intercept,
-  requireDenseMatrix,
-  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
-){
-  
-  if(is.null(analysisId)){
-    stop('analysisId is null')
-  }
-  if(is.null(modelDesignId)){
-    stop('modelName is null')
-  }
-  if(is.null(researcherId)){
-    stop('researcherId is null')
-  }
-  if(is.null(databaseId)){
-    stop('databaseId is null')
-  }
-  if(is.null(plpModelFile)){
-    stop('plpModelFile is null')
-  }
-  if(is.null(executionDateTime)){
-    stop('executionDateTime is null')
-  }
-  if(is.null(intercept)){
-    stop('intercept is null')
-  }
-  
-  if(!is.null(hyperParamSearch)){
-    if(class(hyperParamSearch) != 'character'){
-      hyperParamSearch <- as.character(jsonlite::serializeJSON(hyperParamSearch, digits = 23))
-    }
-  }else{
-    hyperParamSearch <- '' 
-  }
-  
-  # process json to make it ordered...
-  # TODO
-  
-  result <- checkTable(
-    conn = conn, 
-    resultSchema = resultSchema, 
-    stringAppendToTables = stringAppendToTables,
-    targetDialect = targetDialect, 
-    tableName = 'models',
-    columnNames = c(
-      'analysis_id',
-      'model_design_id',
-      'researcher_id',
-      'database_id',
-      'hyper_param_search',
-      'plp_model_file',
-      'execution_date_time',
-      'training_time',
-      'intercept',
-      'require_dense_matrix'
-      ), 
-    values = c(
-      enc(analysisId),
-      modelDesignId,
-      researcherId,
-      databaseId,
-      enc(hyperParamSearch),
-      enc(plpModelFile),
-      enc(executionDateTime),
-      enc(trainingTime),
-      intercept,
-      ifelse(requireDenseMatrix, "'T'", "'F'")
-      ),
-    tempEmulationSchema = tempEmulationSchema
-  )
-  
-  if(nrow(result)==0){
-    # model
-    sql <- "INSERT INTO @my_schema.@string_to_appendmodels(
-    analysis_id,
-    model_design_id,
-    researcher_id,
-    database_id,
-    hyper_param_search,
-    plp_model_file,
-    execution_date_time,
-    training_time,
-    intercept,
-    require_dense_matrix
-    ) VALUES 
-  ('@analysis_id', 
-  '@model_design_id',
-  @researcher_id,
-  @database_id, 
-  '@hyper_param_search', 
-  '@plp_model_file', 
-  '@execution_date_time', 
-  '@training_time', 
-   @intercept,
-  '@require_dense_matrix' 
-  )"
-    sql <- SqlRender::render(
-      sql, 
-      my_schema = resultSchema,
-      analysis_id = analysisId,
-      model_design_id = modelDesignId,
-      researcher_id = researcherId,
-      database_id = databaseId,
-      hyper_param_search = hyperParamSearch,
-      plp_model_file = plpModelFile,
-      execution_date_time = executionDateTime,
-      training_time = trainingTime,
-      intercept = intercept,
-      require_dense_matrix = ifelse(requireDenseMatrix, 'T', 'F'),
-      string_to_append = stringAppendToTables
-    )
-    sql <- SqlRender::translate(
-      sql, 
-      targetDialect = targetDialect,
-      tempEmulationSchema = tempEmulationSchema
-      )
-    DatabaseConnector::executeSql(conn, sql)
-    
-    #getId of new
-    result <- checkTable(
-      conn = conn, 
-      resultSchema = resultSchema, 
-      stringAppendToTables = stringAppendToTables,
-      targetDialect = targetDialect, 
-      tableName = 'models',
-      columnNames = c(
-        'analysis_id',
-        'model_design_id',
-        'researcher_id',
-        'database_id',
-        'hyper_param_search',
-        'plp_model_file',
-        'execution_date_time',
-        'training_time',
-        'intercept',
-        'require_dense_matrix'
-        ), 
-      values = c(
-        enc(analysisId),
-        modelDesignId,
-        researcherId,
-        databaseId,
-        enc(hyperParamSearch),
-        enc(plpModelFile),
-        enc(executionDateTime),
-        enc(trainingTime),
-        intercept,
-        ifelse(requireDenseMatrix, "'T'", "'F'")
-        ),
-      tempEmulationSchema = tempEmulationSchema
-    )
-    
-  } 
-  
-  return(result$modelId[1])
-}
-
-addResult <- function(conn, resultSchema, targetDialect,
-                      stringAppendToTables = '',
-                      modelId,
-                      researcherId,
-                      databaseId,
-                      targetId,
-                      outcomeId,
-                      tarId,
-  restrictPlpDataSettingId,
-                      populationSettingId,
-                      executionDateTime,
-                      plpVersion,
-                      tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")){
-  
-  result <- checkTable(conn = conn, 
-                       resultSchema = resultSchema, 
-                       stringAppendToTables = stringAppendToTables,
-                       targetDialect = targetDialect, 
-                       tableName = 'results',
-                       columnNames = c('model_id',
-                                       'researcher_id',
-                                       'database_id',
-                                       'target_id',
-                                       'outcome_id',
-                                       'tar_id',
-                         'plp_data_setting_id',
-                                       'population_setting_id',
-                                       'execution_date_time',
-                                       'plp_version'), 
-                       values = c(modelId,
-                                  researcherId,
-                                  databaseId,
-                                  targetId,
-                                  outcomeId,
-                                  tarId,
-                         restrictPlpDataSettingId,
-                                  populationSettingId,
-                                  enc(executionDateTime),
-                                  enc(plpVersion)),
-                       tempEmulationSchema = tempEmulationSchema
-  )
-  
-  if(nrow(result)==0){
-    # model
-    sql <- "INSERT INTO @my_schema.@string_to_appendresults (
-    model_id,
-    researcher_id,
-    database_id,
-    target_id,
-    outcome_id,
-    tar_id,
-    plp_data_setting_id,
-    population_setting_id,
-    execution_date_time,
-    plp_version
-  ) 
-  VALUES (@model_id, @researcher_id, @database_id, @target_id, @outcome_id, @tar_id, 
-          @plp_data_setting_id, @population_setting_id, '@execution_date_time', '@plp_version')"
-    sql <- SqlRender::render(sql, 
-                             my_schema = resultSchema,
-                             model_id = modelId,
-                             researcher_id = researcherId,
-                             database_id = databaseId,
-                             target_id = targetId,
-                             outcome_id = outcomeId,
-                             tar_id = tarId,
-      plp_data_setting_id = restrictPlpDataSettingId,
-                             population_setting_id = populationSettingId,
-                             execution_date_time = executionDateTime,
-                             plp_version = plpVersion,
-                             string_to_append = stringAppendToTables)
-    sql <- SqlRender::translate(sql, targetDialect = targetDialect,
-                                tempEmulationSchema = tempEmulationSchema)
-    DatabaseConnector::executeSql(conn, sql)
-    
-    #getId of new
-    result <- checkTable(conn = conn, 
-                         resultSchema = resultSchema, 
-                         stringAppendToTables = stringAppendToTables,
-                         targetDialect = targetDialect, 
-                         tableName = 'results',
-                         columnNames = c('model_id',
-                                         'researcher_id',
-                                         'database_id',
-                                         'target_id',
-                                         'outcome_id',
-                                         'tar_id',
-                           'plp_data_setting_id', 
-                                         'population_setting_id',
-                                         'execution_date_time',
-                                         'plp_version'), 
-                         values = c(modelId,
-                                    researcherId,
-                                    databaseId,
-                                    targetId,
-                                    outcomeId,
-                                    tarId,
-                           restrictPlpDataSettingId,
-                                    populationSettingId,
-                                    enc(executionDateTime),
-                                    enc(plpVersion)),
-                         tempEmulationSchema = tempEmulationSchema
-    )
-    
-  } 
-  
-  return(result$resultId[1])
-}
-
-# attrition
-addAttrition <- function(
-  conn, resultSchema, targetDialect,
-  stringAppendToTables = '',
-  resultId,
-  attrition,
-  overWriteIfExists = T,
-  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")){
-  
-  value <- attrition
-  if(is.null(value)){
-    return(NULL)
-  }
-  
-  # edit names
-  firstLower <- function(x) {
-    substr(x, 1, 1) <- tolower(substr(x, 1, 1))
-    return(x)
-  }
-  colnames(value) <- sapply(colnames(value), firstLower )
-
-  value$resultId <- resultId
-  
-  # get column names and check all present in object
-  columnNames <- getColumnNames(conn = conn, 
-    resultSchema = resultSchema, 
-    targetDialect = targetDialect, 
-    tableName = paste0(stringAppendToTables,'attrition'), 
-    tempEmulationSchema = tempEmulationSchema)
-  isValid <- sum(colnames(value)%in%columnNames) == length(columnNames)
-  
-  exists <- checkResultExists(conn = conn, 
-    resultSchema = resultSchema, 
-    targetDialect = targetDialect, 
-    tableName = paste0(stringAppendToTables,'attrition'),
-    resultId = resultId,
-    tempEmulationSchema = tempEmulationSchema)
-  
-  if(isValid && (!exists || overWriteIfExists)){
-    
-    # REMOVE existing result
-    if(exists){
-      sql <- "delete from @result_schema.@table_name where result_id = @result_id;"
-      sql <- SqlRender::render(sql, 
-        result_id=resultId,
-        result_schema = resultSchema,
-        table_name = paste0(stringAppendToTables,'attrition'))
-      sql <- SqlRender::translate(sql, 
-        targetDialect = targetDialect,
-        tempEmulationSchema = tempEmulationSchema)
-      DatabaseConnector::executeSql(conn, sql)
-    }
-    
-    # add 
-    ParallelLogger::logInfo(paste0('Inserting attrition for result ',resultId))
-    DatabaseConnector::insertTable(connection = conn, 
-      databaseSchema = resultSchema, 
-      tableName = paste0(stringAppendToTables,'attrition'), 
-      data = value[,columnNames], 
-      dropTableIfExists = F, createTable = F, tempTable = F, 
-      bulkLoad = F, camelCaseToSnakeCase = T, progressBar = T,
-      tempEmulationSchema = tempEmulationSchema)
-  }
-  
-  return(invisible(NULL))
-}
-
-
-# evals
-addEvaluation <- function(conn, resultSchema, targetDialect,
-                          stringAppendToTables = '',
-                          resultId,
-                          performanceEvaluation,
-                          overWriteIfExists = T,
-                          tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")){
-  
-  ParallelLogger::logInfo('Adding PredictionDistribution')
-  tryCatch({addPredictionDistribution(conn = conn, resultSchema = resultSchema, targetDialect = targetDialect,
-                                      stringAppendToTables = stringAppendToTables,
-                                      resultId = resultId,
-                                      performanceEvaluation = performanceEvaluation,
-                                      overWriteIfExists = overWriteIfExists,
-                                      tempEmulationSchema = tempEmulationSchema)},
-           error = function(e){ParallelLogger::logError(e);})
-  
-  ParallelLogger::logInfo('Adding ThresholdSummary')
-  tryCatch({addThresholdSummary(conn = conn, resultSchema = resultSchema, targetDialect = targetDialect,
-                                stringAppendToTables = stringAppendToTables,
-                                resultId = resultId,
-                                performanceEvaluation = performanceEvaluation,
-                                overWriteIfExists = overWriteIfExists,
-                                tempEmulationSchema = tempEmulationSchema)},
-           error = function(e){ParallelLogger::logError(e);})
-  
-  ParallelLogger::logInfo('Adding EvaluationStatistics')
-  tryCatch({addEvaluationStatistics(conn = conn, resultSchema = resultSchema, targetDialect = targetDialect,
-                                    stringAppendToTables = stringAppendToTables,
-                                    resultId = resultId,
-                                    performanceEvaluation = performanceEvaluation,
-                                    overWriteIfExists = overWriteIfExists,
-                                    tempEmulationSchema = tempEmulationSchema)},
-           error = function(e){ParallelLogger::logError(e);})
-  
-  ParallelLogger::logInfo('Adding CalibrationSummary')
-  tryCatch({addCalibrationSummary(conn = conn, resultSchema = resultSchema, targetDialect = targetDialect,
-                                  stringAppendToTables = stringAppendToTables,
-                                  resultId = resultId,
-                                  performanceEvaluation = performanceEvaluation,
-                                  overWriteIfExists = overWriteIfExists,
-                                  tempEmulationSchema = tempEmulationSchema)},
-           error = function(e){ParallelLogger::logError(e);})
-  
-  ParallelLogger::logInfo('Adding DemographicSummary')
-  tryCatch({addDemographicSummary(conn = conn, resultSchema = resultSchema, targetDialect = targetDialect,
-                                  stringAppendToTables = stringAppendToTables,
-                                  resultId = resultId,
-                                  performanceEvaluation = performanceEvaluation,
-                                  overWriteIfExists = overWriteIfExists,
-                                  tempEmulationSchema = tempEmulationSchema)},
-           error = function(e){ParallelLogger::logError(e);})
-  
-  return(invisible(NULL))
-  
-}
-
-addPredictionDistribution <- function(conn, resultSchema, targetDialect,
-                                      stringAppendToTables = '',
-                                      resultId,
-                                      performanceEvaluation,
-                                      overWriteIfExists = T,
-                                      tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")){
-  
-  value <- performanceEvaluation$predictionDistribution
-  if(is.null(value)){
-    return(NULL)
-  }
-  
-  # edit names
-  firstLower <- function(x) {
-    substr(x, 1, 1) <- tolower(substr(x, 1, 1))
-    return(x)
-  }
-  colnames(value) <- sapply(colnames(value), firstLower )
-  if(sum(colnames(value)=='class')>0){
-    colnames(value)[colnames(value)=='class'] <- 'classLabel'
-  }
-  
-  value$resultId <- resultId
-  
-  # get column names and check all present in object
-  columnNames <- getColumnNames(conn = conn, 
-                                resultSchema = resultSchema, 
-                                targetDialect = targetDialect, 
-                                tableName = paste0(stringAppendToTables,'prediction_distribution'), 
-                                tempEmulationSchema = tempEmulationSchema)
-  isValid <- sum(colnames(value)%in%columnNames) == length(columnNames)
-  
-  exists <- checkResultExists(conn = conn, 
-                              resultSchema = resultSchema, 
-                              targetDialect = targetDialect, 
-                              tableName = paste0(stringAppendToTables,'prediction_distribution'),
-                              resultId = resultId,
-                              tempEmulationSchema = tempEmulationSchema)
-  
-  if(isValid && (!exists || overWriteIfExists)){
-    
-    # REMOVE existing result
-    if(exists){
-      sql <- "delete from @result_schema.@table_name where result_id = @result_id;"
-      sql <- SqlRender::render(sql, 
-                               result_id=resultId,
-                               result_schema = resultSchema,
-                               table_name = paste0(stringAppendToTables,'prediction_distribution'))
-      sql <- SqlRender::translate(sql, 
-                                  targetDialect = targetDialect,
-                                  tempEmulationSchema = tempEmulationSchema)
-      DatabaseConnector::executeSql(conn, sql)
-    }
-    
-    # add 
-    ParallelLogger::logInfo(paste0('Inserting predictionDistribution for result ',resultId))
-    DatabaseConnector::insertTable(connection = conn, 
-                                   databaseSchema = resultSchema, 
-                                   tableName = paste0(stringAppendToTables,'prediction_distribution'), 
-                                   data = value[,columnNames], 
-                                   dropTableIfExists = F, createTable = F, tempTable = F, 
-                                   bulkLoad = F, camelCaseToSnakeCase = T, progressBar = T,
-                                   tempEmulationSchema = tempEmulationSchema)
-  }
-  
-  return(invisible(NULL))
-}
-
-addThresholdSummary <- function(conn, resultSchema, targetDialect,
-                                stringAppendToTables = '',
-                                resultId,
-                                performanceEvaluation,
-                                overWriteIfExists = T,
-                                tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")){
-  
-  
-  value <- performanceEvaluation$thresholdSummary
-  if(is.null(value)){
-    return(NULL)
-  }
-  
-  #  check numerical columns:
-  value <- cleanNum(value)
-  
-  # edit names
-  firstLower <- function(x) {
-    substr(x, 1, 1) <- tolower(substr(x, 1, 1))
-    return(x)
-  }
-  colnames(value) <- sapply(colnames(value), firstLower )
-  value$resultId <- resultId
-  
-  # get column names and check all present in object
-  columnNames <- getColumnNames(conn = conn, 
-                                resultSchema = resultSchema, 
-                                targetDialect = targetDialect, 
-                                stringAppendToTables = stringAppendToTables,
-                                tableName = 'threshold_summary',
-                                tempEmulationSchema = tempEmulationSchema)
-  isValid <- sum(colnames(value)%in%columnNames) == length(columnNames)
-  
-  exists <- checkResultExists(conn = conn, 
-                              resultSchema = resultSchema, 
-                              targetDialect = targetDialect, 
-                              tableName = paste0(stringAppendToTables,'threshold_summary'),
-                              resultId = resultId,
-                              tempEmulationSchema = tempEmulationSchema)
-  
-  if(isValid && (!exists || overWriteIfExists)){
-    
-    # REMOVE existing result
-    if(exists){
-      sql <- "delete from @result_schema.@table_name where result_id = @result_id;"
-      sql <- SqlRender::render(sql, 
-                               result_schema = resultSchema,
-                               result_id = resultId,
-                               table_name = paste0(stringAppendToTables,'threshold_summary'))
-      sql <- SqlRender::translate(sql, 
-                                  targetDialect = targetDialect,
-                                  tempEmulationSchema = tempEmulationSchema)
-      DatabaseConnector::executeSql(conn, sql)
-    }
-    
-    # add 
-    ParallelLogger::logInfo(paste0('Inserting thresholdSummary for result ',resultId))
-    DatabaseConnector::insertTable(connection = conn, 
-                                   databaseSchema = resultSchema, 
-                                   tableName = paste0(stringAppendToTables,'threshold_summary'), 
-                                   data = value[,columnNames], 
-                                   dropTableIfExists = F, createTable = F, tempTable = F, 
-                                   bulkLoad = F, camelCaseToSnakeCase = T, progressBar = T,
-                                   tempEmulationSchema = tempEmulationSchema)
-  }
-  
-  return(invisible(NULL))
-}
-
-
-addCalibrationSummary <- function(conn, resultSchema, targetDialect, 
-                                  stringAppendToTables = '',
-                                  resultId,
-                                  performanceEvaluation,
-                                  overWriteIfExists = T,
-                                  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")){
-  
-  
-  value <- performanceEvaluation$calibrationSummary
-  if(is.null(value)){
-    return(NULL)
-  }
-  
-  #  check numerical columns:
-  value <- cleanNum(value)
-  
-  # edit names
-  firstLower <- function(x) {
-    substr(x, 1, 1) <- tolower(substr(x, 1, 1))
-    return(x)
-  }
-  colnames(value) <- sapply(colnames(value), firstLower )
-  
-  value$resultId <- resultId
-  
-  # get column names and check all present in object
-  columnNames <- getColumnNames(conn = conn, 
-                                resultSchema = resultSchema, 
-                                targetDialect = targetDialect, 
-                                stringAppendToTables = stringAppendToTables,
-                                tableName = 'calibration_summary',
-                                tempEmulationSchema = tempEmulationSchema)
-  isValid <- sum(colnames(value)%in%columnNames) == length(columnNames)
-  
-  exists <- checkResultExists(conn = conn, 
-                              resultSchema = resultSchema, 
-                              targetDialect = targetDialect, 
-                              tableName = paste0(stringAppendToTables,'calibration_summary'),
-                              resultId = resultId,
-                              tempEmulationSchema = tempEmulationSchema)
-  
-  if(isValid && (!exists || overWriteIfExists)){
-    
-    # REMOVE existing result
-    if(exists){
-      sql <- "delete from @result_schema.@table_name where result_id = @result_id;"
-      sql <- SqlRender::render(sql, 
-                               result_schema = resultSchema,
-                               result_id=resultId,
-                               table_name = paste0(stringAppendToTables,'calibration_summary'))
-      sql <- SqlRender::translate(sql, 
-                                  targetDialect = targetDialect,
-                                  tempEmulationSchema = tempEmulationSchema)
-      DatabaseConnector::executeSql(conn, sql)
-    }
-    
-    # add 
-    ParallelLogger::logInfo(paste0('Inserting calibrationSummary for result ',resultId))
-    DatabaseConnector::insertTable(connection = conn, 
-                                   databaseSchema = resultSchema, 
-                                   tableName = paste0(stringAppendToTables,'calibration_summary'), 
-                                   data = value[,columnNames], 
-                                   dropTableIfExists = F, createTable = F, tempTable = F, 
-                                   bulkLoad = F, camelCaseToSnakeCase = T, progressBar = T,
-                                   tempEmulationSchema = tempEmulationSchema)
-  }
-  
-  return(invisible(NULL))
-}
-
-addEvaluationStatistics <- function(conn, resultSchema, targetDialect,
-                                    stringAppendToTables = '',
-                                    resultId,
-                                    performanceEvaluation,
-                                    overWriteIfExists = T,
-                                    tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")){
-  
-  
-  value <- data.frame(
-    evaluation = unlist(performanceEvaluation$evaluationStatistics$evaluation),
-    metric = unlist(performanceEvaluation$evaluationStatistics$metric),
-    value = as.numeric(unlist(performanceEvaluation$evaluationStatistics$value))
-      )
-  
-  if(is.null(value)){
-    return(NULL)
-  }
-  
-  # edit names
-  firstLower <- function(x) {
-    substr(x, 1, 1) <- tolower(substr(x, 1, 1))
-    return(x)
-  }
-  colnames(value) <- sapply(colnames(value), firstLower )
-  value$resultId <- resultId
-  
-  # get column names and check all present in object
-  columnNames <- getColumnNames(conn = conn, 
-                                resultSchema = resultSchema, 
-                                targetDialect = targetDialect, 
-                                stringAppendToTables = stringAppendToTables,
-                                tableName = 'evaluation_statistics',
-                                tempEmulationSchema = tempEmulationSchema)
-  isValid <- sum(colnames(value)%in%columnNames) == length(columnNames)
-  
-  exists <- checkResultExists(conn = conn, 
-                              resultSchema = resultSchema, 
-                              targetDialect = targetDialect,
-                              tableName = paste0(stringAppendToTables,'evaluation_statistics'),
-                              resultId = resultId,
-                              tempEmulationSchema = tempEmulationSchema)
-  
-  if(isValid && (!exists || overWriteIfExists)){
-    
-    # REMOVE existing result
-    if(exists){
-      sql <- "delete from @result_schema.@table_name where result_id = @result_id;"
-      sql <- SqlRender::render(sql, 
-                               result_schema = resultSchema,
-                               result_id = resultId,
-                               table_name = paste0(stringAppendToTables,'evaluation_statistics'))
-      sql <- SqlRender::translate(sql, 
-                                  targetDialect = targetDialect,
-                                  tempEmulationSchema = tempEmulationSchema)
-      DatabaseConnector::executeSql(conn, sql)
-    }
-    
-    # add 
-    ParallelLogger::logInfo(paste0('Inserting evaluationSummary for result ',resultId))
-    DatabaseConnector::insertTable(connection = conn, 
-                                   databaseSchema = resultSchema, 
-                                   tableName = paste0(stringAppendToTables,'evaluation_statistics'), 
-                                   data = value[,columnNames], 
-                                   dropTableIfExists = F, createTable = F, tempTable = F, 
-                                   bulkLoad = F, camelCaseToSnakeCase = T, progressBar = T,
-                                   tempEmulationSchema = tempEmulationSchema)
-  }
-  
-  return(invisible(NULL))
-}
-
-addDemographicSummary <- function(conn, resultSchema, targetDialect, 
-                                  stringAppendToTables = '',
-                                  resultId,
-                                  performanceEvaluation,
-                                  overWriteIfExists = T,
-                                  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")){
-  
-  
-  value <- performanceEvaluation$demographicSummary
-  if(is.null(value)){
-    return(NULL)
-  }
-  
-  # edit names
-  firstLower <- function(x) {
-    substr(x, 1, 1) <- tolower(substr(x, 1, 1))
-    return(x)
-  }
-  colnames(value) <- sapply(colnames(value), firstLower )
-  #if(sum(colnames(value)=="p50PredictedProbability")>0){
-  #  colnames(value)[colnames(value)=="p50PredictedProbability"] <- 'medianPredictedProbability'
-  #}
-  
-  value$resultId <- resultId
-  
-  # get column names and check all present in object
-  columnNames <- getColumnNames(conn = conn, 
-                                resultSchema = resultSchema, 
-                                targetDialect = targetDialect, 
-                                stringAppendToTables =  stringAppendToTables,
-                                tableName = 'demographic_summary',
-                                tempEmulationSchema = tempEmulationSchema)
-  isValid <- sum(colnames(value)%in%columnNames) == length(columnNames)
-  
-  exists <- checkResultExists(conn = conn, 
-                              resultSchema = resultSchema, 
-                              targetDialect = targetDialect, 
-                              tableName = paste0(stringAppendToTables,'demographic_summary'),
-                              resultId = resultId,
-                              tempEmulationSchema = tempEmulationSchema)
-  
-  if(isValid && (!exists || overWriteIfExists)){
-    
-    # REMOVE existing result
-    if(exists){
-      sql <- "delete from @result_schema.@table_name where result_id = @result_id;"
-      sql <- SqlRender::render(sql, 
-                               result_schema = resultSchema,
-                               result_id = resultId,
-                               table_name = paste0(stringAppendToTables,'demographic_summary'))
-      sql <- SqlRender::translate(sql, 
-                                  targetDialect = targetDialect,
-                                  tempEmulationSchema = tempEmulationSchema)
-      DatabaseConnector::executeSql(conn, sql)
-    }
-    
-    # add 
-    ParallelLogger::logInfo(paste0('Inserting demographicSummary for result ',resultId))
-    DatabaseConnector::insertTable(connection = conn, 
-                                   databaseSchema = resultSchema, 
-                                   tableName = paste0(stringAppendToTables,'demographic_summary'), 
-                                   data = value[,columnNames], 
-                                   dropTableIfExists = F, createTable = F, tempTable = F, 
-                                   bulkLoad = F, camelCaseToSnakeCase = T, progressBar = T,
-                                   tempEmulationSchema = tempEmulationSchema)
-  }
-  
-  return(invisible(NULL))
-}
-
-addCovariateSummary <- function(conn, resultSchema, targetDialect, 
-                                stringAppendToTables = '',
-                                resultId,
-                                covariateSummary,
-                                restrictToIncluded = T,
-                                overWriteIfExists = T,
-                                tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")){
-  
-  
-  value <- covariateSummary
-  if(is.null(value)){
-    return(NULL)
-  }
-  
-  # edit names
-  firstLower <- function(x) {
-    substr(x, 1, 1) <- tolower(substr(x, 1, 1))
-    return(x)
-  }
-  colnames(value) <- sapply(colnames(value), firstLower )
-  value$resultId <- resultId
-  # remove _ from names
-  colnames(value) <- gsub('_','', colnames(value))
-  
-  if(restrictToIncluded){
-    ParallelLogger::logInfo('Restricting to covariates included in model')
-    value <- value[value$covariateValue!=0 & !is.na(value$covariateValue),] 
-  }
-  
-  # get column names and check all present in object
-  columnNames <- getColumnNames(conn = conn, 
-                                resultSchema = resultSchema, 
-                                targetDialect = targetDialect, 
-                                stringAppendToTables = stringAppendToTables,
-                                tableName = 'covariate_summary',
-                                tempEmulationSchema = tempEmulationSchema)
-  isValid <- sum(colnames(value)%in%columnNames) == length(columnNames)
-  
-  exists <- checkResultExists(conn = conn, 
-                              resultSchema = resultSchema, 
-                              targetDialect = targetDialect, 
-                              tableName = paste0(stringAppendToTables,'covariate_summary'),
-                              resultId = resultId,
-                              tempEmulationSchema = tempEmulationSchema)
-  
-  if(isValid && (!exists || overWriteIfExists)){
-    
-    # REMOVE existing result
-    if(exists){
-      ParallelLogger::logTrace('Removing existing covariateSummary')
-      sql <- "delete from @result_schema.@table_name where result_id = @result_id;"
-      sql <- SqlRender::render(sql, 
-                               result_schema = resultSchema,
-                               result_id = resultId,
-                               table_name = paste0(stringAppendToTables,'covariate_summary'))
-      sql <- SqlRender::translate(sql, 
-                                  targetDialect = targetDialect,
-                                  tempEmulationSchema = tempEmulationSchema)
-      DatabaseConnector::executeSql(conn, sql)
-    }
-    
-    # add 
-    ParallelLogger::logInfo(paste0('Inserting covariateSummary for result ',resultId))
-    DatabaseConnector::insertTable(connection = conn, 
-                                   databaseSchema = resultSchema, 
-                                   tableName = paste0(stringAppendToTables,'covariate_summary'), 
-                                   data = value[,columnNames], 
-                                   dropTableIfExists = F, createTable = F, tempTable = F, 
-                                   bulkLoad = F, camelCaseToSnakeCase = T, progressBar = T,
-                                   tempEmulationSchema = tempEmulationSchema)
-  }
-  
-  return(invisible(NULL))
-}
-
diff --git a/R/uploadToDatabase.R b/R/uploadToDatabase.R
new file mode 100644
index 000000000..9f94f1464
--- /dev/null
+++ b/R/uploadToDatabase.R
@@ -0,0 +1,1370 @@
+# @file UploadPlpDbResults.R
+#
+# Copyright 2021 Observational Health Data Sciences and Informatics
+#
+# This file is part of PatientLevelPrediction
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+insertRunPlpToSqlite <- function(
+  runPlp, 
+  externalValidatePlp = NULL, 
+  diagnosePlp = NULL
+  ){
+  
+  sqliteLocation <- tempdir()
+  
+  ensure_installed('RSQLite')
+  
+  # create sqlite database
+  connectionDetails <- DatabaseConnector::createConnectionDetails(
+    dbms = 'sqlite',
+    server = file.path(sqliteLocation,'databaseFile.sqlite')
+  )
+
+  createPlpResultTables(
+    connectionDetails = connectionDetails,
+    targetDialect = 'sqlite',
+    resultSchema = 'main', 
+    deleteTables = T, 
+    createTables = T,
+    tablePrefix = ''
+  )
+  
+  #cohortDefinitions <- data.frame(
+  #  cohortId = c(runPlp$model$modelDesign$targetId, runPlp$model$modelDesign$outcomeId),
+  #  cohortName = c('Target', 'Outcome'),
+  #  json = c('{}', '{}')
+  #  )
+    
+  addRunPlpToDatabase(
+    runPlp = runPlp,
+    connectionDetails = connectionDetails,
+    databaseSchemaSettings = createDatabaseSchemaSettings(resultSchema = 'main'),
+    cohortDefinitions = NULL,#cohortDefinitions,
+    databaseList = NULL, 
+    modelSaveLocation = sqliteLocation
+  )
+  
+  # add validation results if entered
+  if(!is.null(externalValidatePlp)){
+    if(inherits(x = externalValidatePlp, what =  'list')){
+      for(i in 1:length(externalValidatePlp)){
+        tryCatch(
+          {
+            addRunPlpToDatabase(
+              runPlp = externalValidatePlp[[i]],
+              connectionDetails = connectionDetails,
+              databaseSchemaSettings = createDatabaseSchemaSettings(resultSchema = 'main'),
+              cohortDefinitions = NULL,#cohortDefinitions,
+              databaseList = NULL, 
+              modelSaveLocation = sqliteLocation
+            )
+          }, error = function(e){ParallelLogger::logError(e)}
+        )
+      }
+    }
+  }
+  
+  # add diagnosis results if entered
+  if(!is.null(diagnosePlp)){
+    tryCatch(
+      {
+        addDiagnosePlpToDatabase(
+          diagnosePlp = diagnosePlp,
+          connectionDetails = connectionDetails,
+          databaseSchemaSettings = createDatabaseSchemaSettings(resultSchema = 'main'),
+          cohortDefinitions = NULL,#cohortDefinitions,
+          databaseList = NULL
+        )
+      }, error = function(e){ParallelLogger::logError(e)}
+    )
+  }
+  
+  
+  return(file.path(sqliteLocation,'databaseFile.sqlite'))
+}
+
+#' Create sqlite database with the results
+#' @description
+#' This function create an sqlite database with the PLP result schema and inserts all results
+#'
+#' @details
+#' This function can be used upload PatientLevelPrediction results into an sqlite database
+#'
+#' @param resultLocation               (string) location of directory where the main package results were saved
+#' @param cohortDefinitions            A set of one or more cohorts extracted using ROhdsiWebApi::exportCohortDefinitionSet()
+#' @param databaseList             A list created by \code{createDatabaseList} to specify the databases
+#' @param sqliteLocation               (string) location of directory where the sqlite database will be saved
+#'    
+#' @return
+#' Returns the location of the sqlite database file
+#' 
+#' @export
+insertResultsToSqlite <- function(
+  resultLocation, 
+  cohortDefinitions,
+  databaseList = NULL,
+  sqliteLocation = file.path(resultLocation, 'sqlite')
+){
+  
+  if(!dir.exists(sqliteLocation)){
+    dir.create(sqliteLocation, recursive = T)
+  }
+  
+  ensure_installed('RSQLite')
+  
+  # create sqlite database
+  connectionDetails <- DatabaseConnector::createConnectionDetails(
+    dbms = 'sqlite',
+    server = file.path(sqliteLocation,'databaseFile.sqlite')
+  )
+  
+  # create tables if they dont exist
+  createPlpResultTables(
+      connectionDetails = connectionDetails,
+      targetDialect = 'sqlite',
+      resultSchema = 'main', 
+      deleteTables = T, 
+      createTables = T,
+      tablePrefix = ''
+    )
+
+  # run insert models
+  addMultipleRunPlpToDatabase(
+    connectionDetails = connectionDetails, 
+    databaseSchemaSettings = createDatabaseSchemaSettings(resultSchema = 'main'),
+    cohortDefinitions = cohortDefinitions,
+    databaseList = databaseList,
+    resultLocation = resultLocation,
+    modelSaveLocation = sqliteLocation
+  )
+  
+  # run insert diagnosis
+  addMultipleDiagnosePlpToDatabase(
+    connectionDetails = connectionDetails,
+    databaseSchemaSettings = createDatabaseSchemaSettings(resultSchema = 'main'),
+    cohortDefinitions = cohortDefinitions,
+    databaseList = databaseList,
+    resultLocation = resultLocation
+  )
+  
+  return(file.path(sqliteLocation,'databaseFile.sqlite'))
+}
+
+#' Create the results tables to store PatientLevelPrediction models and results into a database
+#' @description
+#' This function executes a large set of SQL statements to create tables that can store models and results
+#'
+#' @details
+#' This function can be used to create (or delete) PatientLevelPrediction result tables
+#'
+#' @param connectionDetails            The database connection details 
+#' @param targetDialect                The database management system being used
+#' @param resultSchema                 The name of the database schema that the result tables will be created.
+#' @param deleteTables                 If true any existing tables matching the PatientLevelPrediction result tables names will be deleted
+#' @param createTables                 If true the PatientLevelPrediction result tables will be created
+#' @param tablePrefix         A string that appends to the PatientLevelPrediction result tables
+#' @param tempEmulationSchema          The temp schema used when the database management system is oracle
+#' 
+#' @param testFile                     (used for testing) The location of an sql file with the table creation code
+#'
+#' @return
+#' Returns NULL but creates the required tables into the specified database schema(s).
+#' 
+#' @export
+createPlpResultTables <- function(
+    connectionDetails,
+  targetDialect = 'postgresql',
+  resultSchema, 
+  deleteTables = T, 
+  createTables = T,
+  tablePrefix = '',
+  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"),
+  testFile = NULL
+){
+  
+  conn <- DatabaseConnector::connect(connectionDetails = connectionDetails)
+  on.exit(DatabaseConnector::disconnect(conn))
+  
+  tablesExists <- sum(tolower(getPlpResultTables()) %in% tolower(DatabaseConnector::getTableNames(connection = conn, databaseSchema = resultSchema)))
+  tablesExists <- tablesExists == length(getPlpResultTables())
+  
+  if(!tablesExists){
+    ParallelLogger::logInfo('All or some PLP result tables do not exist, tables being recreated')
+    if(deleteTables){
+      ParallelLogger::logInfo('Deleting existing tables')
+      
+      tableNames <- getPlpResultTables()
+      
+      deleteTables(
+        conn = conn,
+        databaseSchema = resultSchema,
+        targetDialect = targetDialect,
+        tempEmulationSchema = tempEmulationSchema,
+        tableNames = tableNames, 
+        tablePrefix = tablePrefix
+      )
+      
+    }
+    
+    
+    if(createTables){
+      ParallelLogger::logInfo('Creating PLP results tables')
+      
+      if(tablePrefix != ''){
+        tablePrefix <- paste0(toupper(gsub('_','',gsub(' ','', tablePrefix))), '_')
+      }
+      
+      pathToSql <- system.file(
+        paste("sql/", targetDialect, 
+              sep = ""),
+        "PlpResultTables.sql", 
+        package = "PatientLevelPrediction"
+      )
+      
+      if(!file.exists(pathToSql)){
+        # if no dbms specific file use sql_server
+        pathToSql <- system.file(
+          paste("sql/", 'sql_server', 
+                sep = ""),
+          "PlpResultTables.sql", 
+          package = "PatientLevelPrediction"
+        )
+      }
+      
+      sql <- readChar(pathToSql, file.info(pathToSql)$size) 
+      renderedSql <- SqlRender::render(
+        sql = sql[1],
+        my_schema = resultSchema,
+        string_to_append = tablePrefix
+      )
+      renderedSql <- SqlRender::translate(
+        sql = renderedSql,
+        targetDialect = targetDialect,
+        tempEmulationSchema = tempEmulationSchema
+      )
+      
+      DatabaseConnector::executeSql(conn, renderedSql)
+    }
+    
+  } else{
+    ParallelLogger::logInfo('PLP result tables already exist')
+  }
+  
+  # then migrate
+  ParallelLogger::logInfo('PLP result migrration being applied')
+  migrateDataModel(
+    connectionDetails = connectionDetails, # input is connection
+    databaseSchema = resultSchema,
+    tablePrefix = tablePrefix
+  )
+  
+}
+
+#' Populate the PatientLevelPrediction results tables
+#' @description
+#' This function formats and uploads results that have been generated via an ATLAS prediction package into a database
+#'
+#' @details
+#' This function can be used upload PatientLevelPrediction results into a database
+#'
+#' @param connectionDetails            A connection details created by using the
+#'                                     function \code{createConnectionDetails} in the
+#'                                     \code{DatabaseConnector} package.
+#' @param databaseSchemaSettings       A object created by \code{createDatabaseSchemaSettings} with all the settings specifying the result tables                              
+#' @param cohortDefinitions            A set of one or more cohorts extracted using ROhdsiWebApi::exportCohortDefinitionSet()
+#' @param databaseList              (Optional) A list created by \code{createDatabaseList} to specify the databases
+#' @param resultLocation               (string) location of directory where the main package results were saved
+#' @param resultLocationVector         (only used when resultLocation is missing) a vector of locations with development or validation results  
+#' @param modelSaveLocation              The location of the file system for saving the models in a subdirectory  
+#'    
+#' @return
+#' Returns NULL but uploads all the results in resultLocation to the PatientLevelPrediction result tables in resultSchema
+#' 
+#' @export
+addMultipleRunPlpToDatabase <- function(
+    connectionDetails, 
+    databaseSchemaSettings = createDatabaseSchemaSettings(resultSchema = 'main'),
+    cohortDefinitions,
+    databaseList = NULL,
+    resultLocation = NULL,
+    resultLocationVector,
+    modelSaveLocation
+){
+  conn <- DatabaseConnector::connect(connectionDetails = connectionDetails)
+  on.exit(DatabaseConnector::disconnect(conn))
+  
+  # for each development result add it to the database:
+  
+  if(missing(resultLocationVector)){
+    resultLocationVector <- getResultLocations(resultLocation)
+  }
+  
+  if(length(resultLocationVector) == 0){
+    ParallelLogger::logInfo('No results found')
+    return(NULL)
+  }
+  
+  for(runPlpLocation in resultLocationVector){
+    ParallelLogger::logInfo(paste0('Inserting result @ ', runPlpLocation, ' into database'))
+    
+    # TODO edit csv here
+    runPlp <- tryCatch(
+      {PatientLevelPrediction::loadPlpResult(runPlpLocation)}, 
+      error = function(e){ParallelLogger::logInfo(e);return(NULL)}
+    )
+    
+    if(!is.null(runPlp)){
+      ParallelLogger::logInfo('result loaded')
+      
+      #  Add runPlp to the database
+      tryCatch(
+        {addRunPlpToDatabase(
+          runPlp = runPlp,
+          connectionDetails = connectionDetails,
+          databaseSchemaSettings = databaseSchemaSettings,
+          cohortDefinitions = cohortDefinitions,
+          databaseList = databaseList,
+          modelSaveLocation = modelSaveLocation
+        )}, error = function(e){
+          ParallelLogger::logInfo('result upload failed: '); 
+          ParallelLogger::logInfo(e)
+          }
+      )
+      
+    } #model not null 
+    
+  } # per model
+  
+} #end funct
+
+
+#' Create the PatientLevelPrediction database result schema settings
+#' @description
+#' This function specifies where the results schema is and lets you pick a different schema for the cohorts and databases
+#'
+#' @details
+#' This function can be used to specify the database settings used to upload PatientLevelPrediction results into a database
+#'
+#' @param resultSchema                 (string) The name of the database schema with the result tables.
+#' @param tablePrefix         (string) A string that appends to the PatientLevelPrediction result tables
+#' @param targetDialect                (string) The database management system being used
+#' @param tempEmulationSchema          (string) The temp schema used when the database management system is oracle
+#' @param cohortDefinitionSchema                 (string) The name of the database schema with the cohort definition tables (defaults to resultSchema).
+#' @param tablePrefixCohortDefinitionTables         (string) A string that appends to the cohort definition tables
+#' @param databaseDefinitionSchema                 (string) The name of the database schema with the database definition tables (defaults to resultSchema).
+#' @param tablePrefixDatabaseDefinitionTables         (string) A string that appends to the database definition tables
+#'    
+#' @return
+#' Returns a list of class 'plpDatabaseResultSchema' with all the database settings
+#' 
+#' @export
+createDatabaseSchemaSettings <- function(
+  resultSchema = 'main', 
+  tablePrefix = '',
+  targetDialect = 'sqlite',
+  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"),
+  cohortDefinitionSchema = resultSchema,
+  tablePrefixCohortDefinitionTables = tablePrefix,
+  databaseDefinitionSchema = resultSchema,
+  tablePrefixDatabaseDefinitionTables = tablePrefix
+){
+  
+  if(missing(resultSchema)){
+    stop('resultSchema required')
+  }
+  if(!inherits(x = resultSchema, what = "character")){
+    stop('resultSchema must be a string')
+  }
+  
+  if(tablePrefix != ''){
+    tablePrefix <- paste0(toupper(gsub('_','',gsub(' ','', tablePrefix))), '_')
+  }
+  if(tablePrefixCohortDefinitionTables != ''){
+    tablePrefixCohortDefinitionTables <- paste0(toupper(gsub('_','',gsub(' ','', tablePrefixCohortDefinitionTables))), '_')
+  }
+  if(tablePrefixDatabaseDefinitionTables != ''){
+    tablePrefixDatabaseDefinitionTables <- paste0(toupper(gsub('_','',gsub(' ','', tablePrefixDatabaseDefinitionTables))), '_')
+  }
+  
+  result <- list(
+    resultSchema = resultSchema,
+    tablePrefix = tablePrefix,
+    targetDialect = targetDialect,
+    tempEmulationSchema = tempEmulationSchema,
+    cohortDefinitionSchema  = cohortDefinitionSchema, # could be removed
+    tablePrefixCohortDefinitionTables = tablePrefixCohortDefinitionTables, # could be removed
+    databaseDefinitionSchema = databaseDefinitionSchema, # could be removed
+    tablePrefixDatabaseDefinitionTables = tablePrefixDatabaseDefinitionTables # could be removed
+  )
+  
+  class(result) <- 'plpDatabaseResultSchema'
+  return(result)
+}
+
+
+#' Create a list with the database details and database meta data entries
+#' @description
+#' This function creates a list with the database details and database meta data entries used in the study
+#'
+#' @details
+#' This function is used when inserting database details into the PatientLevelPrediction database results schema
+#' 
+#' @param cdmDatabaseSchemas           (string vector) A vector of the cdmDatabaseSchemas used in the study - if the schemas are not unique per database please also specify databaseRefId
+#' @param cdmDatabaseNames             Sharable names for the databases
+#' @param databaseRefIds               (string vector) Unique database identifiers - what you specified as cdmDatabaseId in \code{PatientLevelPrediction::createDatabaseDetails()} when developing the models
+#'
+#' @return
+#' Returns a data.frame with the database details
+#' 
+#' @export
+createDatabaseList <- function(
+  cdmDatabaseSchemas,
+  cdmDatabaseNames,
+  databaseRefIds = NULL
+){
+  if(missing(cdmDatabaseSchemas)){
+    stop('Need to specify cdmDatabaseSchemas')
+  }
+  
+  if(is.null(databaseRefIds)){
+    ParallelLogger::logInfo('No databaseRefId specified so using schema as unique database identifier')
+    databaseRefIds <- removeInvalidString(cdmDatabaseSchemas)
+  }
+  if(missing(cdmDatabaseNames)){
+    cdmDatabaseNames <- removeInvalidString(cdmDatabaseSchemas)
+  }
+  
+  
+  result <- lapply(
+    1:length(cdmDatabaseSchemas),
+    
+    function(i) list(
+      databaseDetails = list(
+        databaseMetaDataId = databaseRefIds[i]
+      ),
+      databaseMetaData = list(
+        databaseId = databaseRefIds[i],
+        cdmSourceName = cdmDatabaseSchemas[i],
+        cdmSourceAbbreviation = cdmDatabaseNames[i],
+        cdmHolder = '', # could get this from CDM_source inside runPlp in future
+        sourceDesciption = '',
+        sourceDocumentReference = '',
+        cdmEtlReference = '', 
+        sourceReleaseDate = '',
+        cdmReleaseDate = '',
+        cdmVersion = '',
+        vocabularyVersion = '',
+        maxObsPeriodEndDate = ''
+      )
+    )
+  )
+  
+  names(result) <- databaseRefIds #cdmDatabaseSchemas
+  # using id as schema may not be unique
+  # id uses schema if it is not set
+
+  return(result)
+}
+
+
+
+#' Function to add the run plp (development or validation) to database
+#' @description
+#' This function adds a runPlp or external validation result into a database
+#'
+#' @details
+#' This function is used when inserting results into the PatientLevelPrediction database results schema
+#' 
+#' @param runPlp           An object of class \code{runPlp} or class \code{externalValidatePlp}
+#' @param connectionDetails            A connection details created by using the
+#'                                     function \code{createConnectionDetails} in the
+#'                                     \code{DatabaseConnector} package.
+#' @param databaseSchemaSettings       A object created by \code{createDatabaseSchemaSettings} with all the settings specifying the result tables                              
+#' @param cohortDefinitions            A set of one or more cohorts extracted using ROhdsiWebApi::exportCohortDefinitionSet()
+#' @param modelSaveLocation         The location of the directory that models will be saved to
+#' @param databaseList              (Optional) If you want to change the database name then used \code{createDatabaseList} to specify the database settings but use the same cdmDatabaseId was model development/validation
+#' 
+#' @return
+#' Returns a data.frame with the database details
+#' 
+#' @export
+addRunPlpToDatabase <- function(
+  runPlp,
+  connectionDetails,
+  databaseSchemaSettings,
+  cohortDefinitions,
+  modelSaveLocation,
+  databaseList = NULL
+){
+  
+  conn <- DatabaseConnector::connect(connectionDetails = connectionDetails)
+  on.exit(DatabaseConnector::disconnect(conn))
+  
+  modelDesignId <- insertModelDesignInDatabase(
+    object = runPlp$model$modelDesign, 
+    conn = conn, 
+    databaseSchemaSettings = databaseSchemaSettings,
+    cohortDefinitions = cohortDefinitions 
+  )
+  
+  # Add model if runPlp
+  if(inherits(runPlp, 'runPlp')){
+    includesModel <- T
+    developmentDatabase <- runPlp$model$trainDetails$developmentDatabase
+    validationDatabase <- runPlp$model$trainDetails$developmentDatabase
+    developmentDatabaseRefId <- runPlp$model$trainDetails$developmentDatabaseId
+    validationDatabaseRefId <- runPlp$model$trainDetails$developmentDatabaseId
+    
+    populationSettings <- runPlp$model$modelDesign$populationSettings
+    targetId <- runPlp$model$modelDesign$targetId
+    outcomeId <- runPlp$model$modelDesign$outcomeId
+    restrictPlpDataSettings <- runPlp$model$modelDesign$restrictPlpDataSettings
+    
+    modelDevelopment <- 1 #added
+    
+    attrition <- runPlp$model$trainDetails$attrition
+
+  } else{
+    includesModel <- F
+    developmentDatabase <- runPlp$model$validationDetails$developmentDatabase
+    validationDatabase <- runPlp$model$validationDetails$validationDatabase
+    developmentDatabaseRefId <- runPlp$model$validationDetails$developmentDatabaseId
+    validationDatabaseRefId <- runPlp$model$validationDetails$validationDatabaseId
+    
+    populationSettings <- runPlp$model$validationDetails$populationSettings
+    targetId <- runPlp$model$validationDetails$targetId
+    outcomeId <- runPlp$model$validationDetails$outcomeId
+    restrictPlpDataSettings <- runPlp$model$validationDetails$restrictPlpDataSettings
+    
+    modelDevelopment <- 0 #added
+    
+    attrition <- runPlp$model$validationDetails$attrition
+    
+  }
+  
+  # Add databases
+  developmentDatabaseId <- addDatabase(
+    conn = conn, 
+    databaseSchemaSettings = databaseSchemaSettings,
+    databaseList = databaseList,
+    databaseSchema = developmentDatabase, 
+    databaseId = developmentDatabaseRefId
+  )
+  
+  validationDatabaseId <- addDatabase(
+    conn = conn, 
+    databaseSchemaSettings = databaseSchemaSettings,
+    databaseList = databaseList,
+    databaseSchema = validationDatabase,
+    databaseId = validationDatabaseRefId
+  )
+  
+  
+  # add nodel if the result contains it
+  if(includesModel){
+    insertModelInDatabase(
+      model = runPlp$model,
+      conn = conn,
+      databaseSchemaSettings = databaseSchemaSettings,
+      databaseId = developmentDatabaseId,
+      modelDesignId = modelDesignId,
+      modelSaveLocation = modelSaveLocation
+    )
+  }
+  
+  # get validation settings
+  validationTarId <- addTar(
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$resultSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    startDay = populationSettings$riskWindowStart, 
+    startAnchor = populationSettings$startAnchor,
+    endDay = populationSettings$riskWindowEnd,  
+    endAnchor = populationSettings$endAnchor, 
+    tablePrefix = databaseSchemaSettings$tablePrefix,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  
+  validationTargetId <-  addCohort(
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$cohortDefinitionSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    cohortDefinition = getCohortDef(cohortDefinitions,targetId),
+    tablePrefix = databaseSchemaSettings$tablePrefixCohortDefinitionTables,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  
+  validationOutcomeId <-  addCohort(
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$cohortDefinitionSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    cohortDefinition = getCohortDef(cohortDefinitions,outcomeId),
+    tablePrefix = databaseSchemaSettings$tablePrefixCohortDefinitionTables,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  
+  validationPopulationId <- addPopulationSetting(
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$resultSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    json = populationSettings, 
+    tablePrefix = databaseSchemaSettings$tablePrefix,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  
+  validationPlpDataId <-  addPlpDataSetting(
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$resultSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    json = restrictPlpDataSettings, 
+    tablePrefix = databaseSchemaSettings$tablePrefix,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  
+  # Add performance
+  insertPerformanceInDatabase(
+    performanceEvaluation = runPlp$performanceEvaluation,
+    covariateSummary = runPlp$covariateSummary,
+    attrition = attrition,
+    executionDateTime = format(runPlp$executionSummary$ExecutionDateTime, format="%Y-%m-%d"),
+    plpVersion = runPlp$executionSummary$PackageVersion$packageVersion, 
+    conn = conn,
+    databaseSchemaSettings = databaseSchemaSettings,
+    
+    modelDesignId = modelDesignId,
+    developmentDatabaseId = developmentDatabaseId,
+    
+    validationDatabaseId = validationDatabaseId,
+    validationTarId = validationTarId,
+    validationPopulationId= validationPopulationId,
+    validationPlpDataId = validationPlpDataId,
+    validationTargetId = validationTargetId,
+    validationOutcomeId = validationOutcomeId,
+    
+    modelDevelopment = modelDevelopment
+  )
+  
+  return(invisible(NULL))
+}
+
+
+
+###################
+# INSERT MODEL
+####################
+insertModelInDatabase <- function(
+  model,
+  conn,
+  databaseSchemaSettings,
+  databaseId,
+  modelDesignId,
+  modelSaveLocation
+){
+  
+  # save the model to the file system
+  modelLocation <- file.path(
+    modelSaveLocation, 
+    'models',
+    paste0('folder-', modelDesignId, '-', databaseId)
+    )
+  if(!dir.exists(modelLocation)){
+    dir.create(modelLocation, recursive = T)
+  }
+  # savign all the model as preprocessing was too large
+  # for database
+  savePlpModel(
+    plpModel = model, 
+    dirPath = modelLocation
+    )
+  
+  #saveModelPart(
+  #  model = model$model,
+  #  savetype = attr(model, 'saveType'),
+  #  dirPath = modelLocation
+  #)
+  
+  # need hyperParamSearch for shiny app but the other parts
+  # are too large to store into the database
+  trainDetails <- list(hyperParamSearch = model$trainDetails$hyperParamSearch)
+  
+    # create this function
+    modelId <- addModel(
+      conn = conn, 
+      resultSchema = databaseSchemaSettings$resultSchema, 
+      targetDialect = databaseSchemaSettings$targetDialect,
+      analysisId = ifelse(
+        is.null(model$trainDetails$analysisId), 
+        'missing',
+        model$trainDetails$analysisId
+      ), 
+      modelDesignId = modelDesignId,
+      databaseId = databaseId,
+      modelType = model$trainDetails$modelName,
+      plpModelFile = modelLocation, # save the model to a location and add location here
+      trainDetails = as.character(ParallelLogger::convertSettingsToJson(trainDetails)),
+      preprocessing = "",#as.character(ParallelLogger::convertSettingsToJson(model$preprocessing)),
+      
+      executionDateTime = format(model$trainDetails$trainingDate, format="%Y-%m-%d"), 
+      trainingTime = model$trainDetails$trainingTime, 
+      intercept = ifelse(is.list(model$model) & attr(model, 'saveType') != 'xgboost', model$model$coefficients$betas[1], 0),  # using the param useIntercept?
+      
+      tablePrefix = databaseSchemaSettings$tablePrefix,
+      tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+    )
+    ParallelLogger::logInfo(
+      paste0('modelId: ', modelId, 
+             ' inserted for modelDesignId ', modelDesignId, 
+             'and databaseId ', databaseId)
+    )
+    
+  return(invisible(modelId))
+}
+
+addModel <- function(
+  conn, 
+  resultSchema, 
+  targetDialect,
+  tablePrefix,
+  analysisId,
+  modelDesignId,
+  databaseId,
+  
+  modelType,
+  plpModelFile,
+  trainDetails,
+  preprocessing,
+  
+  executionDateTime,
+  trainingTime,
+  intercept,
+  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
+){
+  
+  if(is.null(analysisId)){
+    stop('analysisId is null')
+  }
+  if(is.null(modelDesignId)){
+    stop('modelDesignId is null')
+  }
+  if(is.null(databaseId)){
+    stop('databaseId is null')
+  }
+  
+  if(is.null(plpModelFile)){
+    stop('plpModelFile is null')
+  }
+  if(is.null(executionDateTime)){
+    stop('executionDateTime is null')
+  }
+  if(is.null(intercept)){
+    stop('intercept is null')
+  }
+  
+  # process json to make it ordered...
+  # TODO
+  
+  result <- checkTable(
+    conn = conn, 
+    resultSchema = resultSchema, 
+    tablePrefix = tablePrefix,
+    targetDialect = targetDialect, 
+    tableName = 'models',
+    columnNames = c(
+      'model_design_id',
+      'database_id'
+    ), 
+    values = c(
+      modelDesignId,
+      databaseId
+    ),
+    tempEmulationSchema = tempEmulationSchema
+  )
+  
+  if(nrow(result)==0){
+    # model
+    sql <- "INSERT INTO @my_schema.@string_to_appendmodels(
+    analysis_id,
+    model_design_id,
+    database_id,
+    model_type,
+    plp_model_file,
+    train_details,
+    preprocessing,
+    execution_date_time,
+    training_time,
+    intercept
+    ) VALUES 
+  ('@analysis_id', 
+  @model_design_id,
+  @database_id, 
+  
+  '@model_type',
+  '@plp_model_file', 
+  '@train_details', 
+  '@preprocessing', 
+      
+  '@execution_date_time', 
+  '@training_time', 
+   @intercept
+  )"
+    sql <- SqlRender::render(
+      sql, 
+      my_schema = resultSchema,
+      analysis_id = analysisId,
+      model_design_id = modelDesignId,
+      database_id = databaseId,
+      
+      model_type = modelType,
+      plp_model_file = plpModelFile,
+      train_details = trainDetails,
+      preprocessing = preprocessing,
+      
+      execution_date_time = executionDateTime,
+      training_time = trainingTime,
+      intercept = intercept,
+      
+      string_to_append = tablePrefix
+    )
+    sql <- SqlRender::translate(
+      sql, 
+      targetDialect = targetDialect,
+      tempEmulationSchema = tempEmulationSchema
+    )
+    DatabaseConnector::executeSql(conn, sql)
+    
+    #getId of new
+    result <- checkTable(
+      conn = conn, 
+      resultSchema = resultSchema, 
+      tablePrefix = tablePrefix,
+      targetDialect = targetDialect, 
+      tableName = 'models',
+      columnNames = c(
+        'model_design_id',
+        'database_id'
+      ), 
+      values = c(
+        modelDesignId,
+        databaseId
+      ),
+      tempEmulationSchema = tempEmulationSchema
+    )
+    
+  } 
+  
+  return(result$modelId[1])
+}
+
+#======================
+# Helpers
+#======================
+
+# get a vector with all the result table names
+getPlpResultTables <- function(){
+  return(
+    c(
+      "CALIBRATION_SUMMARY", 
+      "COVARIATE_SUMMARY", 
+      "DEMOGRAPHIC_SUMMARY",
+      "EVALUATION_STATISTICS", 
+      "PREDICTION_DISTRIBUTION", 
+      "THRESHOLD_SUMMARY",
+      
+      "ATTRITION", #new 
+      
+      "DIAGNOSTIC_SUMMARY",
+      "DIAGNOSTIC_PARTICIPANTS",
+      "DIAGNOSTIC_PREDICTORS",
+      "DIAGNOSTIC_OUTCOMES",
+      "DIAGNOSTIC_DESIGNS",
+      
+      "DIAGNOSTICS", #new 
+      "RECALIBRATIONS", #new 
+      
+      "PERFORMANCES", 
+      
+      "MODELS", 
+      
+      "MODEL_DESIGNS",  
+      
+      "MODEL_SETTINGS", 
+      "COVARIATE_SETTINGS",
+      "POPULATION_SETTINGS", 
+      "FEATURE_ENGINEERING_SETTINGS", 
+      "SPLIT_SETTINGS", 
+      "PLP_DATA_SETTINGS", #new
+      "SAMPLE_SETTINGS", 
+      "TIDY_COVARIATES_SETTINGS", #new 
+      "TARS", 
+      
+      "DATABASE_DETAILS",
+      "DATABASE_META_DATA",
+      "COHORT_DEFINITION",
+      "COHORTS"
+    )
+  )
+}
+
+getResultLocations <- function(resultLocation){
+  # get the model locations...
+  
+  resultLocs <- dir(
+      resultLocation, 
+      pattern = 'Analysis_', 
+      full.names = T
+  )
+  # automatically find Results folder, to handle both plpResult/ and validationResult/
+  resultLocs <- file.path(resultLocs, dir(resultLocs, pattern='Result'))
+  
+  
+  if(dir.exists(file.path(resultLocation, 'Validation'))){
+    validationDatabases <- dir(file.path(resultLocation, 'Validation'))
+    
+    valLocs <- dir(
+      unlist(
+        lapply(
+          validationDatabases,
+          function(x) dir(file.path(resultLocation, 'Validation', x), 
+                          pattern = 'Analysis_', 
+                          full.names = T
+          )
+        )
+      ),
+      full.names = T
+    )
+    
+    resultLocs <- c(resultLocs, valLocs)
+    
+  }
+  return(resultLocs)
+  
+}
+
+deleteTables <- function(
+  conn,
+  databaseSchema,
+  targetDialect,
+  tempEmulationSchema,
+  tableNames, 
+  tablePrefix
+){
+  
+  if(tablePrefix != ''){
+    tableNames <- tolower(paste0(gsub('_','',gsub(' ','', tablePrefix)), '_', tableNames))
+  }
+  
+  alltables <- tolower(DatabaseConnector::getTableNames(
+    connection = conn, 
+    databaseSchema = databaseSchema
+  ))
+  
+  
+  for(tb in tableNames){
+    if(tb %in% alltables){
+      
+      if(targetDialect != 'sqlite'){
+        sql <- 'TRUNCATE TABLE @my_schema.@table'
+        sql <- SqlRender::render(
+          sql, 
+          my_schema = databaseSchema, 
+          table = tb
+        )
+        sql <- SqlRender::translate(
+          sql, 
+          targetDialect = targetDialect, 
+          tempEmulationSchema = tempEmulationSchema
+        )
+        DatabaseConnector::executeSql(conn, sql)
+      } else{
+        sql <- 'DELETE FROM @my_schema.@table'
+        sql <- SqlRender::render(
+          sql, 
+          my_schema = databaseSchema, 
+          table = tb
+        )
+        sql <- SqlRender::translate(
+          sql, 
+          targetDialect = targetDialect,
+          tempEmulationSchema = tempEmulationSchema
+        )
+        DatabaseConnector::executeSql(conn, sql)
+      }
+        sql <- 'DROP TABLE @my_schema.@table'
+        sql <- SqlRender::render(
+          sql, 
+          my_schema = databaseSchema, 
+          table = tb
+        )
+        sql <- SqlRender::translate(
+          sql, targetDialect = targetDialect,
+          tempEmulationSchema = tempEmulationSchema
+        )
+        DatabaseConnector::executeSql(conn, sql)
+    }
+    
+  }
+  
+}
+
+
+## Template Helpers
+
+enc <- function(x){
+  return(paste0("'", x, "'"))
+}
+
+cleanNum <- function(x){
+  types <- unlist(lapply(1:ncol(x), function(i) class(x[,i])))
+  
+  ids <- which(types%in% c("numeric", "integer" ))
+  
+  for(id in ids){
+    okVals <- is.finite(x[,id])
+    
+    if(sum(okVals)!=length(okVals)){
+      x[!okVals,id] <- NA
+    }
+    
+  }
+  return(x)  
+}
+
+checkTable <- function(conn,
+                       resultSchema, 
+                       tablePrefix = '',
+                       targetDialect,
+                       tableName,
+                       columnNames, 
+                       values,
+                       tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
+){
+  
+  vals <- paste0(paste0(columnNames," = ", values), collapse = " and ")
+  
+  sql <- "SELECT * from @my_schema.@string_to_append@table where @input_vals;"
+  sql <- SqlRender::render(sql, 
+                           my_schema = resultSchema,
+                           table = tableName,
+                           input_vals = vals,
+                           string_to_append = tablePrefix)
+  sql <- SqlRender::translate(sql, targetDialect = targetDialect,
+                              tempEmulationSchema = tempEmulationSchema)
+  result <- DatabaseConnector::querySql(conn, sql, snakeCaseToCamelCase = T)
+  
+  return(result)
+}
+
+
+checkJson <- function(conn,
+                      resultSchema, 
+                      tablePrefix = '',
+                      targetDialect,
+                      tableName,
+                      jsonColumnName,
+                      id,
+                      json,
+                      tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
+){
+  
+  sql <- "SELECT * from @my_schema.@string_to_append@table;"
+  sql <- SqlRender::render(sql, 
+                           my_schema = resultSchema,
+                           table = tableName,
+                           string_to_append = tablePrefix)
+  sql <- SqlRender::translate(sql, targetDialect = targetDialect,
+                              tempEmulationSchema = tempEmulationSchema)
+  result <- DatabaseConnector::querySql(conn, sql, snakeCaseToCamelCase = T)
+  
+  resultId <- NULL
+  if(nrow(result)>0){
+    colId <- result[,jsonColumnName] == json
+    if(sum(colId)>0){
+      resultId <- result[colId,id][1]
+    }
+  }
+  
+  return(resultId)
+}
+
+
+getCohortDef <- function(cohortDefinitions, cohortId){
+  if(!is.null(cohortDefinitions)){
+    if(sum(cohortDefinitions$cohortId == cohortId) > 0){
+      return(cohortDefinitions[cohortDefinitions$cohortId == cohortId, ])
+    }
+  }
+  return(
+    data.frame(
+      cohortId = cohortId, 
+      cohortName = paste0('Cohort: ', cohortId),
+      json = '{}'
+    )
+  )
+}
+
+
+# adds json from package unless json is specified
+addCohort <- function(
+  conn, 
+  resultSchema, 
+  targetDialect,
+  tablePrefix = '',
+  cohortDefinition, # this is the R data.frame of the cohortDefinition
+  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
+){
+  
+  
+  # make sure the json has been converted 
+  json <- cohortDefinition$json
+  if(!inherits(x = json , what = 'character')){
+    ParallelLogger::logInfo('converting json to character')
+    json <- as.character(json) # now convert to character
+  }
+  
+  # reduce the size to save
+  if(!targetDialect %in% c('sqlite', 'postgres')){
+    json <-  substr(json, 1, 4000) # TESTING - FIX THIS [TODO]
+  }
+  
+  #check whether cohort already in COHORT_DEFINITION table:
+  result <- checkTable(conn = conn, 
+                       resultSchema = resultSchema, 
+                       tablePrefix = tablePrefix,
+                       targetDialect = targetDialect, 
+                       tableName = 'cohort_definition',
+                       columnNames = c('cohort_name'), 
+                       values = c(paste0("'",gsub('\'', '', cohortDefinition$cohortName),"'")),
+                       tempEmulationSchema = tempEmulationSchema
+  )
+  
+  addNew <- F
+  if(nrow(result)>0){
+    addNew <- json %in% result$json
+    ParallelLogger::logInfo(paste0('json in jsons:', addNew))
+  }
+  
+  if(addNew){
+    cohortDefinitionId <- result$cohortDefinitionId[result$json %in% json]
+    ParallelLogger::logInfo(paste0('Cohort ',gsub('\'', '', cohortDefinition$cohortName),' exists in cohort_definition with cohort id', result$cohortDefinitionId[result$json %in% json]))
+  } else{
+    ParallelLogger::logInfo(paste0('Adding cohort ',gsub('\'', '', cohortDefinition$cohortName)))
+    
+    data <- data.frame(
+      cohortName = gsub('\'', '', cohortDefinition$cohortName), 
+      cohortDefinitionId = cohortDefinition$cohortId,
+      json = json
+    )
+    DatabaseConnector::insertTable(
+      connection = conn, 
+      databaseSchema = resultSchema, 
+      tableName = paste0(tablePrefix, 'cohort_definition'),
+      data = data,
+      dropTableIfExists = F, 
+      createTable = F, 
+      tempTable = F, 
+      progressBar = T,
+      camelCaseToSnakeCase = T, 
+      tempEmulationSchema = tempEmulationSchema
+    )
+    
+    # now check and get id
+    result <- checkTable(
+      conn = conn, 
+      resultSchema = resultSchema, 
+      tablePrefix = tablePrefix,
+      targetDialect = targetDialect, 
+      tableName = 'cohort_definition',
+      columnNames = c('cohort_name', 'cohort_definition_id'), 
+      values = c(paste0("'",gsub('\'', '', cohortDefinition$cohortName),"'"), cohortDefinition$cohortId),
+      tempEmulationSchema = tempEmulationSchema
+    )
+    
+    jsonInd <- result$json %in% json
+    cohortDefinitionId <- result$cohortDefinitionId[jsonInd]
+  }
+  
+  # now add to cohorts table
+  result <- checkTable(conn = conn, 
+                       resultSchema = resultSchema, 
+                       tablePrefix = tablePrefix,
+                       targetDialect = targetDialect, 
+                       tableName = 'cohorts',
+                       columnNames = c('cohort_definition_id','cohort_name'), 
+                       values = c(cohortDefinitionId, paste0("'",gsub('\'', '', cohortDefinition$cohortName),"'")),
+                       tempEmulationSchema = tempEmulationSchema
+  )
+  
+  if(nrow(result)>0){
+    ParallelLogger::logInfo(paste0('Cohort ',gsub('\'', '', cohortDefinition$cohortName),' exists in cohorts with cohort id', result$cohortId))
+  } else{
+    ParallelLogger::logInfo(paste0('Adding cohort ',gsub('\'', '', cohortDefinition$cohortName)))
+    
+    data <- data.frame(
+      cohortDefinitionId = cohortDefinitionId,
+      cohortName = gsub('\'', '', cohortDefinition$cohortName)
+    )
+    DatabaseConnector::insertTable(
+      connection = conn, 
+      databaseSchema = resultSchema, 
+      tableName = paste0(tablePrefix, 'cohorts'),
+      data = data,
+      dropTableIfExists = F, 
+      createTable = F, 
+      tempTable = F, 
+      progressBar = T,
+      camelCaseToSnakeCase = T, 
+      tempEmulationSchema = tempEmulationSchema
+    )
+    
+    # now check and get id
+    result <- checkTable(conn = conn, 
+                         resultSchema = resultSchema, 
+                         tablePrefix = tablePrefix,
+                         targetDialect = targetDialect, 
+                         tableName = 'cohorts',
+                         columnNames = c('cohort_definition_id','cohort_name'), 
+                         values = c(cohortDefinitionId, paste0("'",gsub('\'', '', cohortDefinition$cohortName),"'")),
+                         tempEmulationSchema = tempEmulationSchema
+    )
+  }
+  
+  return(result$cohortId[1])
+}
+
+
+
+addDatabase <- function(
+  conn, 
+  databaseSchemaSettings,
+  databaseList = NULL, # list with the database details
+  databaseId = NULL, # the database id
+  databaseSchema # the database schema
+){
+  
+  if(is.null(databaseId)){
+    databaseId <- removeInvalidString(databaseSchema)
+  }
+  
+  # get the database tables for the databaseId 
+  if(is.null(databaseList)){
+    databaseDataFrames <- createDatabaseList(cdmDatabaseSchemas = databaseSchema, databaseRefIds = databaseId)[[1]]
+  } else{
+    if(databaseId %in% names(databaseList)){
+      databaseDataFrames <- databaseList[[as.character(databaseId)]]
+    } else{
+      ParallelLogger::logInfo('database ID not found in databaseList so added new entry')
+      databaseDataFrames <- createDatabaseList(cdmDatabaseSchemas = databaseSchema, databaseRefIds = databaseId)[[1]]
+    }
+  }
+  
+  
+  # check the database_meta_data
+  result <- checkTable(conn = conn, 
+                       resultSchema = databaseSchemaSettings$resultSchema, 
+                       tablePrefix = databaseSchemaSettings$tablePrefix,
+                       targetDialect = databaseSchemaSettings$targetDialect, 
+                       tableName = 'database_meta_data',
+                       columnNames = c('database_id'), 
+                       values = c(paste0("'",databaseDataFrames$databaseMetaData$databaseId,"'")),
+                       tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  
+  if(nrow(result)>0){
+    ParallelLogger::logInfo(paste0('Database meta data ', databaseDataFrames$database_meta_data$databaseId ,' already exists'))
+  } else {
+    
+    sql <- "INSERT INTO @my_schema.@string_to_appenddatabase_meta_data(
+      database_id,
+      cdm_source_name,
+      cdm_source_abbreviation
+    ) 
+          VALUES ('@database_id','@cdm_source_name', '@cdm_source_abbreviation');"
+    sql <- SqlRender::render(
+      sql, 
+      my_schema = databaseSchemaSettings$resultSchema,
+      database_id = databaseDataFrames$databaseMetaData$databaseId, 
+      cdm_source_name = databaseDataFrames$databaseMetaData$cdmSourceName,
+      cdm_source_abbreviation = databaseDataFrames$databaseMetaData$cdmSourceAbbreviation,
+      string_to_append = databaseSchemaSettings$tablePrefix
+    )
+    sql <- SqlRender::translate(sql, targetDialect = databaseSchemaSettings$targetDialect,
+                                tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema)
+    DatabaseConnector::executeSql(conn, sql)
+    
+    result <- checkTable(conn = conn, 
+                         resultSchema = databaseSchemaSettings$resultSchema, 
+                         tablePrefix = databaseSchemaSettings$tablePrefix,
+                         targetDialect = databaseSchemaSettings$targetDialect, 
+                         tableName = 'database_meta_data',
+                         columnNames = c('database_id', 'cdm_source_name',
+                                         'cdm_source_abbreviation'), 
+                         values = c(paste0("'",databaseDataFrames$databaseMetaData$databaseId,"'"), 
+                                    paste0("'",databaseDataFrames$databaseMetaData$cdmSourceName,"'"),
+                                    paste0("'",databaseDataFrames$databaseMetaData$cdmSourceAbbreviation,"'")),
+                         tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+    )
+    
+  }
+  
+  result <- checkTable(conn = conn, 
+                       resultSchema = databaseSchemaSettings$resultSchema, 
+                       tablePrefix = databaseSchemaSettings$tablePrefix,
+                       targetDialect = databaseSchemaSettings$targetDialect, 
+                       tableName = 'database_details',
+                       columnNames = c('database_meta_data_id'), 
+                       values = c(paste0("'",databaseDataFrames$databaseDetails$databaseMetaDataId,"'")
+                                  ),
+                       tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  
+  if(nrow(result)>0){
+    ParallelLogger::logInfo(paste0('Database', result$databaseId ,' already exists'))
+  } else {
+    
+    sql <- "INSERT INTO @my_schema.@string_to_appenddatabase_details(database_meta_data_id) 
+          VALUES ('@database_meta_data_id');"
+    sql <- SqlRender::render(sql, 
+                             my_schema = databaseSchemaSettings$resultSchema,
+                             database_meta_data_id = databaseDataFrames$databaseDetails$databaseMetaDataId, 
+                             string_to_append = databaseSchemaSettings$tablePrefix)
+    sql <- SqlRender::translate(sql, targetDialect = databaseSchemaSettings$targetDialect,
+                                tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema)
+    DatabaseConnector::executeSql(conn, sql)
+    
+    result <- checkTable(conn = conn, 
+                         resultSchema = databaseSchemaSettings$resultSchema, 
+                         tablePrefix = databaseSchemaSettings$tablePrefix,
+                         targetDialect = databaseSchemaSettings$targetDialect, 
+                         tableName = 'database_details',
+                         columnNames = c('database_meta_data_id'), 
+                         values = c(paste0("'",databaseDataFrames$databaseDetails$databaseMetaDataId,"'")
+                         ),
+                         tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+    )
+    
+  }
+  
+  return(result$databaseId[1])
+  
+}
+
+
+
+
+
+
+
+
+
diff --git a/R/uploadToDatabaseDiagnostics.R b/R/uploadToDatabaseDiagnostics.R
new file mode 100644
index 000000000..0d10506e0
--- /dev/null
+++ b/R/uploadToDatabaseDiagnostics.R
@@ -0,0 +1,369 @@
+#' Insert mutliple diagnosePlp results saved to a directory into a PLP result schema database
+#' @description
+#' This function inserts diagnosePlp results into the result schema
+#'
+#' @details
+#' This function can be used to upload diagnosePlp results into a database
+#'
+#' @param connectionDetails            A connection details created by using the
+#'                                     function \code{createConnectionDetails} in the
+#'                                     \code{DatabaseConnector} package.
+#' @param databaseSchemaSettings       A object created by \code{createDatabaseSchemaSettings} with all the settings specifying the result tables                              
+#' @param cohortDefinitions            (list) A list of cohortDefinitions (each list must contain: name, id)
+#' @param databaseList              (Optional) ...
+#' @param resultLocation          The location of the diagnostic results
+#'    
+#' @return
+#' Returns NULL but uploads multiple diagnosePlp results into the database schema specified in databaseSchemaSettings
+#' 
+#' @export
+addMultipleDiagnosePlpToDatabase <- function(
+  connectionDetails,
+  databaseSchemaSettings,
+  cohortDefinitions,
+  databaseList = NULL,
+  resultLocation
+){
+  
+  diagnosisFiles <- file.path(resultLocation, dir(resultLocation, pattern = 'Analysis_'), 'diagnosePlp.rds')
+  
+  if(length(diagnosisFiles) == 0){
+    ParallelLogger::logInfo('No diagnostic results found')
+    return(NULL)
+  }
+  
+  for(diagnosisFile in diagnosisFiles){
+    if(file.exists(diagnosisFile)){
+      diagnosePlp <- readRDS(diagnosisFile)
+      addDiagnosePlpToDatabase(
+        diagnosePlp = diagnosePlp,
+        connectionDetails = connectionDetails,
+        databaseSchemaSettings = databaseSchemaSettings,
+        cohortDefinitions = cohortDefinitions,
+        databaseList = databaseList
+      )
+    }
+  }
+  return(invisible(NULL))
+}
+
+#' Insert a diagnostic result into a PLP result schema database
+#' @description
+#' This function inserts a diagnostic result into the result schema
+#'
+#' @details
+#' This function can be used to upload a diagnostic result into a database
+#'
+#' @param diagnosePlp                  An object of class \code{diagnosePlp} 
+#' @param connectionDetails            A connection details created by using the
+#'                                     function \code{createConnectionDetails} in the
+#'                                     \code{DatabaseConnector} package.
+#' @param databaseSchemaSettings       A object created by \code{createDatabaseSchemaSettings} with all the settings specifying the result tables                              
+#' @param cohortDefinitions            A set of one or more cohorts extracted using ROhdsiWebApi::exportCohortDefinitionSet()
+#' @param databaseList              (Optional) If you wish to overwrite the settings in the plp object use \code{createdatabaseList} to specify the databases
+#' @param overWriteIfExists            (default: T) Whether to delete existing results and overwrite them
+#'    
+#' @return
+#' Returns NULL but uploads the diagnostic into the database schema specified in databaseSchemaSettings
+#' 
+#' @export
+addDiagnosePlpToDatabase <- function(
+  diagnosePlp,
+  connectionDetails,
+  databaseSchemaSettings,
+  cohortDefinitions,
+  databaseList = NULL,
+  overWriteIfExists = T
+){
+  
+  conn <- DatabaseConnector::connect(connectionDetails = connectionDetails)
+  on.exit(DatabaseConnector::disconnect(conn))
+  
+  modelDesignId <- insertModelDesignInDatabase(
+    object = diagnosePlp$modelDesign, 
+    conn = conn, 
+    databaseSchemaSettings = databaseSchemaSettings,
+    cohortDefinitions = cohortDefinitions 
+  )
+  
+  databaseId <- addDatabase(
+    conn = conn, 
+    databaseSchemaSettings = databaseSchemaSettings,
+    databaseList = databaseList,
+    databaseSchema = diagnosePlp$databaseSchema, 
+    databaseId = diagnosePlp$databaseId 
+  )
+  
+  diagnoseId <- insertDiagnosisToDatabase(
+    diagnostics = diagnosePlp,
+    conn = conn,
+    databaseSchemaSettings = databaseSchemaSettings,
+    modelDesignId = modelDesignId,
+    databaseId = databaseId,
+    overWriteIfExists = overWriteIfExists
+  )
+  
+  return(invisible(diagnoseId))
+}  
+  
+  
+insertDiagnosisToDatabase <- function(
+  diagnostics,
+  conn,
+  databaseSchemaSettings,
+  modelDesignId,
+  databaseId,
+  overWriteIfExists = T
+){
+  
+ diagnosticId <- addDiagnostic(
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$resultSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    
+    modelDesignId = modelDesignId,
+    databaseId = databaseId,
+
+    tablePrefix = databaseSchemaSettings$tablePrefix,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  ParallelLogger::logInfo(paste0('diagnosticId: ', diagnosticId))
+  
+  # now add the four tables
+  
+  ParallelLogger::logInfo('Adding DiagnosticSummary')
+  tryCatch({
+    addResultTable(
+      conn = conn, 
+      resultSchema = databaseSchemaSettings$resultSchema, 
+      targetDialect = databaseSchemaSettings$targetDialect,
+      
+      tableName = 'diagnostic_summary',
+      resultIdName = 'diagnosticId',
+      resultId = diagnosticId,
+      object = diagnostics$summary,
+      
+      tablePrefix = databaseSchemaSettings$tablePrefix,
+      tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema,
+      overWriteIfExists = overWriteIfExists
+    )},
+    error = function(e){ParallelLogger::logError(e);}
+  )
+  
+  ParallelLogger::logInfo('Adding DiagnosticParticipants')
+  tryCatch({
+    addResultTable(
+      conn = conn, 
+      resultSchema = databaseSchemaSettings$resultSchema, 
+      targetDialect = databaseSchemaSettings$targetDialect,
+      
+      tableName = 'diagnostic_participants',
+      resultIdName = 'diagnosticId',
+      resultId = diagnosticId,
+      object = diagnostics$participants,
+      
+      tablePrefix = databaseSchemaSettings$tablePrefix,
+      tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema,
+      overWriteIfExists = overWriteIfExists
+    )},
+    error = function(e){ParallelLogger::logError(e);}
+  )
+  
+  ParallelLogger::logInfo('Adding DiagnosticPredictors')
+  tryCatch({
+    addResultTable(
+      conn = conn, 
+      resultSchema = databaseSchemaSettings$resultSchema, 
+      targetDialect = databaseSchemaSettings$targetDialect,
+      
+      tableName = 'diagnostic_predictors',
+      resultIdName = 'diagnosticId',
+      resultId = diagnosticId,
+      object = diagnostics$predictors,
+      
+      tablePrefix = databaseSchemaSettings$tablePrefix,
+      tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema,
+      overWriteIfExists = overWriteIfExists
+    )},
+    error = function(e){ParallelLogger::logError(e);}
+  )
+  
+  ParallelLogger::logInfo('Adding DiagnosticOutcomes')
+  tryCatch({
+    addResultTable(
+      conn = conn, 
+      resultSchema = databaseSchemaSettings$resultSchema, 
+      targetDialect = databaseSchemaSettings$targetDialect,
+      
+      tableName = 'diagnostic_outcomes',
+      resultIdName = 'diagnosticId',
+      resultId = diagnosticId,
+      object = diagnostics$outcomes,
+      
+      tablePrefix = databaseSchemaSettings$tablePrefix,
+      tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema,
+      overWriteIfExists = overWriteIfExists
+    )},
+    error = function(e){ParallelLogger::logError(e);}
+  )
+  
+  ParallelLogger::logInfo('Adding DiagnosticDesigns')
+  tryCatch({
+    addResultTable(
+      conn = conn, 
+      resultSchema = databaseSchemaSettings$resultSchema, 
+      targetDialect = databaseSchemaSettings$targetDialect,
+      
+      tableName = 'diagnostic_designs',
+      resultIdName = 'diagnosticId',
+      resultId = diagnosticId,
+      object = diagnostics$designs,
+      
+      tablePrefix = databaseSchemaSettings$tablePrefix,
+      tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema,
+      overWriteIfExists = overWriteIfExists
+    )},
+    error = function(e){ParallelLogger::logError(e);}
+  )
+  
+  return(invisible(diagnosticId))
+}
+
+
+
+addDiagnostic <- function(
+  conn, 
+  resultSchema, 
+  targetDialect,
+  
+  modelDesignId,
+  databaseId,
+  
+  tablePrefix,
+  tempEmulationSchema
+){
+  
+  result <- checkTable(conn = conn, 
+                       resultSchema = resultSchema, 
+                       tablePrefix = tablePrefix,
+                       targetDialect = targetDialect, 
+                       tableName = 'diagnostics',
+                       columnNames = c(
+                         'model_design_id',
+                         'database_id'
+                       ), 
+                       values = c(
+                         modelDesignId,
+                         databaseId
+                       ),
+                       tempEmulationSchema = tempEmulationSchema
+  )
+  
+  if(nrow(result)==0){
+    # model
+    sql <- "INSERT INTO @my_schema.@string_to_appenddiagnostics (
+    model_design_id,
+    database_id
+  ) 
+  VALUES (
+  @model_design_id, 
+    @database_id
+    )"
+    sql <- SqlRender::render(sql, 
+                             my_schema = resultSchema,
+                             model_design_id = modelDesignId,
+                             database_id = databaseId,
+                             string_to_append = tablePrefix)
+    sql <- SqlRender::translate(sql, targetDialect = targetDialect,
+                                tempEmulationSchema = tempEmulationSchema)
+    DatabaseConnector::executeSql(conn, sql)
+    
+    #getId of new
+    result <- checkTable(conn = conn, 
+                         resultSchema = resultSchema, 
+                         tablePrefix = tablePrefix,
+                         targetDialect = targetDialect, 
+                         tableName = 'diagnostics',
+                         columnNames = c(
+                           'model_design_id',
+                           'database_id'
+                         ), 
+                         values = c(
+                           modelDesignId,
+                           databaseId
+                         ),
+                         tempEmulationSchema = tempEmulationSchema
+    )
+    
+  } 
+  
+  return(result$diagnosticId[1])
+}
+
+# replace the performance inserts with this single function...
+addResultTable <- function(
+  conn = conn, 
+  resultSchema, 
+  targetDialect,
+  tableName = 'diagnostic_summary',
+  resultIdName = 'diagnosticId',
+  resultId,
+  object,
+  tablePrefix,
+  tempEmulationSchema,
+  overWriteIfExists = T
+){
+  
+  object[resultIdName] <- resultId
+    
+    # get column names and check all present in object
+  columnNames <- getColumnNames(
+    conn = conn, 
+    resultSchema = resultSchema, 
+    targetDialect = targetDialect, 
+    tableName = paste0(tablePrefix,tableName), 
+    tempEmulationSchema = tempEmulationSchema
+  )
+    isValid <- sum(colnames(object)%in%columnNames) == length(columnNames)
+    
+    exists <- checkResultExists(
+      conn = conn, 
+      resultSchema = resultSchema, 
+      targetDialect = targetDialect, 
+      tableName = paste0(tablePrefix,tableName),
+      resultIdName = SqlRender::camelCaseToSnakeCase(resultIdName),
+      resultId = resultId,
+      tempEmulationSchema = tempEmulationSchema
+    )
+    
+    if(isValid && (!exists || overWriteIfExists)){
+      
+      # REMOVE existing result
+      if(exists){
+        sql <- "delete from @result_schema.@table_name where @result_id_name = @result_id;"
+        sql <- SqlRender::render(sql, 
+                                 result_id_name = SqlRender::camelCaseToSnakeCase(resultIdName),
+                                 result_id = resultId,
+                                 result_schema = resultSchema,
+                                 table_name = paste0(tablePrefix,tableName)
+                                 )
+        sql <- SqlRender::translate(sql, 
+                                    targetDialect = targetDialect,
+                                    tempEmulationSchema = tempEmulationSchema)
+        DatabaseConnector::executeSql(conn, sql)
+      }
+      
+      # add 
+      DatabaseConnector::insertTable(
+        connection = conn, 
+        databaseSchema = resultSchema, 
+        tableName = paste0(tablePrefix,tableName), 
+        data = as.data.frame(object[,columnNames]), 
+        dropTableIfExists = F, createTable = F, tempTable = F, 
+        bulkLoad = F, camelCaseToSnakeCase = T, progressBar = T,
+        tempEmulationSchema = tempEmulationSchema
+      )
+    }
+    
+    return(invisible(NULL))
+  }
diff --git a/R/uploadToDatabaseModelDesign.R b/R/uploadToDatabaseModelDesign.R
new file mode 100644
index 000000000..f99decd90
--- /dev/null
+++ b/R/uploadToDatabaseModelDesign.R
@@ -0,0 +1,995 @@
+#' Insert a model design into a PLP result schema database
+#' @description
+#' This function inserts a model design and all the settings into the result schema
+#'
+#' @details
+#' This function can be used to upload a model design into a database
+#'
+#' @param object                       An object of class modelDesign, runPlp or externalValidatePlp
+#' @param conn                         A connection to a database created by using the
+#'                                     function \code{connect} in the
+#'                                     \code{DatabaseConnector} package.
+#' @param databaseSchemaSettings       A object created by \code{createDatabaseSchemaSettings} with all the settings specifying the result tables                              
+#' @param cohortDefinitions            A set of one or more cohorts extracted using ROhdsiWebApi::exportCohortDefinitionSet()
+#'    
+#' @return
+#' Returns NULL but uploads the model design into the database schema specified in databaseSchemaSettings
+#' 
+#' @export
+insertModelDesignInDatabase <- function(
+  object,
+  conn,
+  databaseSchemaSettings,
+  cohortDefinitions
+){
+  
+  # REMOVE THIS
+  if(inherits(object, 'externalValidatePlp') | inherits(object, 'runPlp')){
+    
+    object <- PatientLevelPrediction::createModelDesign(
+      targetId = object$model$modelDesign$targetId,
+      outcomeId = object$model$modelDesign$outcomeId,
+      restrictPlpDataSettings = object$model$modelDesign$restrictPlpDataSettings,
+      populationSettings = object$model$modelDesign$populationSettings,
+      covariateSettings = object$model$modelDesign$covariateSettings,
+      featureEngineeringSettings = object$model$modelDesign$featureEngineeringSettings,
+      sampleSettings = object$model$modelDesign$sampleSettings,
+      preprocessSettings = object$model$modelDesign$preprocessSettings,
+      modelSettings = object$model$modelDesign$modelSettings,
+      runCovariateSummary = T
+    )
+    
+  }
+  
+  if(inherits(object, 'modelDesign')){
+    modelDesignId <- insertModelDesignSettings(
+      object = object,
+      conn = conn,
+      databaseSchemaSettings = databaseSchemaSettings,
+      cohortDefinitions = cohortDefinitions
+    )
+    return(modelDesignId)
+  }
+  
+  return(NULL)
+}
+
+
+# this function inserts all the settings for the model design
+# it returns the model_design_id for the database
+insertModelDesignSettings <- function(
+  object,
+  conn,
+  databaseSchemaSettings,
+  cohortDefinitions
+){
+  
+  if(!inherits(x = object, what = 'modelDesign')){
+    stop('object in insertModelDesign() is not a modelDesign')
+  }
+  
+
+  # add TAR
+  tarId <- addTar(
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$resultSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    startDay = object$populationSettings$riskWindowStart, 
+    startAnchor = object$populationSettings$startAnchor,
+    endDay = object$populationSettings$riskWindowEnd,  
+    endAnchor = object$populationSettings$endAnchor, 
+    tablePrefix = databaseSchemaSettings$tablePrefix,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  ParallelLogger::logInfo(paste0('tarId: ', tarId))
+  
+  tId <- addCohort(
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$cohortDefinitionSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    cohortDefinition = getCohortDef(cohortDefinitions, object$targetId),
+    tablePrefix = databaseSchemaSettings$tablePrefixCohortDefinitionTables,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  ParallelLogger::logInfo(paste0('tId: ', tId))
+  
+  oId <- addCohort(
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$cohortDefinitionSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    cohortDefinition = getCohortDef(cohortDefinitions, object$outcomeId),
+    tablePrefix = databaseSchemaSettings$tablePrefixCohortDefinitionTables,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  ParallelLogger::logInfo(paste0('oId: ', oId))
+  
+  popSetId <- addPopulationSetting(
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$resultSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    json = object$populationSettings, 
+    tablePrefix = databaseSchemaSettings$tablePrefix,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  ParallelLogger::logInfo(paste0('popSetId: ', popSetId))
+  
+  covSetId <- addCovariateSetting(
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$resultSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    json = object$covariateSettings, 
+    tablePrefix = databaseSchemaSettings$tablePrefix,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  ParallelLogger::logInfo(paste0('covSetId: ', covSetId))
+  
+  modSetId <- addModelSetting(
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$resultSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    modelType = attr(object$modelSettings$param, 'settings')$modelType, # make this the same as model$trainDetails$modelName?
+    json = object$modelSettings, 
+    tablePrefix = databaseSchemaSettings$tablePrefix,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  ParallelLogger::logInfo(paste0('modSetId: ', modSetId))
+  
+  # NEW: add plp_data_settings
+  plpDataSetId <- addPlpDataSetting(
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$resultSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    json = object$restrictPlpDataSettings, 
+    tablePrefix = databaseSchemaSettings$tablePrefix,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  ParallelLogger::logInfo(paste0('plpDataSetId: ', plpDataSetId))
+  
+  # NEW: add FE_settings
+  FESetId <- addFESetting(
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$resultSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    json = object$featureEngineeringSettings, 
+    tablePrefix = databaseSchemaSettings$tablePrefix,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  ParallelLogger::logInfo(paste0('FESetId: ', FESetId))
+  
+  # NEW: add sample_settings
+  sampleSetId <- addSampleSetting(
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$resultSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    json = object$sampleSettings, 
+    tablePrefix = databaseSchemaSettings$tablePrefix,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  ParallelLogger::logInfo(paste0('sampleSetId: ', sampleSetId))
+  
+  # NEW: add tidy_covariate_settings
+  tidySetId <- addTidySetting(
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$resultSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    json = object$preprocessSettings, 
+    tablePrefix = databaseSchemaSettings$tablePrefix,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  ParallelLogger::logInfo(paste0('tidySetId: ', tidySetId))
+  
+  
+  # this is now split setting - update this function
+  splitId <- addSplitSettings(
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$resultSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    json = object$splitSettings,
+    tablePrefix = databaseSchemaSettings$tablePrefix,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  ParallelLogger::logInfo(paste0('splitId: ', splitId))
+  
+  # create this function
+  modelDesignId <- addModelDesign( # need to create
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$resultSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    targetId = tId,
+    outcomeId = oId,
+    tarId = tarId,
+    plpDataSettingId = plpDataSetId,
+    populationSettingId = popSetId,
+    modelSettingId = modSetId,
+    covariateSettingId = covSetId,
+    sampleSettingId = sampleSetId,
+    splitSettingId = splitId, # changed from trainingId
+    featureEngineeringSettingId = FESetId,
+    tidyCovariatesSettingId = tidySetId,
+    tablePrefix = databaseSchemaSettings$tablePrefix,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  ParallelLogger::logInfo(paste0('modelDesignId: ', modelDesignId))
+  
+  return(modelDesignId)
+}
+
+addModelDesign <- function(
+  conn, 
+  resultSchema, targetDialect,
+  tablePrefix = tablePrefix,
+  targetId,
+  outcomeId,
+  tarId,
+  plpDataSettingId,
+  populationSettingId,
+  modelSettingId,
+  covariateSettingId,
+  sampleSettingId,
+  splitSettingId,
+  featureEngineeringSettingId,
+  tidyCovariatesSettingId,
+  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
+){
+  
+  if(is.null(targetId)){
+    stop('targetId is null')
+  }
+  if(is.null(outcomeId)){
+    stop('outcomeId is null')
+  }
+  if(is.null(tarId)){
+    stop('tarId is null')
+  }
+  
+  if(is.null(plpDataSettingId)){
+    stop('plpDataSettingId is null')
+  }
+  if(is.null(populationSettingId)){
+    stop('populationSettingId is null')
+  }
+  if(is.null(modelSettingId)){
+    stop('modelSettingId is null')
+  }
+  if(is.null(covariateSettingId)){
+    stop('covariateSettingId is null')
+  }
+  if(is.null(sampleSettingId)){
+    stop('sampleSettingId is null')
+  }
+  if(is.null(splitSettingId)){
+    stop('splitSettingId is null')
+  }
+  if(is.null(featureEngineeringSettingId)){
+    stop('featureEngineeringSettingId is null')
+  }
+  if(is.null(tidyCovariatesSettingId)){
+    stop('tidyCovariatesSettingId is null')
+  }
+  
+  # process json to make it ordered...
+  # TODO
+  
+  result <- checkTable(
+    conn = conn, 
+    resultSchema = resultSchema, 
+    tablePrefix = tablePrefix,
+    targetDialect = targetDialect, 
+    tableName = 'model_designs',
+    columnNames = c(
+      'target_id',
+      'outcome_id',
+      'tar_id',
+      'plp_data_setting_id',
+      'population_setting_id',
+      'model_setting_id',
+      'covariate_setting_id',
+      'sample_setting_id',
+      'split_setting_id',
+      'feature_engineering_setting_id',
+      'tidy_covariates_setting_id'
+    ), 
+    values = c(
+      targetId,
+      outcomeId,
+      tarId,
+      plpDataSettingId,
+      populationSettingId,
+      modelSettingId,
+      covariateSettingId,
+      sampleSettingId,
+      splitSettingId,
+      featureEngineeringSettingId,
+      tidyCovariatesSettingId
+    ),
+    tempEmulationSchema = tempEmulationSchema
+  )
+  
+  if(nrow(result)==0){
+    # model
+    sql <- "INSERT INTO @my_schema.@string_to_appendmodel_designs(
+    target_id,
+    outcome_id,
+    tar_id,
+    plp_data_setting_id,
+    population_setting_id,
+    model_setting_id,
+    covariate_setting_id,
+    sample_setting_id,
+    split_setting_id,
+    feature_engineering_setting_id,
+    tidy_covariates_setting_id
+    ) VALUES 
+  ( 
+  @target_id, 
+  @outcome_id, 
+  @tar_id, 
+  @plp_data_setting_id,
+  @population_setting_id, 
+  @model_setting_id, 
+  @covariate_setting_id, 
+  @sample_setting_id, 
+  @split_setting_id, 
+  @feature_engineering_setting_id, 
+  @tidy_covariates_setting_id
+    )"
+    sql <- SqlRender::render(
+      sql, 
+      my_schema = resultSchema,
+      target_id = targetId,
+      outcome_id = outcomeId,
+      tar_id = tarId,
+      plp_data_setting_id= plpDataSettingId,
+      population_setting_id = populationSettingId,
+      model_setting_id = modelSettingId,
+      covariate_setting_id = covariateSettingId,
+      sample_setting_id = sampleSettingId,
+      split_setting_id = splitSettingId,
+      feature_engineering_setting_id = featureEngineeringSettingId,
+      tidy_covariates_setting_id = tidyCovariatesSettingId,
+      string_to_append = tablePrefix
+    )
+    sql <- SqlRender::translate(sql, targetDialect = targetDialect,
+                                tempEmulationSchema = tempEmulationSchema)
+    DatabaseConnector::executeSql(conn, sql)
+    
+    #getId of new
+    result <- checkTable(conn = conn, 
+                         resultSchema = resultSchema, 
+                         tablePrefix = tablePrefix,
+                         targetDialect = targetDialect, 
+                         tableName = 'model_designs',
+                         columnNames = c(
+                           'target_id',
+                           'outcome_id',
+                           'tar_id',
+                           'plp_data_setting_id',
+                           'population_setting_id',
+                           'model_setting_id',
+                           'covariate_setting_id',
+                           'sample_setting_id',
+                           'split_setting_id',
+                           'feature_engineering_setting_id',
+                           'tidy_covariates_setting_id'
+                           ), 
+                         values = c(targetId,
+                                    outcomeId,
+                                    tarId,
+                                    plpDataSettingId,
+                                    populationSettingId,
+                                    modelSettingId,
+                                    covariateSettingId,
+                                    sampleSettingId,
+                                    splitSettingId,
+                                    featureEngineeringSettingId,
+                                    tidyCovariatesSettingId
+                                    ),
+                         tempEmulationSchema = tempEmulationSchema
+    )
+    
+  } 
+  
+  return(result$modelDesignId[1])
+}
+
+
+addTar <- function(conn, resultSchema, targetDialect,
+                   tablePrefix = '',
+                   startDay, 
+                   startAnchor,
+                   endDay,  
+                   endAnchor,
+                   tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
+){
+  
+  result <- checkTable(conn = conn, 
+                       resultSchema = resultSchema, 
+                       tablePrefix = tablePrefix,
+                       targetDialect = targetDialect, 
+                       tableName = 'tars',
+                       columnNames = c('tar_start_day', 'tar_start_anchor',
+                                       'tar_end_day', 'tar_end_anchor'), 
+                       values = c(startDay, 
+                                  paste0("'",startAnchor,"'"),
+                                  endDay,
+                                  paste0("'",endAnchor,"'")),
+                       tempEmulationSchema = tempEmulationSchema
+  )
+  
+  if(nrow(result)==0){
+    
+    ParallelLogger::logInfo('Adding TAR')
+    # tars - id 1
+    sql <- "INSERT INTO @my_schema.@string_to_appendtars(tar_start_day, tar_start_anchor,
+                           tar_end_day, tar_end_anchor) 
+          VALUES (@tar_start_day, @tar_start_anchor, @tar_end_day, @tar_end_anchor);"
+    sql <- SqlRender::render(sql, 
+                             my_schema = resultSchema,
+                             tar_start_day = startDay,
+                             tar_start_anchor = paste0("'",startAnchor,"'"),
+                             tar_end_day = endDay,
+                             tar_end_anchor = paste0("'",endAnchor,"'"),
+                             string_to_append = tablePrefix)
+    
+    sql <- SqlRender::translate(sql, targetDialect = targetDialect, 
+                                tempEmulationSchema = tempEmulationSchema)
+    
+    DatabaseConnector::executeSql(conn, sql)
+    
+    #getId of new
+    result <- checkTable(conn = conn, 
+                         resultSchema = resultSchema, 
+                         tablePrefix = tablePrefix,
+                         targetDialect = targetDialect, 
+                         tableName = 'tars',
+                         columnNames = c('tar_start_day', 'tar_start_anchor',
+                                         'tar_end_day', 'tar_end_anchor'), 
+                         values = c(startDay, 
+                                    paste0("'",startAnchor,"'"),
+                                    endDay,
+                                    paste0("'",endAnchor,"'")),
+                         tempEmulationSchema = tempEmulationSchema
+    )
+    
+  } else {
+    ParallelLogger::logInfo('TAR exists')
+  }
+  
+  
+  return(result$tarId[1])
+  
+}
+
+addPopulationSetting <- function(conn, resultSchema, targetDialect,
+                                 tablePrefix = '',
+                                 json,
+                                 tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
+){
+  
+  # process json to make it ordered...
+  # make sure the json has been converted 
+  if(!inherits(x = json, what = 'character')){
+      json <- orderJson(json) # to ensure attributes are alphabetic order
+      json <- ParallelLogger::convertSettingsToJson(json)
+      json <- as.character(json) # now convert to character
+  }
+  
+  jsonId <- checkJson(conn = conn,
+                      resultSchema = resultSchema, 
+                      tablePrefix = tablePrefix,
+                      targetDialect = targetDialect, 
+                      tableName = 'population_settings',
+                      jsonColumnName = 'populationSettingsJson',
+                      id = 'populationSettingId',
+                      json = json, 
+                      tempEmulationSchema = tempEmulationSchema)
+  
+  if(is.null(jsonId)){
+    ParallelLogger::logInfo('Adding new population settings')
+    
+    data <- data.frame(populationSettingsJson = json)
+    DatabaseConnector::insertTable(connection = conn, 
+                                   databaseSchema = resultSchema, 
+                                   tableName = paste0(tablePrefix, 'population_settings'),
+                                   data = data, 
+                                   dropTableIfExists = F, 
+                                   createTable = F, 
+                                   tempTable = F, 
+                                   progressBar = T,
+                                   camelCaseToSnakeCase = T,
+                                   tempEmulationSchema = tempEmulationSchema
+    )
+    
+    #getId of new
+    jsonId <- checkJson(conn = conn,
+                        resultSchema = resultSchema, 
+                        tablePrefix = tablePrefix,
+                        targetDialect = targetDialect, 
+                        tableName = 'population_settings',
+                        jsonColumnName = 'populationSettingsJson',
+                        id = 'populationSettingId',
+                        json = json,
+                        tempEmulationSchema = tempEmulationSchema)
+    
+  } else{
+    ParallelLogger::logInfo('Population settings exists')
+  }
+  
+  return(jsonId)
+}
+
+
+addCovariateSetting <- function(conn, resultSchema, targetDialect,
+                                tablePrefix = '',
+                                json,
+                                tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
+){
+  
+  # process json to make it ordered...
+  # make sure the json has been converted 
+  if(!inherits(x = json, what = 'character')){
+    json <- orderJson(json) # to ensure attributes are alphabetic order
+    json <- ParallelLogger::convertSettingsToJson(json)
+    json <- as.character(json) # now convert to character
+  }
+  
+  jsonId <- checkJson(
+    conn = conn,
+    resultSchema = resultSchema, 
+    tablePrefix = tablePrefix,
+    targetDialect = targetDialect, 
+    tableName = 'covariate_settings',
+    jsonColumnName = 'covariateSettingsJson',
+    id = 'covariateSettingId',
+    json = json,
+    tempEmulationSchema = tempEmulationSchema
+  )
+  
+  if(is.null(jsonId)){
+    
+    ParallelLogger::logInfo('Adding new covariate settings')
+    
+    data <- data.frame(covariateSettingsJson = json)
+    DatabaseConnector::insertTable(connection = conn, 
+                                   databaseSchema = resultSchema, 
+                                   tableName = paste0(tablePrefix, 'covariate_settings'),
+                                   data = data, 
+                                   dropTableIfExists = F, 
+                                   createTable = F, 
+                                   tempTable = F, 
+                                   progressBar = T,
+                                   camelCaseToSnakeCase = T,
+                                   tempEmulationSchema = tempEmulationSchema
+    )
+    
+    #getId of new
+    jsonId <- checkJson(conn = conn,
+                        resultSchema = resultSchema, 
+                        tablePrefix = tablePrefix,
+                        targetDialect = targetDialect, 
+                        tableName = 'covariate_settings',
+                        jsonColumnName = 'covariateSettingsJson',
+                        id = 'covariateSettingId',
+                        json = json,
+                        tempEmulationSchema = tempEmulationSchema)
+    
+  } else{
+    ParallelLogger::logInfo('Covariate setting exists')
+  }
+  
+  return(jsonId)
+}
+
+
+addModelSetting <- function(conn, resultSchema, targetDialect,
+                            tablePrefix = '',
+                            modelType, json,
+                            tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
+){
+  
+  # process json to make it ordered...
+  # make sure the json has been converted 
+  if(!inherits(x = json, what = 'character')){
+    json <- orderJson(json) # to ensure attributes are alphabetic order
+    json <- ParallelLogger::convertSettingsToJson(json)
+    json <- as.character(json) # now convert to character
+  }
+  
+  jsonId <- checkJson(conn = conn,
+                      resultSchema = resultSchema, 
+                      tablePrefix = tablePrefix,
+                      targetDialect = targetDialect, 
+                      tableName = 'model_settings',
+                      jsonColumnName = 'modelSettingsJson',
+                      id = 'modelSettingId',
+                      json = json,
+                      tempEmulationSchema = tempEmulationSchema)
+  
+  if(is.null(jsonId)){
+    
+    ParallelLogger::logInfo('Adding new model settings')
+    
+    data <- data.frame(modelType = modelType,
+                       modelSettingsJson = json)
+    DatabaseConnector::insertTable(connection = conn, 
+                                   databaseSchema = resultSchema, 
+                                   tableName = paste0(tablePrefix, 'model_settings'),
+                                   data = data, 
+                                   dropTableIfExists = F, 
+                                   createTable = F, 
+                                   tempTable = F, 
+                                   progressBar = T,
+                                   camelCaseToSnakeCase = T,
+                                   tempEmulationSchema = tempEmulationSchema)
+    
+    #getId of new
+    jsonId <- checkJson(conn = conn,
+                        resultSchema = resultSchema, 
+                        tablePrefix = tablePrefix,
+                        targetDialect = targetDialect, 
+                        tableName = 'model_settings',
+                        jsonColumnName = 'modelSettingsJson',
+                        id = 'modelSettingId',
+                        json = json,
+                        tempEmulationSchema = tempEmulationSchema)
+    
+  } else{
+    ParallelLogger::logInfo('Model setting exists')
+  }
+  
+  return(jsonId)
+}
+
+addTidySetting <- function(
+  conn, 
+  resultSchema, 
+  targetDialect,
+  tablePrefix = '',
+  json,
+  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
+){
+  
+  if(!inherits(x = json, what = 'character')){
+    json <- orderJson(json) # to ensure attributes are alphabetic order
+    json <- ParallelLogger::convertSettingsToJson(json)
+    json <- as.character(json) # now convert to character
+  }
+  
+  jsonId <- checkJson(conn = conn,
+                      resultSchema = resultSchema, 
+                      tablePrefix = tablePrefix,
+                      targetDialect = targetDialect, 
+                      tableName = 'tidy_covariates_settings',
+                      jsonColumnName = 'tidyCovariatesSettingsJson',
+                      id = 'tidyCovariatesSettingId',
+                      json = json,
+                      tempEmulationSchema = tempEmulationSchema)
+  
+  if(is.null(jsonId)){
+    
+    ParallelLogger::logInfo('Adding new tidy covariates settings')
+    
+    data <- data.frame(
+      tidyCovariatesSettingsJson = json
+    )
+    
+    DatabaseConnector::insertTable(
+      connection = conn, 
+      databaseSchema = resultSchema, 
+      tableName = paste0(tablePrefix, 'tidy_covariates_settings'),
+      data = data, 
+      dropTableIfExists = F, 
+      createTable = F, 
+      tempTable = F, 
+      progressBar = T,
+      camelCaseToSnakeCase = T,
+      tempEmulationSchema = tempEmulationSchema
+    )
+    
+    #getId of new
+    jsonId <- checkJson(
+      conn = conn,
+      resultSchema = resultSchema, 
+      tablePrefix = tablePrefix,
+      targetDialect = targetDialect, 
+      tableName = 'tidy_covariates_settings',
+      jsonColumnName = 'tidyCovariatesSettingsJson',
+      id = 'tidyCovariatesSettingId',
+      json = json,
+      tempEmulationSchema = tempEmulationSchema
+    )
+    
+  } else{
+    ParallelLogger::logInfo('tidy covariates setting exists')
+  }
+  
+  return(jsonId)
+  
+}
+
+addSampleSetting <- function(
+  conn, 
+  resultSchema, 
+  targetDialect,
+  tablePrefix = '',
+  json,
+  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
+){
+  
+  if(!inherits(x = json, what = 'character')){
+    json <- orderJson(json) # to ensure attributes are alphabetic order
+    json <- ParallelLogger::convertSettingsToJson(json)
+    json <- as.character(json) # now convert to character
+  }
+  
+  jsonId <- checkJson(
+    conn = conn,
+    resultSchema = resultSchema, 
+    tablePrefix = tablePrefix,
+    targetDialect = targetDialect, 
+    tableName = 'sample_settings',
+    jsonColumnName = 'sampleSettingsJson',
+    id = 'sampleSettingId',
+    json = json,
+    tempEmulationSchema = tempEmulationSchema
+  )
+  
+  if(is.null(jsonId)){
+    
+    ParallelLogger::logInfo('Adding new sample settings')
+    
+    data <- data.frame(
+      sampleSettingsJson = json
+    )
+    
+    DatabaseConnector::insertTable(
+      connection = conn, 
+      databaseSchema = resultSchema, 
+      tableName = paste0(tablePrefix, 'sample_settings'),
+      data = data, 
+      dropTableIfExists = F, 
+      createTable = F, 
+      tempTable = F, 
+      progressBar = T,
+      camelCaseToSnakeCase = T,
+      tempEmulationSchema = tempEmulationSchema
+    )
+    
+    #getId of new
+    jsonId <- checkJson(
+      conn = conn,
+      resultSchema = resultSchema, 
+      tablePrefix = tablePrefix,
+      targetDialect = targetDialect, 
+      tableName = 'sample_settings',
+      jsonColumnName = 'sampleSettingsJson',
+      id = 'sampleSettingId',
+      json = json,
+      tempEmulationSchema = tempEmulationSchema
+    )
+    
+  } else{
+    ParallelLogger::logInfo('sample setting exists')
+  }
+  
+  return(jsonId)
+  
+}
+
+addPlpDataSetting <- function(
+  conn, 
+  resultSchema, 
+  targetDialect,
+  tablePrefix = '',
+  json,
+  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
+){
+  
+  if(!inherits(x = json, what = 'character')){
+    json <- orderJson(json) # to ensure attributes are alphabetic order
+    json <- ParallelLogger::convertSettingsToJson(json)
+    json <- as.character(json) # now convert to character
+  }
+  
+  jsonId <- checkJson(conn = conn,
+                      resultSchema = resultSchema, 
+                      tablePrefix = tablePrefix,
+                      targetDialect = targetDialect, 
+                      tableName = 'plp_data_settings',
+                      jsonColumnName = 'plpDataSettingsJson',
+                      id = 'plpDataSettingId',
+                      json = json,
+                      tempEmulationSchema = tempEmulationSchema)
+  
+  if(is.null(jsonId)){
+    
+    ParallelLogger::logInfo('Adding new plp data settings')
+    
+    data <- data.frame(
+      plpDataSettingsJson = json
+    )
+    
+    DatabaseConnector::insertTable(
+      connection = conn, 
+      databaseSchema = resultSchema, 
+      tableName = paste0(tablePrefix, 'plp_data_settings'),
+      data = data, 
+      dropTableIfExists = F, 
+      createTable = F, 
+      tempTable = F, 
+      progressBar = T,
+      camelCaseToSnakeCase = T,
+      tempEmulationSchema = tempEmulationSchema
+    )
+    
+    #getId of new
+    jsonId <- checkJson(
+      conn = conn,
+      resultSchema = resultSchema, 
+      tablePrefix = tablePrefix,
+      targetDialect = targetDialect, 
+      tableName = 'plp_data_settings',
+      jsonColumnName = 'plpDataSettingsJson',
+      id = 'plpDataSettingId',
+      json = json,
+      tempEmulationSchema = tempEmulationSchema
+    )
+    
+  } else{
+    ParallelLogger::logInfo('Split setting exists')
+  }
+  
+  return(jsonId)
+  
+}
+
+addFESetting <- function(
+  conn, 
+  resultSchema, 
+  targetDialect,
+  tablePrefix = '',
+  json,
+  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
+){
+  
+  if(!inherits(x = json, what = 'character')){
+    json <- orderJson(json) # to ensure attributes are alphabetic order
+    json <- ParallelLogger::convertSettingsToJson(json)
+    json <- as.character(json) # now convert to character
+  }
+  
+  jsonId <- checkJson(
+    conn = conn,
+    resultSchema = resultSchema, 
+    tablePrefix = tablePrefix,
+    targetDialect = targetDialect, 
+    tableName = 'feature_engineering_settings',
+    jsonColumnName = 'featureEngineeringSettingsJson',
+    id = 'featureEngineeringSettingId',
+    json = json,
+    tempEmulationSchema = tempEmulationSchema
+  )
+  
+  if(is.null(jsonId)){
+    
+    ParallelLogger::logInfo('Adding new feature_engineering settings')
+    
+    data <- data.frame(
+      featureEngineeringSettingsJson = json
+    )
+    
+    DatabaseConnector::insertTable(
+      connection = conn, 
+      databaseSchema = resultSchema, 
+      tableName = paste0(tablePrefix, 'feature_engineering_settings'),
+      data = data, 
+      dropTableIfExists = F, 
+      createTable = F, 
+      tempTable = F, 
+      progressBar = T,
+      camelCaseToSnakeCase = T,
+      tempEmulationSchema = tempEmulationSchema
+    )
+    
+    #getId of new
+    jsonId <- checkJson(
+      conn = conn,
+      resultSchema = resultSchema, 
+      tablePrefix = tablePrefix,
+      targetDialect = targetDialect, 
+      tableName = 'feature_engineering_settings',
+      jsonColumnName = 'featureEngineeringSettingsJson',
+      id = 'featureEngineeringSettingId',
+      json = json,
+      tempEmulationSchema = tempEmulationSchema
+    )
+    
+  } else{
+    ParallelLogger::logInfo('feature engineering setting exists')
+  }
+  
+  return(jsonId)
+  
+}
+
+addSplitSettings <- function(
+  conn, 
+  resultSchema, 
+  targetDialect,
+  tablePrefix = '',
+  json,
+  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
+){
+  
+  if(!inherits(x = json, what = 'character')){
+    json <- orderJson(json) # to ensure attributes are alphabetic order
+    json <- ParallelLogger::convertSettingsToJson(json)
+    json <- as.character(json) # now convert to character
+  }
+  
+  jsonId <- checkJson(
+    conn = conn,
+    resultSchema = resultSchema, 
+    tablePrefix = tablePrefix,
+    targetDialect = targetDialect, 
+    tableName = 'split_settings',
+    jsonColumnName = 'splitSettingsJson',
+    id = 'splitSettingId',
+    json = json,
+    tempEmulationSchema = tempEmulationSchema
+  )
+  
+  if(is.null(jsonId)){
+    
+    ParallelLogger::logInfo('Adding new split settings')
+    
+    data <- data.frame(
+      splitSettingsJson = json
+    )
+    
+    DatabaseConnector::insertTable(
+      connection = conn, 
+      databaseSchema = resultSchema, 
+      tableName = paste0(tablePrefix, 'split_settings'),
+      data = data, 
+      dropTableIfExists = F, 
+      createTable = F, 
+      tempTable = F, 
+      progressBar = T,
+      camelCaseToSnakeCase = T,
+      tempEmulationSchema = tempEmulationSchema
+    )
+    
+    #getId of new
+    jsonId <- checkJson(
+      conn = conn,
+      resultSchema = resultSchema, 
+      tablePrefix = tablePrefix,
+      targetDialect = targetDialect, 
+      tableName = 'split_settings',
+      jsonColumnName = 'splitSettingsJson',
+      id = 'splitSettingId',
+      json = json,
+      tempEmulationSchema = tempEmulationSchema
+    )
+    
+  } else{
+    ParallelLogger::logInfo('Split setting exists')
+  }
+  
+  return(jsonId)
+  
+}
+
+
+
+# the ParallelLogger conversion orders attributes - use this for consistency
+orderJson <- function(x){
+newx <- ParallelLogger::convertJsonToSettings(
+  ParallelLogger::convertSettingsToJson(
+    x
+    )
+  )
+
+return(newx)
+}
diff --git a/R/uploadToDatabasePerformance.R b/R/uploadToDatabasePerformance.R
new file mode 100644
index 000000000..4b36af84c
--- /dev/null
+++ b/R/uploadToDatabasePerformance.R
@@ -0,0 +1,835 @@
+insertPerformanceInDatabase <- function(
+  performanceEvaluation = NULL,
+  covariateSummary = NULL,
+  attrition = NULL,
+  executionDateTime,
+  plpVersion,
+  
+  conn,
+  databaseSchemaSettings,
+  
+  modelDesignId,
+  developmentDatabaseId,
+  
+  validationDatabaseId,
+  validationTarId,
+  validationPopulationId,
+  validationPlpDataId,
+  validationTargetId,
+  validationOutcomeId,
+  
+  modelDevelopment #added 
+  
+){
+  
+  ParallelLogger::logInfo(paste0('Inserting performance...'))
+  
+  # add the results
+  performanceId <- addPerformance(
+    conn = conn, 
+    resultSchema = databaseSchemaSettings$resultSchema, 
+    targetDialect = databaseSchemaSettings$targetDialect,
+    
+    modelDesignId = modelDesignId,
+    developmentDatabaseId = developmentDatabaseId,
+    
+    validationDatabaseId = validationDatabaseId,
+    validationTargetId = validationTargetId,
+    validationOutcomeId = validationOutcomeId,
+    validationTarId = validationTarId,
+    validationPlpDataId = validationPlpDataId,
+    validationPopulationId = validationPopulationId,
+    
+    modelDevelopment = modelDevelopment,
+    executionDateTime = executionDateTime,
+    plpVersion = plpVersion, 
+    tablePrefix = databaseSchemaSettings$tablePrefix,
+    tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+  )
+  ParallelLogger::logInfo(paste0('performanceId: ', performanceId))
+  
+  # add attrition
+  if(!is.null(attrition)){
+    addAttrition(
+      conn = conn, 
+      resultSchema = databaseSchemaSettings$resultSchema, 
+      targetDialect = databaseSchemaSettings$targetDialect,
+      performanceId = performanceId,
+      attrition = attrition,
+      overWriteIfExists = T, 
+      tablePrefix = databaseSchemaSettings$tablePrefix,
+      tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+    )
+  }
+  
+  # add performance 
+  #=============
+  if(!is.null(performanceEvaluation)){
+    addEvaluation(
+      conn = conn, 
+      resultSchema = databaseSchemaSettings$resultSchema, 
+      targetDialect = databaseSchemaSettings$targetDialect,
+      performanceId = performanceId,
+      performanceEvaluation = performanceEvaluation,
+      overWriteIfExists = T, 
+      tablePrefix = databaseSchemaSettings$tablePrefix,
+      tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+    )
+  }
+  
+  if(!is.null(covariateSummary)){
+    addCovariateSummary(
+      conn = conn, 
+      resultSchema = databaseSchemaSettings$resultSchema, 
+      targetDialect = databaseSchemaSettings$targetDialect,
+      performanceId = performanceId,
+      covariateSummary = covariateSummary,
+      restrictToIncluded = T,
+      overWriteIfExists = T, 
+      tablePrefix = databaseSchemaSettings$tablePrefix,
+      tempEmulationSchema = databaseSchemaSettings$tempEmulationSchema
+    )
+  }
+  
+  return(invisible(performanceId))
+  
+}
+
+
+addPerformance <- function(
+  conn, 
+  resultSchema, 
+  targetDialect,
+  
+  modelDesignId,
+  developmentDatabaseId,
+  
+  validationDatabaseId,
+  validationTargetId,
+  validationOutcomeId,
+  validationTarId,
+  validationPlpDataId,
+  validationPopulationId,
+  
+  modelDevelopment,
+  executionDateTime,
+  plpVersion, 
+  tablePrefix,
+  tempEmulationSchema
+  
+){
+  
+  result <- checkTable(conn = conn, 
+                       resultSchema = resultSchema, 
+                       tablePrefix = tablePrefix,
+                       targetDialect = targetDialect, 
+                       tableName = 'performances',
+                       columnNames = c(
+                         'model_design_id',
+                         'development_database_id',
+                         
+                         'validation_database_id',
+                         'target_id',
+                         'outcome_id',
+                         'tar_id',
+                         'plp_data_setting_id',
+                         'population_setting_id',
+                         'model_development'
+                       ), 
+                       values = c(
+                         modelDesignId,
+                         developmentDatabaseId,
+                         
+                         validationDatabaseId,
+                         validationTargetId,
+                         validationOutcomeId,
+                         validationTarId,
+                         validationPlpDataId,
+                         validationPopulationId,
+                         modelDevelopment
+                       ),
+                       tempEmulationSchema = tempEmulationSchema
+  )
+  
+  if(nrow(result)==0){
+    # model
+    sql <- "INSERT INTO @my_schema.@string_to_appendperformances (
+    model_design_id,
+    development_database_id,
+    
+    validation_database_id,
+    target_id,
+    outcome_id,
+    tar_id,
+    plp_data_setting_id,
+    population_setting_id,
+    
+    model_development,
+    
+    execution_date_time,
+    plp_version
+  ) 
+  VALUES (
+  @model_design_id, @development_database_id, 
+  @validation_database_id, @validation_target_id, @validation_outcome_id, @validation_tar_id, 
+          @validation_plp_data_setting_id, @validation_population_setting_id, 
+          @model_development, 
+    '@execution_date_time', '@plp_version')"
+    sql <- SqlRender::render(sql, 
+                             my_schema = resultSchema,
+                             model_design_id = modelDesignId,
+                             development_database_id = developmentDatabaseId,
+                             
+                             validation_database_id = validationDatabaseId,
+                             validation_target_id = validationTargetId,
+                             validation_outcome_id = validationOutcomeId,
+                             validation_tar_id = validationTarId,
+                             validation_plp_data_setting_id = validationPlpDataId,
+                             validation_population_setting_id = validationPopulationId, 
+                             
+                             model_development = modelDevelopment, 
+                             
+                             execution_date_time = executionDateTime,
+                             plp_version = plpVersion,
+                             string_to_append = tablePrefix)
+    sql <- SqlRender::translate(sql, targetDialect = targetDialect,
+                                tempEmulationSchema = tempEmulationSchema)
+    DatabaseConnector::executeSql(conn, sql)
+    
+    #getId of new
+    result <- checkTable(conn = conn, 
+                         resultSchema = resultSchema, 
+                         tablePrefix = tablePrefix,
+                         targetDialect = targetDialect, 
+                         tableName = 'performances',
+                         columnNames = c(
+                           'model_design_id',
+                           'development_database_id',
+                           
+                           'validation_database_id',
+                           'target_id',
+                           'outcome_id',
+                           'tar_id',
+                           'plp_data_setting_id',
+                           'population_setting_id',
+                           
+                           'model_development'
+                           ), 
+                         values = c(
+                           modelDesignId,
+                           developmentDatabaseId,
+                           
+                           validationDatabaseId,
+                           validationTargetId,
+                           validationOutcomeId,
+                           validationTarId,
+                           validationPlpDataId,
+                           validationPopulationId,
+                           
+                           modelDevelopment
+                           ),
+                         tempEmulationSchema = tempEmulationSchema
+    )
+    
+  } 
+  
+  return(result$performanceId[1])
+}
+
+# attrition
+addAttrition <- function(
+  conn, resultSchema, targetDialect,
+  tablePrefix = '',
+  performanceId,
+  attrition,
+  overWriteIfExists = T,
+  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")){
+  
+  value <- attrition
+  if(is.null(value)){
+    return(NULL)
+  }
+  
+  # edit names
+  firstLower <- function(x) {
+    substr(x, 1, 1) <- tolower(substr(x, 1, 1))
+    return(x)
+  }
+  colnames(value) <- sapply(colnames(value), firstLower )
+  
+  value$performanceId <- performanceId
+  
+  # get column names and check all present in object
+  columnNames <- getColumnNames(conn = conn, 
+                                resultSchema = resultSchema, 
+                                targetDialect = targetDialect, 
+                                tableName = paste0(tablePrefix,'attrition'), 
+                                tempEmulationSchema = tempEmulationSchema)
+  isValid <- sum(colnames(value)%in%columnNames) == length(columnNames)
+  
+  exists <- checkResultExists(conn = conn, 
+                              resultSchema = resultSchema, 
+                              targetDialect = targetDialect, 
+                              tableName = paste0(tablePrefix,'attrition'),
+                              resultId = performanceId,
+                              tempEmulationSchema = tempEmulationSchema)
+  
+  if(isValid && (!exists || overWriteIfExists)){
+    
+    # REMOVE existing result
+    if(exists){
+      sql <- "delete from @result_schema.@table_name where performance_id = @performance_id;"
+      sql <- SqlRender::render(sql, 
+                               performance_id = performanceId,
+                               result_schema = resultSchema,
+                               table_name = paste0(tablePrefix,'attrition'))
+      sql <- SqlRender::translate(sql, 
+                                  targetDialect = targetDialect,
+                                  tempEmulationSchema = tempEmulationSchema)
+      DatabaseConnector::executeSql(conn, sql)
+    }
+    
+    # add 
+    ParallelLogger::logInfo(paste0('Inserting attrition for performance ',performanceId))
+    DatabaseConnector::insertTable(connection = conn, 
+                                   databaseSchema = resultSchema, 
+                                   tableName = paste0(tablePrefix,'attrition'), 
+                                   data = value[,columnNames], 
+                                   dropTableIfExists = F, createTable = F, tempTable = F, 
+                                   bulkLoad = F, camelCaseToSnakeCase = T, progressBar = T,
+                                   tempEmulationSchema = tempEmulationSchema)
+  }
+  
+  return(invisible(NULL))
+}
+
+
+# evals
+addEvaluation <- function(conn, resultSchema, targetDialect,
+                          tablePrefix = '',
+                          performanceId,
+                          performanceEvaluation,
+                          overWriteIfExists = T,
+                          tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")){
+  
+  ParallelLogger::logInfo('Adding PredictionDistribution')
+  tryCatch({addPredictionDistribution(conn = conn, resultSchema = resultSchema, targetDialect = targetDialect,
+                                      tablePrefix = tablePrefix,
+                                      performanceId = performanceId,
+                                      performanceEvaluation = performanceEvaluation,
+                                      overWriteIfExists = overWriteIfExists,
+                                      tempEmulationSchema = tempEmulationSchema)},
+           error = function(e){ParallelLogger::logError(e);})
+  
+  ParallelLogger::logInfo('Adding ThresholdSummary')
+  tryCatch({addThresholdSummary(conn = conn, resultSchema = resultSchema, targetDialect = targetDialect,
+                                tablePrefix = tablePrefix,
+                                performanceId = performanceId,
+                                performanceEvaluation = performanceEvaluation,
+                                overWriteIfExists = overWriteIfExists,
+                                tempEmulationSchema = tempEmulationSchema)},
+           error = function(e){ParallelLogger::logError(e);})
+  
+  ParallelLogger::logInfo('Adding EvaluationStatistics')
+  tryCatch({addEvaluationStatistics(conn = conn, resultSchema = resultSchema, targetDialect = targetDialect,
+                                    tablePrefix = tablePrefix,
+                                    performanceId = performanceId,
+                                    performanceEvaluation = performanceEvaluation,
+                                    overWriteIfExists = overWriteIfExists,
+                                    tempEmulationSchema = tempEmulationSchema)},
+           error = function(e){ParallelLogger::logError(e);})
+  
+  ParallelLogger::logInfo('Adding CalibrationSummary')
+  tryCatch({addCalibrationSummary(conn = conn, resultSchema = resultSchema, targetDialect = targetDialect,
+                                  tablePrefix = tablePrefix,
+                                  performanceId = performanceId,
+                                  performanceEvaluation = performanceEvaluation,
+                                  overWriteIfExists = overWriteIfExists,
+                                  tempEmulationSchema = tempEmulationSchema)},
+           error = function(e){ParallelLogger::logError(e);})
+  
+  ParallelLogger::logInfo('Adding DemographicSummary')
+  tryCatch({addDemographicSummary(conn = conn, resultSchema = resultSchema, targetDialect = targetDialect,
+                                  tablePrefix = tablePrefix,
+                                  performanceId = performanceId,
+                                  performanceEvaluation = performanceEvaluation,
+                                  overWriteIfExists = overWriteIfExists,
+                                  tempEmulationSchema = tempEmulationSchema)},
+           error = function(e){ParallelLogger::logError(e);})
+  
+  return(invisible(NULL))
+  
+}
+
+addPredictionDistribution <- function(conn, resultSchema, targetDialect,
+                                      tablePrefix = '',
+                                      performanceId,
+                                      performanceEvaluation,
+                                      overWriteIfExists = T,
+                                      tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")){
+  
+  value <- performanceEvaluation$predictionDistribution
+  if(is.null(value)){
+    return(NULL)
+  }
+  
+  # edit names
+  firstLower <- function(x) {
+    substr(x, 1, 1) <- tolower(substr(x, 1, 1))
+    return(x)
+  }
+  colnames(value) <- sapply(colnames(value), firstLower )
+  if(sum(colnames(value)=='class')>0){
+    colnames(value)[colnames(value)=='class'] <- 'classLabel'
+  }
+  
+  value$performanceId <- performanceId
+  
+  # get column names and check all present in object
+  columnNames <- getColumnNames(conn = conn, 
+                                resultSchema = resultSchema, 
+                                targetDialect = targetDialect, 
+                                tableName = paste0(tablePrefix,'prediction_distribution'), 
+                                tempEmulationSchema = tempEmulationSchema)
+  isValid <- sum(colnames(value)%in%columnNames) == length(columnNames)
+  
+  exists <- checkResultExists(conn = conn, 
+                              resultSchema = resultSchema, 
+                              targetDialect = targetDialect, 
+                              tableName = paste0(tablePrefix,'prediction_distribution'),
+                              resultIdName = 'performance_id',
+                              resultId = performanceId,
+                              tempEmulationSchema = tempEmulationSchema)
+  
+  if(isValid && (!exists || overWriteIfExists)){
+    
+    # REMOVE existing result
+    if(exists){
+      sql <- "delete from @result_schema.@table_name where performance_id = @performance_id;"
+      sql <- SqlRender::render(sql, 
+                               performance_id = performanceId,
+                               result_schema = resultSchema,
+                               table_name = paste0(tablePrefix,'prediction_distribution'))
+      sql <- SqlRender::translate(sql, 
+                                  targetDialect = targetDialect,
+                                  tempEmulationSchema = tempEmulationSchema)
+      DatabaseConnector::executeSql(conn, sql)
+    }
+    
+    # add 
+    ParallelLogger::logInfo(paste0('Inserting predictionDistribution for performance ', performanceId))
+    DatabaseConnector::insertTable(connection = conn, 
+                                   databaseSchema = resultSchema, 
+                                   tableName = paste0(tablePrefix,'prediction_distribution'), 
+                                   data = value[,columnNames], 
+                                   dropTableIfExists = F, createTable = F, tempTable = F, 
+                                   bulkLoad = F, camelCaseToSnakeCase = T, progressBar = T,
+                                   tempEmulationSchema = tempEmulationSchema)
+  }
+  
+  return(invisible(NULL))
+}
+
+addThresholdSummary <- function(conn, resultSchema, targetDialect,
+                                tablePrefix = '',
+                                performanceId,
+                                performanceEvaluation,
+                                overWriteIfExists = T,
+                                tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")){
+  
+  
+  value <- performanceEvaluation$thresholdSummary
+  if(is.null(value)){
+    return(NULL)
+  }
+  
+  #  check numerical columns:
+  value <- cleanNum(value)
+  
+  # edit names
+  firstLower <- function(x) {
+    substr(x, 1, 1) <- tolower(substr(x, 1, 1))
+    return(x)
+  }
+  colnames(value) <- sapply(colnames(value), firstLower )
+  value$performanceId <- performanceId
+  
+  # get column names and check all present in object
+  columnNames <- getColumnNames(conn = conn, 
+                                resultSchema = resultSchema, 
+                                targetDialect = targetDialect, 
+                                tablePrefix = tablePrefix,
+                                tableName = 'threshold_summary',
+                                tempEmulationSchema = tempEmulationSchema)
+  isValid <- sum(colnames(value)%in%columnNames) == length(columnNames)
+  
+  exists <- checkResultExists(conn = conn, 
+                              resultSchema = resultSchema, 
+                              targetDialect = targetDialect, 
+                              tableName = paste0(tablePrefix,'threshold_summary'),
+                              resultIdName = 'performance_id',
+                              resultId = performanceId,
+                              tempEmulationSchema = tempEmulationSchema)
+  
+  if(isValid && (!exists || overWriteIfExists)){
+    
+    # REMOVE existing result
+    if(exists){
+      sql <- "delete from @result_schema.@table_name where performance_id = @performance_id;"
+      sql <- SqlRender::render(sql, 
+                               result_schema = resultSchema,
+                               performance_id = performanceId,
+                               table_name = paste0(tablePrefix,'threshold_summary'))
+      sql <- SqlRender::translate(sql, 
+                                  targetDialect = targetDialect,
+                                  tempEmulationSchema = tempEmulationSchema)
+      DatabaseConnector::executeSql(conn, sql)
+    }
+    
+    # add 
+    ParallelLogger::logInfo(paste0('Inserting thresholdSummary for performance ',performanceId))
+    DatabaseConnector::insertTable(connection = conn, 
+                                   databaseSchema = resultSchema, 
+                                   tableName = paste0(tablePrefix,'threshold_summary'), 
+                                   data = value[,columnNames], 
+                                   dropTableIfExists = F, createTable = F, tempTable = F, 
+                                   bulkLoad = F, camelCaseToSnakeCase = T, progressBar = T,
+                                   tempEmulationSchema = tempEmulationSchema)
+  }
+  
+  return(invisible(NULL))
+}
+
+
+addCalibrationSummary <- function(conn, resultSchema, targetDialect, 
+                                  tablePrefix = '',
+                                  performanceId,
+                                  performanceEvaluation,
+                                  overWriteIfExists = T,
+                                  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")){
+  
+  
+  value <- performanceEvaluation$calibrationSummary
+  if(is.null(value)){
+    return(NULL)
+  }
+  
+  #  check numerical columns:
+  value <- cleanNum(value)
+  
+  # edit names
+  firstLower <- function(x) {
+    substr(x, 1, 1) <- tolower(substr(x, 1, 1))
+    return(x)
+  }
+  colnames(value) <- sapply(colnames(value), firstLower )
+  
+  value$performanceId <- performanceId
+  
+  # get column names and check all present in object
+  columnNames <- getColumnNames(conn = conn, 
+                                resultSchema = resultSchema, 
+                                targetDialect = targetDialect, 
+                                tablePrefix = tablePrefix,
+                                tableName = 'calibration_summary',
+                                tempEmulationSchema = tempEmulationSchema)
+  isValid <- sum(colnames(value)%in%columnNames) == length(columnNames)
+  
+  exists <- checkResultExists(conn = conn, 
+                              resultSchema = resultSchema, 
+                              targetDialect = targetDialect, 
+                              tableName = paste0(tablePrefix,'calibration_summary'),
+                              resultIdName = 'performance_id',
+                              resultId = performanceId,
+                              tempEmulationSchema = tempEmulationSchema)
+  
+  if(isValid && (!exists || overWriteIfExists)){
+    
+    # REMOVE existing result
+    if(exists){
+      sql <- "delete from @result_schema.@table_name where performance_id = @performance_id;"
+      sql <- SqlRender::render(sql, 
+                               result_schema = resultSchema,
+                               performance_id= performanceId,
+                               table_name = paste0(tablePrefix,'calibration_summary'))
+      sql <- SqlRender::translate(sql, 
+                                  targetDialect = targetDialect,
+                                  tempEmulationSchema = tempEmulationSchema)
+      DatabaseConnector::executeSql(conn, sql)
+    }
+    
+    # add 
+    ParallelLogger::logInfo(paste0('Inserting calibrationSummary for performance ', performanceId))
+    DatabaseConnector::insertTable(connection = conn, 
+                                   databaseSchema = resultSchema, 
+                                   tableName = paste0(tablePrefix,'calibration_summary'), 
+                                   data = value[,columnNames], 
+                                   dropTableIfExists = F, createTable = F, tempTable = F, 
+                                   bulkLoad = F, camelCaseToSnakeCase = T, progressBar = T,
+                                   tempEmulationSchema = tempEmulationSchema)
+  }
+  
+  return(invisible(NULL))
+}
+
+addEvaluationStatistics <- function(conn, resultSchema, targetDialect,
+                                    tablePrefix = '',
+                                    performanceId,
+                                    performanceEvaluation,
+                                    overWriteIfExists = T,
+                                    tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")){
+  
+  
+  value <- data.frame(
+    evaluation = unlist(performanceEvaluation$evaluationStatistics$evaluation),
+    metric = unlist(performanceEvaluation$evaluationStatistics$metric),
+    value = as.numeric(unlist(performanceEvaluation$evaluationStatistics$value))
+  )
+  
+  if(is.null(value)){
+    return(NULL)
+  }
+  
+  # edit names
+  firstLower <- function(x) {
+    substr(x, 1, 1) <- tolower(substr(x, 1, 1))
+    return(x)
+  }
+  colnames(value) <- sapply(colnames(value), firstLower )
+  value$performanceId <- performanceId
+  
+  # get column names and check all present in object
+  columnNames <- getColumnNames(conn = conn, 
+                                resultSchema = resultSchema, 
+                                targetDialect = targetDialect, 
+                                tablePrefix = tablePrefix,
+                                tableName = 'evaluation_statistics',
+                                tempEmulationSchema = tempEmulationSchema)
+  isValid <- sum(colnames(value)%in%columnNames) == length(columnNames)
+  
+  exists <- checkResultExists(conn = conn, 
+                              resultSchema = resultSchema, 
+                              targetDialect = targetDialect,
+                              tableName = paste0(tablePrefix,'evaluation_statistics'),
+                              resultIdName = 'performance_id',
+                              resultId = performanceId,
+                              tempEmulationSchema = tempEmulationSchema)
+  
+  if(isValid && (!exists || overWriteIfExists)){
+    
+    # REMOVE existing result
+    if(exists){
+      sql <- "delete from @result_schema.@table_name where performance_id = @performance_id;"
+      sql <- SqlRender::render(sql, 
+                               result_schema = resultSchema,
+                               performance_id = performanceId,
+                               table_name = paste0(tablePrefix,'evaluation_statistics'))
+      sql <- SqlRender::translate(sql, 
+                                  targetDialect = targetDialect,
+                                  tempEmulationSchema = tempEmulationSchema)
+      DatabaseConnector::executeSql(conn, sql)
+    }
+    
+    # add 
+    ParallelLogger::logInfo(paste0('Inserting evaluationSummary for performance ',performanceId))
+    DatabaseConnector::insertTable(connection = conn, 
+                                   databaseSchema = resultSchema, 
+                                   tableName = paste0(tablePrefix,'evaluation_statistics'), 
+                                   data = value[,columnNames], 
+                                   dropTableIfExists = F, createTable = F, tempTable = F, 
+                                   bulkLoad = F, camelCaseToSnakeCase = T, progressBar = T,
+                                   tempEmulationSchema = tempEmulationSchema)
+  }
+  
+  return(invisible(NULL))
+}
+
+addDemographicSummary <- function(conn, resultSchema, targetDialect, 
+                                  tablePrefix = '',
+                                  performanceId,
+                                  performanceEvaluation,
+                                  overWriteIfExists = T,
+                                  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")){
+  
+  
+  value <- performanceEvaluation$demographicSummary
+  if(is.null(value)){
+    return(NULL)
+  }
+  
+  # edit names
+  firstLower <- function(x) {
+    substr(x, 1, 1) <- tolower(substr(x, 1, 1))
+    return(x)
+  }
+  colnames(value) <- sapply(colnames(value), firstLower )
+  #if(sum(colnames(value)=="p50PredictedProbability")>0){
+  #  colnames(value)[colnames(value)=="p50PredictedProbability"] <- 'medianPredictedProbability'
+  #}
+  
+  value$performanceId <- performanceId
+  
+  # get column names and check all present in object
+  columnNames <- getColumnNames(conn = conn, 
+                                resultSchema = resultSchema, 
+                                targetDialect = targetDialect, 
+                                tablePrefix =  tablePrefix,
+                                tableName = 'demographic_summary',
+                                tempEmulationSchema = tempEmulationSchema)
+  isValid <- sum(colnames(value)%in%columnNames) == length(columnNames)
+  
+  exists <- checkResultExists(conn = conn, 
+                              resultSchema = resultSchema, 
+                              targetDialect = targetDialect, 
+                              tableName = paste0(tablePrefix,'demographic_summary'),
+                              resultIdName = 'performance_id',
+                              resultId = performanceId,
+                              tempEmulationSchema = tempEmulationSchema)
+  
+  if(isValid && (!exists || overWriteIfExists)){
+    
+    # REMOVE existing result
+    if(exists){
+      sql <- "delete from @result_schema.@table_name where performance_id = @performance_id;"
+      sql <- SqlRender::render(sql, 
+                               result_schema = resultSchema,
+                               performance_id = performanceId,
+                               table_name = paste0(tablePrefix,'demographic_summary'))
+      sql <- SqlRender::translate(sql, 
+                                  targetDialect = targetDialect,
+                                  tempEmulationSchema = tempEmulationSchema)
+      DatabaseConnector::executeSql(conn, sql)
+    }
+    
+    # add 
+    ParallelLogger::logInfo(paste0('Inserting demographicSummary for performance ',performanceId))
+    DatabaseConnector::insertTable(connection = conn, 
+                                   databaseSchema = resultSchema, 
+                                   tableName = paste0(tablePrefix,'demographic_summary'), 
+                                   data = value[,columnNames], 
+                                   dropTableIfExists = F, createTable = F, tempTable = F, 
+                                   bulkLoad = F, camelCaseToSnakeCase = T, progressBar = T,
+                                   tempEmulationSchema = tempEmulationSchema)
+  }
+  
+  return(invisible(NULL))
+}
+
+addCovariateSummary <- function(conn, resultSchema, targetDialect, 
+                                tablePrefix = '',
+                                performanceId,
+                                covariateSummary,
+                                restrictToIncluded = T,
+                                overWriteIfExists = T,
+                                tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")){
+  
+  
+  value <- covariateSummary
+  if(is.null(value)){
+    return(NULL)
+  }
+  
+  # edit names
+  firstLower <- function(x) {
+    substr(x, 1, 1) <- tolower(substr(x, 1, 1))
+    return(x)
+  }
+  colnames(value) <- sapply(colnames(value), firstLower )
+  value$performanceId <- performanceId
+  # remove _ from names
+  colnames(value) <- gsub('_','', colnames(value))
+  
+  if(restrictToIncluded){
+    ParallelLogger::logInfo('Restricting to covariates included in model')
+    value <- value[value$covariateValue!=0 & !is.na(value$covariateValue),] 
+  }
+  
+  # get column names and check all present in object
+  columnNames <- getColumnNames(conn = conn, 
+                                resultSchema = resultSchema, 
+                                targetDialect = targetDialect, 
+                                tablePrefix = tablePrefix,
+                                tableName = 'covariate_summary',
+                                tempEmulationSchema = tempEmulationSchema)
+  isValid <- sum(colnames(value)%in%columnNames) == length(columnNames)
+  
+  exists <- checkResultExists(conn = conn, 
+                              resultSchema = resultSchema, 
+                              targetDialect = targetDialect, 
+                              tableName = paste0(tablePrefix,'covariate_summary'),
+                              resultIdName = 'performance_id',
+                              resultId = performanceId,
+                              tempEmulationSchema = tempEmulationSchema)
+  
+  if(isValid && (!exists || overWriteIfExists)){
+    
+    # REMOVE existing result
+    if(exists){
+      ParallelLogger::logTrace('Removing existing covariateSummary')
+      sql <- "delete from @result_schema.@table_name where performance_id = @performance_id;"
+      sql <- SqlRender::render(sql, 
+                               result_schema = resultSchema,
+                               performance_id = performanceId,
+                               table_name = paste0(tablePrefix,'covariate_summary'))
+      sql <- SqlRender::translate(sql, 
+                                  targetDialect = targetDialect,
+                                  tempEmulationSchema = tempEmulationSchema)
+      DatabaseConnector::executeSql(conn, sql)
+    }
+    
+    # add 
+    ParallelLogger::logInfo(paste0('Inserting covariateSummary for result ', performanceId))
+    DatabaseConnector::insertTable(connection = conn, 
+                                   databaseSchema = resultSchema, 
+                                   tableName = paste0(tablePrefix,'covariate_summary'), 
+                                   data = value[,columnNames], 
+                                   dropTableIfExists = F, createTable = F, tempTable = F, 
+                                   bulkLoad = F, camelCaseToSnakeCase = T, progressBar = T,
+                                   tempEmulationSchema = tempEmulationSchema)
+  }
+  
+  return(invisible(NULL))
+}
+
+
+
+#====================
+# Helpers
+#====================
+
+# gets the column names in camelCase of a table
+getColumnNames <- function(conn, resultSchema, targetDialect, tableName, tablePrefix = '',
+                           tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
+){
+  sql <- "select top 1 * from @my_schema.@string_to_append@table;"
+  sql <- SqlRender::render(sql, 
+                           my_schema = resultSchema,
+                           table = tableName,
+                           string_to_append = tablePrefix)
+  sql <- SqlRender::translate(sql, targetDialect = targetDialect,
+                              tempEmulationSchema = tempEmulationSchema)
+  result <- DatabaseConnector::querySql(connection = conn, sql = sql, snakeCaseToCamelCase = T)
+  
+  return(colnames(result))
+}
+
+# True/False check whether results exist in table 
+checkResultExists <- function(conn, resultSchema, targetDialect,
+                              snakeCaseToCamelCase,
+                              tableName,
+                              resultIdName = 'performance_id',
+                              resultId,
+                              tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")
+){
+  
+  sql <- "select * from @my_schema.@table where @result_id_name = @result_id;"
+  sql <- SqlRender::render(sql, 
+                           my_schema = resultSchema,
+                           table = tableName,
+                           result_id_name = resultIdName,
+                           result_id = resultId)
+  sql <- SqlRender::translate(sql, targetDialect = targetDialect,
+                              tempEmulationSchema = tempEmulationSchema)
+  result <- DatabaseConnector::querySql(connection = conn, sql = sql, snakeCaseToCamelCase = T)
+  return(nrow(result)>0)
+}
\ No newline at end of file
diff --git a/README.md b/README.md
index 5ee60d9f6..2126ec474 100644
--- a/README.md
+++ b/README.md
@@ -18,11 +18,11 @@ Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. [Design and implementati
 
 The figure below illustrates the prediction problem we address. Among a population at risk, we aim to predict which patients at a defined moment in time (t = 0) will experience some outcome during a time-at-risk. Prediction is done using only information about the patients in an observation window prior to that moment in time.
 
-![](vignettes/Figure1.png)
+![](vignettes/Figure1.webp)
 
 To define a prediction problem we have to define t=0 by a Target Cohort (T), the outcome we like to predict by an outcome cohort (O), and the time-at-risk (TAR). Furthermore, we  have to make design choices for the model we like to develop, and determine the observational datasets to perform internal and external validation. This conceptual framework works for all type of prediction problems, for example those presented below (T=green, O=red).
 
-![](vignettes/problems.png)
+![](vignettes/problems.webp)
 
 Features
 ========
@@ -30,16 +30,18 @@ Features
 - Allows for multiple prediction design options.
 - Extracts the necessary data from a database in OMOP Common Data Model format for multiple covariate settings.
 - Uses a large set of covariates including for example all drugs, diagnoses, procedures, as well as age, comorbidity indexes, and custom covariates.
-- Includes a large number of state-of-the-art machine learning algorithms that can be used to develop predictive models, including Regularized logistic regression, Random forest, Gradient boosting machines, Decision tree, Naive Bayes, K-nearest neighbours, Neural network and Deep learning (Convolutional neural networks, Recurrent neural network and Deep nets).
+- Allows you to add custom covariates or cohort covariates.
+- Includes a large number of state-of-the-art machine learning algorithms that can be used to develop predictive models, including Regularized logistic regression, Random forest, Gradient boosting machines, Decision tree, Naive Bayes, K-nearest neighbours, Neural network, AdaBoost and Support vector machines.
 - Allows you to add custom algorithms.
+- Allows you to add custom feature engineering
+- Allows you to add custom under/over sampling (or any other sampling) [note: based on existing research this is not recommended]
 - Contains functionality to externally validate models.
 - Includes functions to plot and explore model performance (ROC + Calibration).
-- Includes a shiny app to interactively view and explore results.
-- Implements existing models.
-- Builds ensemble models.
-- Builds Deep Learning models.
+- Build ensemble models using EnsemblePatientLevelPrediction.
+- Build Deep Learning models using DeepPatientLevelPrediction.
 - Generates learning curves.
-- Automatically creates a word document containing all the study results.
+- Includes a shiny app to interactively view and explore results.
+- In the shiny app you can create a html file document (report or protocol) containing all the study results.
 
 
 Screenshots
@@ -49,11 +51,11 @@ Screenshots
 <tr valign="bottom">
 <td width = 50%>
 
-<img src="https://github.com/OHDSI/PatientLevelPrediction/raw/main/vignettes/sparseCalibration.png" alt="Calibration plot" title="Calibration plot" />
+<img src="https://github.com/OHDSI/PatientLevelPrediction/raw/main/vignettes/sparseCalibration.webp" alt="Calibration plot" title="Calibration plot" />
 
 </td>
 <td width = 50%>
-<img src="https://github.com/OHDSI/PatientLevelPrediction/raw/main/vignettes/sparseROC.png" alt="ROC plot" title="ROC plot" />
+<img src="https://github.com/OHDSI/PatientLevelPrediction/raw/main/vignettes/sparseROC.webp" alt="ROC plot" title="ROC plot" />
 
 </td>
 </tr><tr>
@@ -64,20 +66,19 @@ Screenshots
 
 Demo of the Shiny Apps can be found here:
 
-- [Single Prediction Viewer Shiny App](http://data.ohdsi.org/PredictionViewer/)
 - [Multiple Prediction Viewer Shiny App](http://data.ohdsi.org/smokingPhenotypeExplorer/)
 
 
 Technology
 ==========
-PatientLevelPrediction is an R package, with some functions implemented in C++ and python.
+PatientLevelPrediction is an R package, with some functions using python through reticulate.
 
 System Requirements
 ===================
-Requires R (version 3.3.0 or higher). Installation on Windows requires [RTools](http://cran.r-project.org/bin/windows/Rtools/). Libraries used in PatientLevelPrediction require Java and Python.
+Requires R (version 4.0 or higher). Installation on Windows requires [RTools](http://cran.r-project.org/bin/windows/Rtools/). Libraries used in PatientLevelPrediction require Java and Python.
 
 The python installation is required for some of the machine learning algorithms. We advise to
-install Python 3.7 using Anaconda (https://www.continuum.io/downloads). 
+install Python 3.8 or higher using Anaconda (https://www.continuum.io/downloads). 
 
 Getting Started
 ===============
@@ -99,8 +100,8 @@ In addition we have created vignettes that describe advanced functionality in mo
 - [Building Multiple Patient-Level Predictive Models](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingMultiplePredictiveModels.pdf)
 - [Implementing Existing Patient-Level Predictive Models](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/ImplementingExistingModels.pdf)
 - [Adding Custom Machine Learning Algorithms](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/AddingCustomAlgorithms.pdf)
-- [Building Deep Learning Models](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingDeepLearningModels.pdf)
-- [Building Ensemble Models](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingEnsembleModels.pdf)
+- [Building Deep Learning Models](https://github.com/OHDSI/DeepPatientLevelPrediction)
+- [Building Ensemble Models](https://github.com/OHDSI/EnsemblePatientLevelPrediction)
 - [Creating Learning Curves](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/CreatingLearningCurves.pdf)
 
 Package manual: [PatientLevelPrediction.pdf](https://github.com/OHDSI/PatientLevelPrediction/blob/main/extras/PatientLevelPrediction.pdf)
@@ -128,10 +129,9 @@ Development
 ===========
 PatientLevelPrediction is being developed in R Studio.
 
-Beta
 
 # Acknowledgements
 
 - The package is maintained by Jenna Reps and Peter Rijnbeek and has been developed with major contributions from Martijn Schuemie, Patrick Ryan, and Marc Suchard.
-- We like to thank the following persons for their contributions to the package: Seng Chan You, Ross Williams, Henrik John, Xiaoyong Pan, James Wiggins.
+- We like to thank the following persons for their contributions to the package: Seng Chan You, Ross Williams, Henrik John, Xiaoyong Pan, James Wiggins, Egill Fridgeirsson, Alex Rekkas
 - This project is supported in part through the National Science Foundation grant IIS 1251151.
diff --git a/_pkgdown.yml b/_pkgdown.yml
index 8e3b50682..763c23504 100644
--- a/_pkgdown.yml
+++ b/_pkgdown.yml
@@ -1,7 +1,11 @@
 template:
   params:
     bootswatch: cosmo
-    
+
+development:
+  mode: auto
+  development: docs/dev
+
 home:
   links:
   - text: Ask a question
@@ -16,7 +20,10 @@ navbar:
     - reference
     - articles
     - tutorial
+    - benchmarks
+    - predictors
     - bestpractice
+    - clinicalmodels
     - news
     right: [hades, github]
   components:
@@ -35,6 +42,15 @@ navbar:
     bestpractice:
       text: Best Practices
       href: articles/BestPractices.html
+    clinicalmodels:
+      text: Clinical Models
+      href: articles/ClinicalModels.html
+    benchmarks:
+      text: Benchmarks
+      href: articles/BenchmarkTasks.html
+    predictors:
+      text: Predictors
+      href: articles/ConstrainedPredictors.html
     news:
       text: Changelog
       href: news/index.html
@@ -57,7 +73,8 @@ reference:
       - createRestrictPlpDataSettings
       - getPlpData
       - savePlpData
-      - loadPlpData   
+      - loadPlpData
+      - getCohortCovariateData
   - title: "Settings for designing a prediction models"
     desc: >
       Design settings required when developing a model.
@@ -67,12 +84,29 @@ reference:
       - createSampleSettings
       - createFeatureEngineeringSettings
       - createPreprocessSettings
+  - title: "Optional design settings"
+    desc: >
+      Settings for optional steps that can be used in the PLP pipeline
+    contents:
+      - createCohortCovariateSettings
+      - createRandomForestFeatureSelection
+      - createUnivariateFeatureSelection
+      - createSplineSettings
+      - createStratifiedImputationSettings
+  - title: "External validation"
+    contents:
+      - createValidationDesign
+      - validateExternal
+      - createValidationSettings
+      - recalibratePlp
+      - recalibratePlpRefit
   - title: "Execution settings when developing a model"
     desc: >
       Execution settings required when developing a model.
     contents:
       - createLogSettings
       - createExecuteSettings
+      - createDefaultExecuteSettings
   - title: "Binary Classification Models"
     desc: >
       Functions for setting binary classifiers and their hyper-parameter search.
@@ -86,6 +120,8 @@ reference:
       - setNaiveBayes
       - setRandomForest
       - setSVM
+      - setIterativeHardThresholding
+      - setLightGBM
   - title: "Survival Models"
     desc: >
       Functions for setting survival models and their hyper-parameter search.
@@ -101,6 +137,7 @@ reference:
       - loadPlpModel
       - savePlpResult
       - loadPlpResult
+      - diagnosePlp
   - title: "Multiple Patient-Level Prediction Models"
     desc: >
       Functions for training mutliple patient-level-prediction model in an efficient way.
@@ -110,12 +147,34 @@ reference:
       - validateMultiplePlp
       - savePlpAnalysesJson 
       - loadPlpAnalysesJson
+      - diagnoseMultiplePlp
+  - title: "Individual pipeline functions"
+    desc: >
+      Functions for running parts of the PLP workflow
+    contents:
+      - createStudyPopulation
+      - splitData
+      - preprocessData
+      - fitPlp
+      - predictPlp
+      - evaluatePlp
+      - covariateSummary
   - title: "Saving results into database"
     desc: >
       Functions for saving the prediction model and performances into a database.
     contents:
+      - insertResultsToSqlite
       - createPlpResultTables
-      - populatePlpResultTables     
+      - addMultipleRunPlpToDatabase
+      - addRunPlpToDatabase
+      - createDatabaseSchemaSettings 
+      - createDatabaseList
+      - addDiagnosePlpToDatabase
+      - addMultipleDiagnosePlpToDatabase
+      - extractDatabaseToCsv
+      - insertCsvToDatabase
+      - insertModelDesignInDatabase
+      - migrateDataModel
   - title: "Shiny Viewers"
     desc: >
       Functions for viewing results via a shiny app
@@ -140,6 +199,7 @@ reference:
       - plotPreferencePDF
       - plotPredictionDistribution
       - plotVariableScatterplot
+      - outcomeSurvivalPlot
   - title: "Learning Curves"
     desc: >
       Functions for creating and plotting learning curves
@@ -151,3 +211,67 @@ reference:
       Functions for simulating cohort method data objects.
     contents:
       - simulatePlpData
+      - plpDataSimulationProfile
+  - title: "Data manipulation functions"
+    desc: >
+      Functions for manipulating data
+    contents:
+      - toSparseM
+      - MapIds
+  - title: "Helper/utility functions"
+    contents:
+      - listAppend
+      - listCartesian
+      - createTempModelLoc
+      - configurePython
+      - setPythonEnvironment
+  - title: "Evaluation measures"
+    contents:
+      - accuracy
+      - averagePrecision
+      - brierScore
+      - calibrationLine
+      - computeAuc
+      - f1Score
+      - falseDiscoveryRate
+      - falseNegativeRate
+      - falseOmissionRate
+      - falsePositiveRate
+      - ici
+      - modelBasedConcordance
+      - negativeLikelihoodRatio
+      - negativePredictiveValue
+      - positiveLikelihoodRatio
+      - positivePredictiveValue
+      - sensitivity
+      - specificity
+      - computeGridPerformance
+      - diagnosticOddsRatio
+      - getCalibrationSummary
+      - getDemographicSummary
+      - getThresholdSummary
+      - getThresholdSummary_binary
+      - getPredictionDistribution
+      - getPredictionDistribution_binary
+  - title: "Saving/loading models as json"
+    desc: >
+      Functions for saving or loading models as json
+    contents:
+      - sklearnFromJson
+      - sklearnToJson
+  - title: "Load/save for sharing"
+    desc: >
+      Functions for loading/saving objects for sharing
+    contents:
+      - savePlpShareable
+      - loadPlpShareable
+      - loadPrediction
+      - savePrediction
+  - title: "Feature importance"
+    contents:
+      - pfi
+  - title: "Other functions"
+    contents:
+      - predictCyclops
+    
+
diff --git a/docs/404.html b/docs/404.html
index c63753b85..7dbf2ed28 100644
--- a/docs/404.html
+++ b/docs/404.html
@@ -1,66 +1,27 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<meta charset="utf-8">
 <meta http-equiv="X-UA-Compatible" content="IE=edge">
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
 <title>Page not found (404) • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="bootstrap-toc.css">
-<script src="bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="pkgdown.css" rel="stylesheet">
-<script src="pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Page not found (404)" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous">
+<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="bootstrap-toc.css">
+<script src="bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous">
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous">
+<!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="pkgdown.css" rel="stylesheet">
+<script src="pkgdown.js"></script><meta property="og:title" content="Page not found (404)">
+<!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
 <![endif]-->
+</head>
+<body data-spy="scroll" data-target="#toc">
+    
 
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-title-body">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -71,13 +32,13 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
       <ul class="nav navbar-nav">
-        <li>
+<li>
   <a href="index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -93,13 +54,13 @@
   <a href="reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
   <ul class="dropdown-menu" role="menu">
-    <li>
+<li>
       <a href="articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -112,10 +73,10 @@
       <a href="articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="articles/BestPractices.html">Best Practice Research</a>
+      <a href="articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -124,13 +85,16 @@
       <a href="articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -141,36 +105,45 @@
   </ul>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="news/index.html">Changelog</a>
 </li>
       </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+<ul class="nav navbar-nav navbar-right">
+<li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
       </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
+</div>
+<!--/.nav-collapse -->
+  </div>
+<!--/.container -->
+</div>
+<!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="contents col-md-9">
     <div class="page-header">
       <h1>Page not found (404)</h1>
@@ -181,31 +154,31 @@ <h1>Page not found (404)</h1>
   </div>
 
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
     </nav>
-  </div>
+</div>
 
 </div>
 
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p>
+<p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p>
+<p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
       </footer>
-   </div>
+</div>
 
   
 
 
+  
+
   </body>
 </html>
-
-
diff --git a/docs/articles/ATLAS_O.png b/docs/articles/ATLAS_O.png
deleted file mode 100644
index 3cda2abf7..000000000
Binary files a/docs/articles/ATLAS_O.png and /dev/null differ
diff --git a/docs/articles/ATLAS_T.png b/docs/articles/ATLAS_T.png
deleted file mode 100644
index 8be57dc9e..000000000
Binary files a/docs/articles/ATLAS_T.png and /dev/null differ
diff --git a/docs/articles/AddingCustomAlgorithms.html b/docs/articles/AddingCustomAlgorithms.html
deleted file mode 100644
index d9dbffc64..000000000
--- a/docs/articles/AddingCustomAlgorithms.html
+++ /dev/null
@@ -1,527 +0,0 @@
-<!DOCTYPE html>
-<!-- Generated by pkgdown: do not edit by hand --><html lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-<title>Adding Custom Patient-Level Prediction Algorithms • PatientLevelPrediction</title>
-<!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous">
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous">
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous">
-<!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script><meta property="og:title" content="Adding Custom Patient-Level Prediction Algorithms">
-<meta property="og:description" content="PatientLevelPrediction">
-<!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-</head>
-<body data-spy="scroll" data-target="#toc">
-    <div class="container template-article">
-      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-<li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-<li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-<ul class="nav navbar-nav navbar-right">
-<li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-</div>
-<!--/.nav-collapse -->
-  </div>
-<!--/.container -->
-</div>
-<!--/.navbar -->
-
-      
-
-      </header><script src="AddingCustomAlgorithms_files/header-attrs-2.7/header-attrs.js"></script><div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header toc-ignore">
-      <h1 data-toc-skip>Adding Custom Patient-Level Prediction Algorithms</h1>
-                        <h4 class="author">Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek</h4>
-            
-            <h4 class="date">2021-09-29</h4>
-      
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/AddingCustomAlgorithms.Rmd"><code>vignettes/AddingCustomAlgorithms.Rmd</code></a></small>
-      <div class="hidden name"><code>AddingCustomAlgorithms.Rmd</code></div>
-
-    </div>
-
-    
-    
-<!--
-%\VignetteEngine{knitr}
-%\VignetteIndexEntry{Adding Custom Patient-Level Prediction Algorithms}
--->
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
-<p>This vignette describes how you can add your own custom algorithms in the Observational Health Data Sciencs and Informatics (OHDSI) <a href="http://github.com/OHDSI/PatientLevelPrediction"><code>PatientLevelPrediction</code></a> package. This allows you to fully leverage the OHDSI PatientLevelPrediction framework for model development and validation. This vignette assumes you have read and are comfortable with building single patient level prediction models as described in the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf"><code>BuildingPredictiveModels</code> vignette</a>.</p>
-<p><strong>We invite you to share your new algorithms with the OHDSI community through our <a href="http://github.com/OHDSI/PatientLevelPrediction">GitHub repository</a>.</strong></p>
-</div>
-<div id="algorithm-code-structure" class="section level1">
-<h1 class="hasAnchor">
-<a href="#algorithm-code-structure" class="anchor"></a>Algorithm Code Structure</h1>
-<p>Each algorithm in the package should be implemented in its own &lt;Name&gt;.R file, e.g. KNN.R, containing a set&lt;Name&gt; function and a fit&lt;Name&gt; function. Furthermore, a corresponding predict function in predict.R is needed (if there isn’t one available that would work, see example at the end of the document). We will now describe each of these functions in more detail below.</p>
-<div id="set" class="section level2">
-<h2 class="hasAnchor">
-<a href="#set" class="anchor"></a>Set</h2>
-<p>The set&lt;Name&gt; is a function that takes as input the different hyper-parameter values to do a grid search when training. The output of the functions needs to be a list as class <code>modelSettings</code> containing:</p>
-<ul>
-<li>param - all the combinations of the hyper-parameter values input</li>
-<li>model - a string specifying what function to call to fit the model</li>
-<li>name - a string containing the name of the model.</li>
-</ul>
-<p>For example, if you were adding a model called madeUp that has two hyper-parameters then the set function should be:</p>
-<div class="sourceCode" id="cb1"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">setMadeUp</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">a</span><span class="op">=</span><span class="fl">1</span>, <span class="va">b</span><span class="op">=</span><span class="fl">2</span>, <span class="va">seed</span><span class="op">=</span><span class="cn">NULL</span><span class="op">)</span><span class="op">{</span>
-  <span class="co"># add input checks here...</span>
-  
-  <span class="co"># now create list of all combinations:</span>
-  <span class="va">result</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>model<span class="op">=</span><span class="st">'fitMadeUp'</span>, <span class="co"># this will be called to train the made up model</span>
-                 param<span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/split.html">split</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/expand.grid.html">expand.grid</a></span><span class="op">(</span>a<span class="op">=</span><span class="va">a</span>, 
-                                          b<span class="op">=</span><span class="va">b</span>,
-                                          seed<span class="op">=</span><span class="fu"><a href="https://rdrr.io/r/base/ifelse.html">ifelse</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/NULL.html">is.null</a></span><span class="op">(</span><span class="va">seed</span><span class="op">)</span>,<span class="st">'NULL'</span>, <span class="va">seed</span><span class="op">)</span><span class="op">)</span>,
-                              <span class="fl">1</span><span class="op">:</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="va">a</span><span class="op">)</span><span class="op">*</span><span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="va">b</span><span class="op">)</span>  <span class="op">)</span><span class="op">)</span>,
-                 name<span class="op">=</span><span class="st">'Made Up Algorithm'</span>
-  <span class="op">)</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/class.html">class</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'modelSettings'</span> 
-  
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span>
-<span class="op">}</span></code></pre></div>
-</div>
-<div id="fit" class="section level2">
-<h2 class="hasAnchor">
-<a href="#fit" class="anchor"></a>Fit</h2>
-<p>This function should train your custom model for each parameter entry, pick the best parameters and train a final model for that setting.</p>
-<p>The fit&lt;Model&gt; should have as inputs:</p>
-<ul>
-<li>population - the study popualation the model is being developed on</li>
-<li>plpData - the plpData object</li>
-<li>param - the hyper-parameters as a list of all combinations</li>
-<li>quiet - T or F indicating whether to output progress</li>
-<li>outcomeId - the outcome id</li>
-<li>cohortId - the target population id</li>
-</ul>
-<p>The fit function should return a list of class <code>plpModel</code> with the following objects:</p>
-<ul>
-<li>model - a trained model</li>
-<li>modelSettings - a list containing the model and input param</li>
-<li>trainCVAuc - a value with the train AUC value</li>
-<li>hyperParamSearch - a dataframe with the hyperparameter grid and corresponding AUCs</li>
-<li>metaData - the metaData from the plpData object</li>
-<li>populationSettings - the settings used to create the population and define the time-at-risk</li>
-<li>outcomeId - the outcomeId being predicted</li>
-<li>cohortId - the cohortId corresponding to the target cohort</li>
-<li>varImp - a dataframe with the covaraites and a measure of importance</li>
-<li>trainingTime - how long it took to develop/evaluate the model</li>
-<li>covariateMap - if the plpData are converted to a matrix for model compatibility this tells us what covariate each row in the matrix correpsonds to and is need when implementing the model on new data</li>
-</ul>
-<p>The plpModel returned by fit also has a type attribute, this points to the predict function, for example <code>attr(result, 'type') &lt;- 'madeup'</code> means when the model is applied to new data, the ‘predict.madeup’ function in Predict.R is called. if this doesnt exist, then the model will fail. Another attribute is the predictionType <code>attr(result, 'predictionType') &lt;- 'binary'</code> this is currently not needed but may be important in the future when we expand to regression or multiclass classification.</p>
-<p>For example:</p>
-<div class="sourceCode" id="cb2"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">fitMadeUp</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">population</span>, <span class="va">plpData</span>, <span class="va">param</span>, <span class="va">quiet</span><span class="op">=</span><span class="cn">F</span>,
-                        <span class="va">outcomeId</span>, <span class="va">cohortId</span>, <span class="va">...</span><span class="op">)</span><span class="op">{</span>
-  
-  <span class="co"># **************** code to train the model here</span>
-  <span class="co"># trainedModel &lt;- this code should apply each hyper-parameter using the cross validation</span>
-  <span class="co">#                 then pick out the best hyper-parameter setting</span>
-  <span class="co">#                 and finally fit a model on the whole train data using the </span>
-  <span class="co">#                 optimal hyper-parameter settings</span>
-  <span class="co"># ****************</span>
-  
-  <span class="co"># construct the standard output for a model:</span>
-  <span class="va">result</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>model <span class="op">=</span> <span class="va">trainedModel</span>,
-                 modelSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>model<span class="op">=</span><span class="st">'made_up'</span>, modelParameters<span class="op">=</span><span class="va">param</span><span class="op">)</span>,
-                 trainCVAuc <span class="op">=</span> <span class="cn">NULL</span>,
-                 hyperParamSearch <span class="op">=</span> <span class="va">hyperSummary</span>,
-                 metaData <span class="op">=</span> <span class="va">plpData</span><span class="op">$</span><span class="va">metaData</span>,
-                 populationSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">population</span>, <span class="st">'metaData'</span><span class="op">)</span>,
-                 outcomeId<span class="op">=</span><span class="va">outcomeId</span>,<span class="co"># can use populationSettings$outcomeId?</span>
-                 cohortId<span class="op">=</span><span class="va">cohortId</span>,
-                 varImp <span class="op">=</span> <span class="cn">NULL</span>,
-                 trainingTime<span class="op">=</span><span class="va">comp</span>,
-                 covariateMap<span class="op">=</span><span class="va">result</span><span class="op">$</span><span class="va">map</span>
-  <span class="op">)</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/class.html">class</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'plpModel'</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">result</span>, <span class="st">'type'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'madeup'</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">result</span>, <span class="st">'predictionType'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'binary'</span>
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span>
-    
-<span class="op">}</span></code></pre></div>
-<p>You could make the fitMadeUp function cleaner by adding helper function in the MadeUp.R file that are called by the fit function. As the end of the fit function specified <code>attr(result, 'type') &lt;- 'madeup'</code> we also need to make sure there is a <code>predict.madeup</code> function in Predict.R:</p>
-</div>
-<div id="predict" class="section level2">
-<h2 class="hasAnchor">
-<a href="#predict" class="anchor"></a>Predict</h2>
-<p>The prediction function takes as input the plpModel returned by fit, a population and corresponding plpData. It returns a data.frame with the columns:</p>
-<ul>
-<li>rowId - the id for each person in the population</li>
-<li>value - the predicted risk from the plpModel</li>
-</ul>
-<p>If the population contains the columns outcomeCount and indexes, then these are also in the output.</p>
-<p>For example:</p>
-<div class="sourceCode" id="cb3"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">predict.madeup</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">plpModel</span>,<span class="va">population</span>, <span class="va">plpData</span>, <span class="va">...</span><span class="op">)</span><span class="op">{</span> 
-
-  <span class="co"># ************* code to do prediction for each rowId in population</span>
-  <span class="co"># prediction &lt;- code to do prediction here returning columns: rowId </span>
-  <span class="co">#               and value (predicted risk)</span>
-  <span class="co">#**************</span>
-  
-  <span class="va">prediction</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/merge.html">merge</a></span><span class="op">(</span><span class="va">population</span>, <span class="va">prediction</span>, by<span class="op">=</span><span class="st">'rowId'</span><span class="op">)</span>
-  <span class="va">prediction</span> <span class="op">&lt;-</span> <span class="va">prediction</span><span class="op">[</span>,<span class="fu"><a href="https://rdrr.io/r/base/colnames.html">colnames</a></span><span class="op">(</span><span class="va">prediction</span><span class="op">)</span><span class="op">%in%</span><span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="st">'rowId'</span>,<span class="st">'outcomeCount'</span>,
-                                                      <span class="st">'indexes'</span>, <span class="st">'value'</span><span class="op">)</span><span class="op">]</span> 
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">prediction</span>, <span class="st">"metaData"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>predictionType <span class="op">=</span> <span class="st">"binary"</span><span class="op">)</span> 
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">prediction</span><span class="op">)</span>
-  
-<span class="op">}</span></code></pre></div>
-</div>
-</div>
-<div id="algorithm-example" class="section level1">
-<h1 class="hasAnchor">
-<a href="#algorithm-example" class="anchor"></a>Algorithm Example</h1>
-<p>Below a fully functional algorithm example is given, however we highly recommend you to have a look at the available algorithms in the package.</p>
-<div id="set-1" class="section level2">
-<h2 class="hasAnchor">
-<a href="#set-1" class="anchor"></a>Set</h2>
-<div class="sourceCode" id="cb4"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a>setMadeUp <span class="ot">&lt;-</span> <span class="cf">function</span>(<span class="at">a=</span><span class="dv">1</span>, <span class="at">b=</span><span class="dv">2</span>, <span class="at">seed=</span><span class="cn">NULL</span>){</span>
-<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a>  <span class="co"># check a is valid positive value</span></span>
-<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a>  <span class="cf">if</span>(<span class="fu">missing</span>(a)){</span>
-<span id="cb4-4"><a href="#cb4-4" aria-hidden="true" tabindex="-1"></a>    <span class="fu">stop</span>(<span class="st">'a must be input'</span>)</span>
-<span id="cb4-5"><a href="#cb4-5" aria-hidden="true" tabindex="-1"></a>  }</span>
-<span id="cb4-6"><a href="#cb4-6" aria-hidden="true" tabindex="-1"></a>  <span class="cf">if</span>(<span class="sc">!</span><span class="fu">class</span>(a)<span class="sc">%in%</span><span class="fu">c</span>(<span class="st">'numeric'</span>,<span class="st">'integer'</span>){</span>
-<span id="cb4-7"><a href="#cb4-7" aria-hidden="true" tabindex="-1"></a>    <span class="fu">stop</span>(<span class="st">'a must be numeric'</span>)</span>
-<span id="cb4-8"><a href="#cb4-8" aria-hidden="true" tabindex="-1"></a>  }</span>
-<span id="cb4-9"><a href="#cb4-9" aria-hidden="true" tabindex="-1"></a>  <span class="cf">if</span>(a <span class="sc">&lt;</span> <span class="dv">0</span>){</span>
-<span id="cb4-10"><a href="#cb4-10" aria-hidden="true" tabindex="-1"></a>    <span class="fu">stop</span>(<span class="st">'a must be positive'</span>)</span>
-<span id="cb4-11"><a href="#cb4-11" aria-hidden="true" tabindex="-1"></a>  }</span>
-<span id="cb4-12"><a href="#cb4-12" aria-hidden="true" tabindex="-1"></a>  <span class="co"># check b is numeric</span></span>
-<span id="cb4-13"><a href="#cb4-13" aria-hidden="true" tabindex="-1"></a>  <span class="cf">if</span>(<span class="fu">missing</span>(b)){</span>
-<span id="cb4-14"><a href="#cb4-14" aria-hidden="true" tabindex="-1"></a>    <span class="fu">stop</span>(<span class="st">'b must be input'</span>)</span>
-<span id="cb4-15"><a href="#cb4-15" aria-hidden="true" tabindex="-1"></a>  }</span>
-<span id="cb4-16"><a href="#cb4-16" aria-hidden="true" tabindex="-1"></a>  <span class="cf">if</span>(<span class="sc">!</span><span class="fu">class</span>(b)<span class="sc">%in%</span><span class="fu">c</span>(<span class="st">'numeric'</span>,<span class="st">'integer'</span>){</span>
-<span id="cb4-17"><a href="#cb4-17" aria-hidden="true" tabindex="-1"></a>    <span class="fu">stop</span>(<span class="st">'b must be numeric'</span>)</span>
-<span id="cb4-18"><a href="#cb4-18" aria-hidden="true" tabindex="-1"></a>  }</span>
-<span id="cb4-19"><a href="#cb4-19" aria-hidden="true" tabindex="-1"></a>  </span>
-<span id="cb4-20"><a href="#cb4-20" aria-hidden="true" tabindex="-1"></a>  <span class="co"># now create list of all combinations:</span></span>
-<span id="cb4-21"><a href="#cb4-21" aria-hidden="true" tabindex="-1"></a>  result <span class="ot">&lt;-</span> <span class="fu">list</span>(<span class="at">model=</span><span class="st">'fitMadeUp'</span>, </span>
-<span id="cb4-22"><a href="#cb4-22" aria-hidden="true" tabindex="-1"></a>                 <span class="at">param=</span> <span class="fu">split</span>(<span class="fu">expand.grid</span>(<span class="at">a=</span>a, </span>
-<span id="cb4-23"><a href="#cb4-23" aria-hidden="true" tabindex="-1"></a>                                          <span class="at">b=</span>b,</span>
-<span id="cb4-24"><a href="#cb4-24" aria-hidden="true" tabindex="-1"></a>                                          <span class="at">seed=</span><span class="fu">ifelse</span>(<span class="fu">is.null</span>(seed),<span class="st">'NULL'</span>, seed)),</span>
-<span id="cb4-25"><a href="#cb4-25" aria-hidden="true" tabindex="-1"></a>                              <span class="dv">1</span><span class="sc">:</span>(<span class="fu">length</span>(a)<span class="sc">*</span><span class="fu">length</span>(b)  )),</span>
-<span id="cb4-26"><a href="#cb4-26" aria-hidden="true" tabindex="-1"></a>                 <span class="at">name=</span><span class="st">'Made Up Algorithm'</span></span>
-<span id="cb4-27"><a href="#cb4-27" aria-hidden="true" tabindex="-1"></a>  )</span>
-<span id="cb4-28"><a href="#cb4-28" aria-hidden="true" tabindex="-1"></a>  <span class="fu">class</span>(result) <span class="ot">&lt;-</span> <span class="st">'modelSettings'</span> </span>
-<span id="cb4-29"><a href="#cb4-29" aria-hidden="true" tabindex="-1"></a>  </span>
-<span id="cb4-30"><a href="#cb4-30" aria-hidden="true" tabindex="-1"></a>  <span class="fu">return</span>(result)</span>
-<span id="cb4-31"><a href="#cb4-31" aria-hidden="true" tabindex="-1"></a>    </span>
-<span id="cb4-32"><a href="#cb4-32" aria-hidden="true" tabindex="-1"></a>  </span>
-<span id="cb4-33"><a href="#cb4-33" aria-hidden="true" tabindex="-1"></a><span class="er">}</span></span></code></pre></div>
-</div>
-<div id="fit-1" class="section level2">
-<h2 class="hasAnchor">
-<a href="#fit-1" class="anchor"></a>Fit</h2>
-<div class="sourceCode" id="cb5"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">fitMadeUp</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">population</span>, <span class="va">plpData</span>, <span class="va">param</span>, <span class="va">quiet</span><span class="op">=</span><span class="cn">F</span>,
-                        <span class="va">outcomeId</span>, <span class="va">cohortId</span>, <span class="va">...</span><span class="op">)</span><span class="op">{</span>
-    <span class="kw">if</span><span class="op">(</span><span class="op">!</span><span class="va">quiet</span><span class="op">)</span>
-    <span class="fu"><a href="https://rdrr.io/r/base/writeLines.html">writeLines</a></span><span class="op">(</span><span class="st">'Training Made Up model'</span><span class="op">)</span>
-  
-  <span class="kw">if</span><span class="op">(</span><span class="va">param</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">$</span><span class="va">seed</span><span class="op">!=</span><span class="st">'NULL'</span><span class="op">)</span>
-    <span class="fu"><a href="https://rdrr.io/r/base/Random.html">set.seed</a></span><span class="op">(</span><span class="va">param</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">$</span><span class="va">seed</span><span class="op">)</span>
-  
-    <span class="co"># check plpData is coo format:</span>
-  <span class="kw">if</span><span class="op">(</span><span class="op">!</span><span class="st">'ffdf'</span><span class="op">%in%</span><span class="fu"><a href="https://rdrr.io/r/base/class.html">class</a></span><span class="op">(</span><span class="va">plpData</span><span class="op">$</span><span class="va">covariates</span><span class="op">)</span> <span class="op">)</span>
-    <span class="kw"><a href="https://rdrr.io/r/base/stop.html">stop</a></span><span class="op">(</span><span class="st">'This algorithm requires plpData in coo format'</span><span class="op">)</span>
-  
-  <span class="va">metaData</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">population</span>, <span class="st">'metaData'</span><span class="op">)</span>
-  <span class="kw">if</span><span class="op">(</span><span class="op">!</span><span class="fu"><a href="https://rdrr.io/r/base/NULL.html">is.null</a></span><span class="op">(</span><span class="va">population</span><span class="op">$</span><span class="va">indexes</span><span class="op">)</span><span class="op">)</span>
-    <span class="va">population</span> <span class="op">&lt;-</span> <span class="va">population</span><span class="op">[</span><span class="va">population</span><span class="op">$</span><span class="va">indexes</span><span class="op">&gt;</span><span class="fl">0</span>,<span class="op">]</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">population</span>, <span class="st">'metaData'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">metaData</span>
-  
-  <span class="co"># convert data into sparse R Matrix:</span>
-  <span class="va">result</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/toSparseM.html">toSparseM</a></span><span class="op">(</span><span class="va">plpData</span>,<span class="va">population</span>,map<span class="op">=</span><span class="cn">NULL</span><span class="op">)</span>
-  <span class="va">data</span> <span class="op">&lt;-</span> <span class="va">result</span><span class="op">$</span><span class="va">data</span>
-  
-  <span class="va">data</span> <span class="op">&lt;-</span> <span class="va">data</span><span class="op">[</span><span class="va">population</span><span class="op">$</span><span class="va">rowId</span>,<span class="op">]</span>
-  
-  <span class="co"># set test/train sets (for printing performance as it trains)</span>
-  <span class="kw">if</span><span class="op">(</span><span class="op">!</span><span class="va">quiet</span><span class="op">)</span>
-    <span class="fu"><a href="https://rdrr.io/r/base/writeLines.html">writeLines</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste0</a></span><span class="op">(</span><span class="st">'Training made up model on train set containing '</span>, <span class="fu"><a href="https://rdrr.io/r/base/nrow.html">nrow</a></span><span class="op">(</span><span class="va">population</span><span class="op">)</span>, 
-                      <span class="st">' people with '</span>,<span class="fu"><a href="https://rdrr.io/r/base/sum.html">sum</a></span><span class="op">(</span><span class="va">population</span><span class="op">$</span><span class="va">outcomeCount</span><span class="op">&gt;</span><span class="fl">0</span><span class="op">)</span>, <span class="st">' outcomes'</span><span class="op">)</span><span class="op">)</span>
-  <span class="va">start</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/Sys.time.html">Sys.time</a></span><span class="op">(</span><span class="op">)</span>
-  
-  <span class="co">#============= STEP 1 ======================================</span>
-  <span class="co"># pick the best hyper-params and then do final training on all data...</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/writeLines.html">writeLines</a></span><span class="op">(</span><span class="st">'train'</span><span class="op">)</span>
-  <span class="va">datas</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>population<span class="op">=</span><span class="va">population</span>, data<span class="op">=</span><span class="va">data</span><span class="op">)</span>
-  <span class="va">param.sel</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/lapply.html">lapply</a></span><span class="op">(</span><span class="va">param</span>, <span class="kw">function</span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="fu"><a href="https://rdrr.io/r/base/do.call.html">do.call</a></span><span class="op">(</span><span class="va">made_up_model</span>, <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="va">x</span>,<span class="va">datas</span><span class="op">)</span>  <span class="op">)</span><span class="op">)</span>
-  <span class="va">hyperSummary</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/do.call.html">do.call</a></span><span class="op">(</span><span class="va">rbind</span>, <span class="fu"><a href="https://rdrr.io/r/base/lapply.html">lapply</a></span><span class="op">(</span><span class="va">param.sel</span>, <span class="kw">function</span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="va">x</span><span class="op">$</span><span class="va">hyperSum</span><span class="op">)</span><span class="op">)</span>
-  <span class="va">hyperSummary</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/as.data.frame.html">as.data.frame</a></span><span class="op">(</span><span class="va">hyperSummary</span><span class="op">)</span>
-  <span class="va">hyperSummary</span><span class="op">$</span><span class="va">auc</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/unlist.html">unlist</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/lapply.html">lapply</a></span><span class="op">(</span><span class="va">param.sel</span>, <span class="kw">function</span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="va">x</span><span class="op">$</span><span class="va">auc</span><span class="op">)</span><span class="op">)</span> 
-  <span class="va">param.sel</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/unlist.html">unlist</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/lapply.html">lapply</a></span><span class="op">(</span><span class="va">param.sel</span>, <span class="kw">function</span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="va">x</span><span class="op">$</span><span class="va">auc</span><span class="op">)</span><span class="op">)</span>
-  <span class="va">param</span> <span class="op">&lt;-</span> <span class="va">param</span><span class="op">[[</span><span class="fu"><a href="https://rdrr.io/r/base/which.min.html">which.max</a></span><span class="op">(</span><span class="va">param.sel</span><span class="op">)</span><span class="op">]</span><span class="op">]</span>
-  
-  <span class="co"># set this so you do a final model train </span>
-  <span class="va">param</span><span class="op">$</span><span class="va">final</span><span class="op">=</span><span class="cn">T</span>
-  
-  <span class="fu"><a href="https://rdrr.io/r/base/writeLines.html">writeLines</a></span><span class="op">(</span><span class="st">'final train'</span><span class="op">)</span>
-  <span class="va">trainedModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/do.call.html">do.call</a></span><span class="op">(</span><span class="va">made_up_model</span>, <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="va">param</span>,<span class="va">datas</span><span class="op">)</span>  <span class="op">)</span><span class="op">$</span><span class="va">model</span>
-  
-  <span class="va">comp</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/Sys.time.html">Sys.time</a></span><span class="op">(</span><span class="op">)</span> <span class="op">-</span> <span class="va">start</span>
-  <span class="kw">if</span><span class="op">(</span><span class="op">!</span><span class="va">quiet</span><span class="op">)</span>
-    <span class="fu"><a href="https://rdrr.io/r/base/writeLines.html">writeLines</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste0</a></span><span class="op">(</span><span class="st">'Model Made Up trained - took:'</span>,  <span class="fu"><a href="https://rdrr.io/r/base/format.html">format</a></span><span class="op">(</span><span class="va">comp</span>, digits<span class="op">=</span><span class="fl">3</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>
-  
-  <span class="co"># construct the standard output for a model:</span>
-  <span class="va">result</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>model <span class="op">=</span> <span class="va">trainedModel</span>,
-                 modelSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>model<span class="op">=</span><span class="st">'made_up'</span>, modelParameters<span class="op">=</span><span class="va">param</span><span class="op">)</span>,
-                 trainCVAuc <span class="op">=</span> <span class="cn">NULL</span>,
-                 hyperParamSearch <span class="op">=</span> <span class="va">hyperSummary</span>,
-                 metaData <span class="op">=</span> <span class="va">plpData</span><span class="op">$</span><span class="va">metaData</span>,
-                 populationSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">population</span>, <span class="st">'metaData'</span><span class="op">)</span>,
-                 outcomeId<span class="op">=</span><span class="va">outcomeId</span>,<span class="co"># can use populationSettings$outcomeId?</span>
-                 cohortId<span class="op">=</span><span class="va">cohortId</span>,
-                 varImp <span class="op">=</span> <span class="cn">NULL</span>,
-                 trainingTime<span class="op">=</span><span class="va">comp</span>,
-                 covariateMap<span class="op">=</span><span class="va">result</span><span class="op">$</span><span class="va">map</span>
-  <span class="op">)</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/class.html">class</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'plpModel'</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">result</span>, <span class="st">'type'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'madeup'</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">result</span>, <span class="st">'predictionType'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'binary'</span>
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span>
-    
-<span class="op">}</span></code></pre></div>
-</div>
-<div id="helpers" class="section level2">
-<h2 class="hasAnchor">
-<a href="#helpers" class="anchor"></a>Helpers</h2>
-<p>In the fit model a helper function <code>made_up_model</code> is called, this is the function that trains a model given the data and population (where the popualtion contains a column outcomeCount corresponding to the outcome). Both the data and population are ordered the same way:</p>
-<div class="sourceCode" id="cb6"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">made_up_model</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">data</span>, <span class="va">population</span>,
-                       <span class="va">a</span><span class="op">=</span><span class="fl">1</span>,<span class="va">b</span><span class="op">=</span><span class="fl">1</span>, <span class="va">final</span><span class="op">=</span><span class="cn">F</span>, <span class="va">...</span><span class="op">)</span><span class="op">{</span>
-  
-  <span class="fu"><a href="https://rdrr.io/r/base/writeLines.html">writeLines</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste</a></span><span class="op">(</span><span class="st">'Training Made Up model with '</span>,<span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/unique.html">unique</a></span><span class="op">(</span><span class="va">population</span><span class="op">$</span><span class="va">indexes</span><span class="op">)</span><span class="op">)</span>,
-                   <span class="st">' fold CV'</span><span class="op">)</span><span class="op">)</span>
-  <span class="kw">if</span><span class="op">(</span><span class="op">!</span><span class="fu"><a href="https://rdrr.io/r/base/NULL.html">is.null</a></span><span class="op">(</span><span class="va">population</span><span class="op">$</span><span class="va">indexes</span><span class="op">)</span> <span class="op">&amp;&amp;</span> <span class="va">final</span><span class="op">==</span><span class="cn">F</span><span class="op">)</span><span class="op">{</span>
-    <span class="va">index_vect</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/unique.html">unique</a></span><span class="op">(</span><span class="va">population</span><span class="op">$</span><span class="va">indexes</span><span class="op">)</span>
-    <span class="va">perform</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="op">)</span>
-    
-    <span class="co"># create prediction matrix to store all predictions</span>
-    <span class="va">predictionMat</span> <span class="op">&lt;-</span> <span class="va">population</span>
-    <span class="va">predictionMat</span><span class="op">$</span><span class="va">value</span> <span class="op">&lt;-</span> <span class="fl">0</span>
-    <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">predictionMat</span>, <span class="st">"metaData"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>predictionType <span class="op">=</span> <span class="st">"binary"</span><span class="op">)</span>
-    
-    <span class="kw">for</span><span class="op">(</span><span class="va">index</span> <span class="kw">in</span> <span class="fl">1</span><span class="op">:</span><span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="va">index_vect</span> <span class="op">)</span><span class="op">)</span><span class="op">{</span>
-      <span class="fu"><a href="https://rdrr.io/r/base/writeLines.html">writeLines</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste</a></span><span class="op">(</span><span class="st">'Fold '</span>,<span class="va">index</span>, <span class="st">' -- with '</span>, <span class="fu"><a href="https://rdrr.io/r/base/sum.html">sum</a></span><span class="op">(</span><span class="va">population</span><span class="op">$</span><span class="va">indexes</span><span class="op">!=</span><span class="va">index</span><span class="op">)</span>,
-                       <span class="st">'train rows'</span><span class="op">)</span><span class="op">)</span>
-      <span class="va">model</span> <span class="op">&lt;-</span> <span class="fu">madeup</span><span class="fu">::</span><span class="fu">model</span><span class="op">(</span>x <span class="op">=</span> <span class="va">data</span><span class="op">[</span><span class="va">population</span><span class="op">$</span><span class="va">indexes</span><span class="op">!=</span><span class="va">index</span>,<span class="op">]</span>,
-                             y<span class="op">=</span> <span class="va">population</span><span class="op">$</span><span class="va">outcomeCount</span><span class="op">[</span><span class="va">population</span><span class="op">$</span><span class="va">indexes</span><span class="op">!=</span><span class="va">index</span><span class="op">]</span>,
-                                  a<span class="op">=</span><span class="va">a</span>, b<span class="op">=</span><span class="va">b</span><span class="op">)</span>
-      
-      <span class="va">pred</span> <span class="op">&lt;-</span> <span class="fu">stats</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/r/stats/predict.html">predict</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">data</span><span class="op">[</span><span class="va">population</span><span class="op">$</span><span class="va">indexes</span><span class="op">==</span><span class="va">index</span>,<span class="op">]</span><span class="op">)</span>
-      <span class="va">prediction</span> <span class="op">&lt;-</span> <span class="va">population</span><span class="op">[</span><span class="va">population</span><span class="op">$</span><span class="va">indexes</span><span class="op">==</span><span class="va">index</span>,<span class="op">]</span>
-      <span class="va">prediction</span><span class="op">$</span><span class="va">value</span> <span class="op">&lt;-</span> <span class="va">pred</span>
-      <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">prediction</span>, <span class="st">"metaData"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>predictionType <span class="op">=</span> <span class="st">"binary"</span><span class="op">)</span>
-      <span class="va">aucVal</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/computeAuc.html">computeAuc</a></span><span class="op">(</span><span class="va">prediction</span><span class="op">)</span>
-      <span class="va">perform</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="va">perform</span>,<span class="va">aucVal</span><span class="op">)</span>
-      
-      <span class="co"># add the fold predictions and compute AUC after loop</span>
-      <span class="va">predictionMat</span><span class="op">$</span><span class="va">value</span><span class="op">[</span><span class="va">population</span><span class="op">$</span><span class="va">indexes</span><span class="op">==</span><span class="va">index</span><span class="op">]</span> <span class="op">&lt;-</span> <span class="va">pred</span>
-      
-     <span class="op">}</span>
-    <span class="co">##auc &lt;- mean(perform) # want overal rather than mean</span>
-    <span class="va">auc</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/computeAuc.html">computeAuc</a></span><span class="op">(</span><span class="va">predictionMat</span><span class="op">)</span>
-    
-    <span class="va">foldPerm</span> <span class="op">&lt;-</span> <span class="va">perform</span>
-  <span class="op">}</span> <span class="kw">else</span> <span class="op">{</span>
-    <span class="va">model</span> <span class="op">&lt;-</span> <span class="fu">madeup</span><span class="fu">::</span><span class="fu">model</span><span class="op">(</span>x<span class="op">=</span> <span class="va">data</span>, 
-                                y<span class="op">=</span> <span class="va">population</span><span class="op">$</span><span class="va">outcomeCount</span>,
-                                a<span class="op">=</span><span class="va">a</span>,b<span class="op">=</span><span class="va">b</span><span class="op">)</span>
-    
-    <span class="va">pred</span> <span class="op">&lt;-</span> <span class="fu">stats</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/r/stats/predict.html">predict</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">data</span><span class="op">)</span>
-    <span class="va">prediction</span> <span class="op">&lt;-</span> <span class="va">population</span>
-    <span class="va">prediction</span><span class="op">$</span><span class="va">value</span> <span class="op">&lt;-</span> <span class="va">pred</span>
-    <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">prediction</span>, <span class="st">"metaData"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>predictionType <span class="op">=</span> <span class="st">"binary"</span><span class="op">)</span> 
-    <span class="va">auc</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/computeAuc.html">computeAuc</a></span><span class="op">(</span><span class="va">prediction</span><span class="op">)</span>
-    <span class="va">foldPerm</span> <span class="op">&lt;-</span> <span class="va">auc</span>
-  <span class="op">}</span>
-  
-  <span class="va">result</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>model<span class="op">=</span><span class="va">model</span>,
-                 auc<span class="op">=</span><span class="va">auc</span>,
-                 hyperSum <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/unlist.html">unlist</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>a <span class="op">=</span> <span class="va">a</span>, b <span class="op">=</span> <span class="va">b</span>, fold_auc<span class="op">=</span><span class="va">foldPerm</span><span class="op">)</span><span class="op">)</span>
-  <span class="op">)</span>
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span>
-<span class="op">}</span></code></pre></div>
-</div>
-<div id="predict-1" class="section level2">
-<h2 class="hasAnchor">
-<a href="#predict-1" class="anchor"></a>Predict</h2>
-<p>The final step is to create a predict function for the model. This gets added to the predict.R file. In the example above the type <code>attr(result, 'type') &lt;- 'madeup'</code> was madeup, so a <code>predict.madeup</code> function is required to be added into the predict.R. The predict function needs to take as input the plpModel returned by the fit function, the population to apply the model on and the plpData specifying the covariates of the population.</p>
-<div class="sourceCode" id="cb7"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">predict.madeup</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">plpModel</span>,<span class="va">population</span>, <span class="va">plpData</span>, <span class="va">...</span><span class="op">)</span><span class="op">{</span> 
-  <span class="va">result</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/toSparseM.html">toSparseM</a></span><span class="op">(</span><span class="va">plpData</span>, <span class="va">population</span>, map<span class="op">=</span><span class="va">plpModel</span><span class="op">$</span><span class="va">covariateMap</span><span class="op">)</span>
-  <span class="va">data</span> <span class="op">&lt;-</span> <span class="va">result</span><span class="op">$</span><span class="va">data</span><span class="op">[</span><span class="va">population</span><span class="op">$</span><span class="va">rowId</span>,<span class="op">]</span>
-  <span class="va">prediction</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html">data.frame</a></span><span class="op">(</span>rowId<span class="op">=</span><span class="va">population</span><span class="op">$</span><span class="va">rowId</span>, 
-                           value<span class="op">=</span><span class="fu">stats</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/r/stats/predict.html">predict</a></span><span class="op">(</span><span class="va">plpModel</span><span class="op">$</span><span class="va">model</span>, <span class="va">data</span><span class="op">)</span>
-                           <span class="op">)</span>
-  
-  <span class="va">prediction</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/merge.html">merge</a></span><span class="op">(</span><span class="va">population</span>, <span class="va">prediction</span>, by<span class="op">=</span><span class="st">'rowId'</span><span class="op">)</span>
-  <span class="va">prediction</span> <span class="op">&lt;-</span> <span class="va">prediction</span><span class="op">[</span>,<span class="fu"><a href="https://rdrr.io/r/base/colnames.html">colnames</a></span><span class="op">(</span><span class="va">prediction</span><span class="op">)</span><span class="op">%in%</span>
-                           <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="st">'rowId'</span>,<span class="st">'outcomeCount'</span>,<span class="st">'indexes'</span>, <span class="st">'value'</span><span class="op">)</span><span class="op">]</span> <span class="co"># need to fix no index issue</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">prediction</span>, <span class="st">"metaData"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>predictionType <span class="op">=</span> <span class="st">"binary"</span><span class="op">)</span> 
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">prediction</span><span class="op">)</span>
-  
-<span class="op">}</span></code></pre></div>
-<p>As the madeup model uses the standard R prediction, it has the same prediction function as xgboost, so we could have not added a new prediction function and instead made the type of the result returned by fitMadeUpModel to <code>attr(result, 'type') &lt;- 'xgboost'</code>.</p>
-</div>
-</div>
-<div id="acknowledgments" class="section level1">
-<h1 class="hasAnchor">
-<a href="#acknowledgments" class="anchor"></a>Acknowledgments</h1>
-<p>Considerable work has been dedicated to provide the <code>PatientLevelPrediction</code> package.</p>
-<div class="sourceCode" id="cb8"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/utils/citation.html">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></code></pre></div>
-<pre><code>## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design
-## and implementation of a standardized framework to generate and evaluate
-## patient-level prediction models using observational healthcare data."
-## _Journal of the American Medical Informatics Association_, *25*(8),
-## 969-975. &lt;URL: https://doi.org/10.1093/jamia/ocy032&gt;.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }</code></pre>
-<p><strong>Please reference this paper if you use the PLP Package in your work:</strong></p>
-<p><a href="http://dx.doi.org/10.1093/jamia/ocy032">Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.</a></p>
-<p>This work is supported in part through the National Science Foundation grant IIS 1251151.</p>
-</div>
-  </div>
-
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-
-        <nav id="toc" data-toggle="toc"><h2 data-toc-skip>Contents</h2>
-    </nav>
-</div>
-
-</div>
-
-
-
-      <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-</div>
-
-  
-
-
-  </body>
-</html>
diff --git a/docs/articles/AddingCustomAlgorithms_files/header-attrs-2.7/header-attrs.js b/docs/articles/AddingCustomAlgorithms_files/header-attrs-2.7/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/AddingCustomAlgorithms_files/header-attrs-2.7/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/AddingCustomFeatureEngineering.html b/docs/articles/AddingCustomFeatureEngineering.html
index ba6f86c94..b7261a0cc 100644
--- a/docs/articles/AddingCustomFeatureEngineering.html
+++ b/docs/articles/AddingCustomFeatureEngineering.html
@@ -19,6 +19,8 @@
 <![endif]-->
 </head>
 <body data-spy="scroll" data-target="#toc">
+    
+
     <div class="container template-article">
       <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
@@ -31,7 +33,7 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
@@ -53,7 +55,7 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
@@ -72,10 +74,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -84,13 +86,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -101,21 +106,30 @@
   </ul>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
       </ul>
 <ul class="nav navbar-nav navbar-right">
 <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
@@ -130,15 +144,16 @@
 
       
 
-      </header><script src="AddingCustomFeatureEngineering_files/header-attrs-2.11/header-attrs.js"></script><div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header toc-ignore">
       <h1 data-toc-skip>Adding Custom Feature Engineering Functions</h1>
-                        <h4 class="author">Jenna Reps</h4>
+                        <h4 data-toc-skip class="author">Jenna Reps,
+Egill Fridgeirsson</h4>
             
-            <h4 class="date">2022-03-09</h4>
+            <h4 data-toc-skip class="date">2024-09-09</h4>
       
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/AddingCustomFeatureEngineering.Rmd"><code>vignettes/AddingCustomFeatureEngineering.Rmd</code></a></small>
+      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/vignettes/AddingCustomFeatureEngineering.Rmd" class="external-link"><code>vignettes/AddingCustomFeatureEngineering.Rmd</code></a></small>
       <div class="hidden name"><code>AddingCustomFeatureEngineering.Rmd</code></div>
 
     </div>
@@ -146,138 +161,206 @@ <h4 class="date">2022-03-09</h4>
     
     
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Adding Custom Feature Engineering}
 -->
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
-<p>This vignette describes how you can add your own custom function for feature engineering in the Observational Health Data Sciencs and Informatics (OHDSI) <a href="http://github.com/OHDSI/PatientLevelPrediction"><code>PatientLevelPrediction</code></a> package. This vignette assumes you have read and are comfortable with building single patient level prediction models as described in the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf"><code>BuildingPredictiveModels</code> vignette</a>.</p>
-<p><strong>We invite you to share your new feature engineering functions with the OHDSI community through our <a href="http://github.com/OHDSI/PatientLevelPrediction">GitHub repository</a>.</strong></p>
+<div class="section level2">
+<h2 id="introduction">Introduction<a class="anchor" aria-label="anchor" href="#introduction"></a>
+</h2>
+<p>This vignette describes how you can add your own custom function for
+feature engineering in the Observational Health Data Sciences and
+Informatics (OHDSI) <a href="http://github.com/OHDSI/PatientLevelPrediction" class="external-link"><code>PatientLevelPrediction</code></a>
+package. This vignette assumes you have read and are comfortable with
+building single patient level prediction models as described in the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingPredictiveModels.pdf" class="external-link"><code>BuildingPredictiveModels</code>
+vignette</a>.</p>
+<p><strong>We invite you to share your new feature engineering functions
+with the OHDSI community through our <a href="http://github.com/OHDSI/PatientLevelPrediction" class="external-link">GitHub
+repository</a>.</strong></p>
 </div>
-<div id="feature-engineering-function-code-structure" class="section level1">
-<h1 class="hasAnchor">
-<a href="#feature-engineering-function-code-structure" class="anchor"></a>Feature Engineering Function Code Structure</h1>
-<p>To make a custom feature engineering function that can be used within PatientLevelPrediction you need to write two different functions. The ‘create’ function and the ‘implement’ function.</p>
-<p>The ‘create’ function, e.g., create&lt;FeatureEngineeringFunctionName&gt;, takes the parameters of the feature engineering ‘implement’ function as input, checks these are valid and outputs these as a list of class ‘featureEngineeringSettings’ with the ‘fun’ attribute specifying the ‘implement’ function to call.</p>
-<p>The ‘implement’ function, e.g., implement&lt;FeatureEngineeringFunctionName&gt;, must take as input: * trainData - a list containing: - covariateData: the plpData$covariateData restricted to the training patients - labels: a data frame that contain rowId (patient identifier) and outcomeCount (the class labels) - folds: a data.frame that contains rowId (patient identifier) and index (the cross validation fold) * featureEngineeringSettings - the output of your create&lt;FeatureEngineeringFunctionName&gt;</p>
-<p>The ‘implement’ function can then do any manipulation of the trainData (adding new features or removing features) but must output a trainData object containing the new covariateData, labels and folds for the training data patients.</p>
+<div class="section level2">
+<h2 id="feature-engineering-function-code-structure">Feature Engineering Function Code Structure<a class="anchor" aria-label="anchor" href="#feature-engineering-function-code-structure"></a>
+</h2>
+<p>To make a custom feature engineering function that can be used within
+PatientLevelPrediction you need to write two different functions. The
+‘create’ function and the ‘implement’ function.</p>
+<p>The ‘create’ function, e.g.,
+create&lt;FeatureEngineeringFunctionName&gt;, takes the parameters of
+the feature engineering ‘implement’ function as input, checks these are
+valid and outputs these as a list of class ‘featureEngineeringSettings’
+with the ‘fun’ attribute specifying the ‘implement’ function to
+call.</p>
+<p>The ‘implement’ function, e.g.,
+implement&lt;FeatureEngineeringFunctionName&gt;, must take as input:</p>
+<ul>
+<li>
+<p><code>trainData</code> - a list containing:</p>
+<ul>
+<li><p><code>covariateData</code>: the
+<code>plpData$covariateData</code>restricted to the training
+patients</p></li>
+<li><p><code>labels</code>: a data frame that contain
+<code>rowId</code>(patient identifier) and <code>outcomeCount</code>
+(the class labels)</p></li>
+<li><p><code>folds</code>: a data.frame that contains <code>rowId</code>
+(patient identifier) and <code>index</code> (the cross validation
+fold)</p></li>
+</ul>
+</li>
+<li><p><code>featureEngineeringSettings</code> - the output of your
+create&lt;FeatureEngineeringFunctionName&gt;</p></li>
+</ul>
+<p>The ‘implement’ function can then do any manipulation of the
+<code>trainData</code> (adding new features or removing features) but
+must output a <code>trainData</code> object containing the new
+<code>covariateData</code>, <code>labels</code> and <code>folds</code>
+for the training data patients.</p>
 </div>
-<div id="example" class="section level1">
-<h1 class="hasAnchor">
-<a href="#example" class="anchor"></a>Example</h1>
-<p>Let’s consider the situation where we wish to create an age spline feature. To make this custom feature engineering function we need to write the ‘create’ and ‘implement’ R functions.</p>
-<div id="create-function" class="section level2">
-<h2 class="hasAnchor">
-<a href="#create-function" class="anchor"></a>Create function</h2>
-<p>Our age spline feature function will create a new feature using the plpData$cohorts ageYear column. We will implement a restricted cubic spline that requires specifying the number of knots. . Therefore, the inputs for this are: * <code>knots</code> an integer/double specifying the number of knots</p>
+<div class="section level2">
+<h2 id="example">Example<a class="anchor" aria-label="anchor" href="#example"></a>
+</h2>
+<p>Let’s consider the situation where we wish to create an age spline
+feature. To make this custom feature engineering function we need to
+write the ‘create’ and ‘implement’ R functions.</p>
+<div class="section level3">
+<h3 id="create-function">Create function<a class="anchor" aria-label="anchor" href="#create-function"></a>
+</h3>
+<p>Our age spline feature function will create a new feature using the
+<code>plpData$cohorts$ageYear</code> column. We will implement a
+restricted cubic spline that requires specifying the number of knots.
+Therefore, the inputs for this are: <code>knots</code> - an
+integer/double specifying the number of knots.</p>
 <div class="sourceCode" id="cb1"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">createAgeSpine</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span>
-                     <span class="va">knots</span> <span class="op">=</span> <span class="fl">5</span>
-                     <span class="op">)</span><span class="op">{</span>
-  
-  <span class="co"># add input checks</span>
-  <span class="fu">checkIsClass</span><span class="op">(</span><span class="va">knots</span>, <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="st">'numeric'</span>,<span class="st">'integer'</span><span class="op">)</span><span class="op">)</span>
-  <span class="fu">checkHigher</span><span class="op">(</span><span class="va">knots</span>,<span class="fl">0</span><span class="op">)</span>
-  
-  <span class="co"># create list of inputs to implement function</span>
-  <span class="va">featureEngineeringSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>
-    knots <span class="op">=</span> <span class="va">knots</span>
-    <span class="op">)</span>
-  
-  <span class="co"># specify the function that will implement the sampling</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">featureEngineeringSettings</span>, <span class="st">"fun"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">"implementAgeSpine"</span>
-
-  <span class="co"># make sure the object returned is of class "sampleSettings"</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/class.html">class</a></span><span class="op">(</span><span class="va">featureEngineeringSettings</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">"featureEngineeringSettings"</span>
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">featureEngineeringSettings</span><span class="op">)</span>
-  
-<span class="op">}</span></code></pre></div>
-<p>We now need to create the ‘implement’ function <code>implementAgeSpine()</code></p>
+<code class="sourceCode R"><span><span class="va">createAgeSpline</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span></span>
+<span>                     <span class="va">knots</span> <span class="op">=</span> <span class="fl">5</span></span>
+<span>                     <span class="op">)</span><span class="op">{</span></span>
+<span>  </span>
+<span>  <span class="co"># create list of inputs to implement function</span></span>
+<span>  <span class="va">featureEngineeringSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span></span>
+<span>    knots <span class="op">=</span> <span class="va">knots</span></span>
+<span>    <span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="co"># specify the function that will implement the sampling</span></span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">featureEngineeringSettings</span>, <span class="st">"fun"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">"implementAgeSplines"</span></span>
+<span></span>
+<span>  <span class="co"># make sure the object returned is of class "sampleSettings"</span></span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/class.html" class="external-link">class</a></span><span class="op">(</span><span class="va">featureEngineeringSettings</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">"featureEngineeringSettings"</span></span>
+<span>  <span class="kw"><a href="https://rdrr.io/r/base/function.html" class="external-link">return</a></span><span class="op">(</span><span class="va">featureEngineeringSettings</span><span class="op">)</span></span>
+<span>  </span>
+<span><span class="op">}</span></span></code></pre></div>
+<p>We now need to create the ‘implement’ function
+<code>implementAgeSplines()</code></p>
 </div>
-<div id="implement-function" class="section level2">
-<h2 class="hasAnchor">
-<a href="#implement-function" class="anchor"></a>Implement function</h2>
-<p>All ‘implement’ functions must take as input the trainData and the featureEngineeringSettings (this is the output of the ‘create’ function). They must return a trainData object containing the new covariateData, labels and folds.</p>
-<p>In our example, the <code>createAgeSpine()</code> will return a list with ‘knots’. The featureEngineeringSettings therefore contains this.</p>
+<div class="section level3">
+<h3 id="implement-function">Implement function<a class="anchor" aria-label="anchor" href="#implement-function"></a>
+</h3>
+<p>All ‘implement’ functions must take as input the
+<code>trainData</code> and the <code>featureEngineeringSettings</code>
+(this is the output of the ‘create’ function). They must return a
+<code>trainData</code> object containing the new
+<code>covariateData</code>, <code>labels</code> and
+<code>folds</code>.</p>
+<p>In our example, the <code>createAgeSpline()</code> will return a list
+with ‘knots’. The <code>featureEngineeringSettings</code> therefore
+contains this.</p>
 <div class="sourceCode" id="cb2"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">implementAgeSpine</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">trainData</span>, <span class="va">featureEngineeringSettings</span><span class="op">)</span><span class="op">{</span>
-
-  <span class="co"># currently not used</span>
-  <span class="va">knots</span> <span class="op">&lt;-</span> <span class="va">featureEngineeringSettings</span><span class="op">$</span><span class="va">knots</span>
-  
-  
-  <span class="co"># age in in trainData$labels as ageYear</span>
-  <span class="va">ageData</span> <span class="op">&lt;-</span> <span class="va">trainData</span><span class="op">$</span><span class="va">labels</span>
-  
-  <span class="co"># now implement the code to do your desired feature engineering</span>
-  
-  <span class="va">data</span> <span class="op">&lt;-</span> <span class="fu">Matrix</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/Matrix/man/sparseMatrix.html">sparseMatrix</a></span><span class="op">(</span>
-    i <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="va">ageData</span><span class="op">$</span><span class="va">rowId</span><span class="op">)</span>,
-    j <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/rep.html">rep</a></span><span class="op">(</span><span class="fl">1</span>, <span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="va">ageData</span><span class="op">$</span><span class="va">rowId</span><span class="op">)</span><span class="op">)</span>,
-    x <span class="op">=</span> <span class="va">ageData</span><span class="op">$</span><span class="va">ageYear</span>,
-    dims<span class="op">=</span><span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="va">ageData</span><span class="op">$</span><span class="va">rowId</span><span class="op">)</span>,<span class="fl">1</span><span class="op">)</span>
-  <span class="op">)</span>
-  
-  <span class="va">data</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/matrix.html">as.matrix</a></span><span class="op">(</span><span class="va">data</span><span class="op">)</span>
-  <span class="va">x</span> <span class="op">&lt;-</span> <span class="va">data</span><span class="op">[</span>,<span class="fl">1</span><span class="op">]</span>
-  <span class="va">y</span> <span class="op">&lt;-</span> <span class="va">ageData</span><span class="op">$</span><span class="va">outcomeCount</span>
-  
-<span class="va">mRCS</span> <span class="op">&lt;-</span> <span class="fu">rms</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/rms/man/ols.html">ols</a></span><span class="op">(</span>
-  <span class="va">y</span><span class="op">~</span><span class="fu">rms</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/rms/man/rms.trans.html">rcs</a></span><span class="op">(</span><span class="va">x</span>, 
-             <span class="fu">stats</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/r/stats/quantile.html">quantile</a></span><span class="op">(</span>
-               <span class="va">x</span>, 
-               <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="fl">0</span>, <span class="fl">.05</span>, <span class="fl">.275</span>, <span class="fl">.5</span>, <span class="fl">.775</span>, <span class="fl">.95</span>, <span class="fl">1</span><span class="op">)</span>,
-               include.lowest <span class="op">=</span> <span class="cn">TRUE</span>
-               <span class="op">)</span>
-             <span class="op">)</span>
-  <span class="op">)</span>
-
-<span class="va">newData</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html">data.frame</a></span><span class="op">(</span>
-  rowId <span class="op">=</span> <span class="va">ageData</span><span class="op">$</span><span class="va">rowId</span>,
-  covariateId <span class="op">=</span> <span class="fl">2002</span>,
-  covariateValue <span class="op">=</span> <span class="va">mRCS</span><span class="op">$</span><span class="va">fitted.values</span>
-  <span class="op">)</span>
-
-<span class="co"># add new data</span>
-<span class="fu">Andromeda</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/Andromeda/man/appendToTable.html">appendToTable</a></span><span class="op">(</span>tbl <span class="op">=</span> <span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span><span class="op">$</span><span class="va">covariates</span>, 
-                         data <span class="op">=</span> <span class="va">newData</span><span class="op">)</span>
-  
-  <span class="co"># return the updated trainData</span>
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">)</span>
-<span class="op">}</span></code></pre></div>
+<code class="sourceCode R"><span><span class="va">implementAgeSplines</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">trainData</span>, <span class="va">featureEngineeringSettings</span>, <span class="va">model</span><span class="op">=</span><span class="cn">NULL</span><span class="op">)</span> <span class="op">{</span></span>
+<span>  <span class="co"># if there is a model, it means this function is called through applyFeatureengineering, meaning it   # should apply the model fitten on training data to the test data</span></span>
+<span>  <span class="kw">if</span> <span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/NULL.html" class="external-link">is.null</a></span><span class="op">(</span><span class="va">model</span><span class="op">)</span><span class="op">)</span> <span class="op">{</span></span>
+<span>    <span class="va">knots</span> <span class="op">&lt;-</span> <span class="va">featureEngineeringSettings</span><span class="op">$</span><span class="va">knots</span></span>
+<span>    <span class="va">ageData</span> <span class="op">&lt;-</span> <span class="va">trainData</span><span class="op">$</span><span class="va">labels</span></span>
+<span>    <span class="va">y</span> <span class="op">&lt;-</span> <span class="va">ageData</span><span class="op">$</span><span class="va">outcomeCount</span></span>
+<span>    <span class="va">X</span> <span class="op">&lt;-</span> <span class="va">ageData</span><span class="op">$</span><span class="va">ageYear</span></span>
+<span>    <span class="va">model</span> <span class="op">&lt;-</span> <span class="fu">mgcv</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/mgcv/man/gam.html" class="external-link">gam</a></span><span class="op">(</span></span>
+<span>      <span class="va">y</span> <span class="op">~</span> <span class="fu">s</span><span class="op">(</span><span class="va">X</span>, bs<span class="op">=</span><span class="st">'cr'</span>, k<span class="op">=</span><span class="va">knots</span>, m<span class="op">=</span><span class="fl">2</span><span class="op">)</span></span>
+<span>    <span class="op">)</span></span>
+<span>    <span class="va">newData</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span></span>
+<span>      rowId <span class="op">=</span> <span class="va">ageData</span><span class="op">$</span><span class="va">rowId</span>,</span>
+<span>      covariateId <span class="op">=</span> <span class="fl">2002</span>,</span>
+<span>      covariateValue <span class="op">=</span> <span class="va">model</span><span class="op">$</span><span class="va">fitted.values</span></span>
+<span>    <span class="op">)</span></span>
+<span>  <span class="op">}</span></span>
+<span>  <span class="kw">else</span> <span class="op">{</span></span>
+<span>    <span class="va">ageData</span> <span class="op">&lt;-</span> <span class="va">trainData</span><span class="op">$</span><span class="va">labels</span></span>
+<span>    <span class="va">X</span> <span class="op">&lt;-</span> <span class="va">trainData</span><span class="op">$</span><span class="va">labels</span><span class="op">$</span><span class="va">ageYear</span></span>
+<span>    <span class="va">y</span> <span class="op">&lt;-</span> <span class="va">ageData</span><span class="op">$</span><span class="va">outcomeCount</span></span>
+<span>    <span class="va">newData</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>y<span class="op">=</span><span class="va">y</span>, X<span class="op">=</span><span class="va">X</span><span class="op">)</span></span>
+<span>    <span class="va">yHat</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/stats/predict.html" class="external-link">predict</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">newData</span><span class="op">)</span></span>
+<span>    <span class="va">newData</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span></span>
+<span>      rowId <span class="op">=</span> <span class="va">trainData</span><span class="op">$</span><span class="va">labels</span><span class="op">$</span><span class="va">rowId</span>,</span>
+<span>      covariateId <span class="op">=</span> <span class="fl">2002</span>,</span>
+<span>      covariateValue <span class="op">=</span> <span class="va">yHat</span></span>
+<span>    <span class="op">)</span></span>
+<span>  <span class="op">}</span></span>
+<span>  </span>
+<span>  <span class="co"># remove existing age if in covariates </span></span>
+<span>  <span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span><span class="op">$</span><span class="va">covariates</span> <span class="op">&lt;-</span> <span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span><span class="op">$</span><span class="va">covariates</span> <span class="op">|&gt;</span> </span>
+<span>    <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/filter.html" class="external-link">filter</a></span><span class="op">(</span><span class="op">!</span><span class="va">covariateId</span> <span class="op"><a href="https://rdrr.io/r/base/match.html" class="external-link">%in%</a></span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">1002</span><span class="op">)</span><span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="co"># update covRef</span></span>
+<span>  <span class="fu">Andromeda</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/Andromeda/man/appendToTable.html" class="external-link">appendToTable</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span><span class="op">$</span><span class="va">covariateRef</span>, </span>
+<span>                           <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>covariateId<span class="op">=</span><span class="fl">2002</span>,</span>
+<span>                                      covariateName<span class="op">=</span><span class="st">'Cubic restricted age splines'</span>,</span>
+<span>                                      analysisId<span class="op">=</span><span class="fl">2</span>,</span>
+<span>                                      conceptId<span class="op">=</span><span class="fl">2002</span><span class="op">)</span><span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="co"># update covariates</span></span>
+<span>  <span class="fu">Andromeda</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/Andromeda/man/appendToTable.html" class="external-link">appendToTable</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span><span class="op">$</span><span class="va">covariates</span>, <span class="va">newData</span><span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="va">featureEngineering</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span></span>
+<span>    funct <span class="op">=</span> <span class="st">'implementAgeSplines'</span>,</span>
+<span>    settings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span></span>
+<span>      featureEngineeringSettings <span class="op">=</span> <span class="va">featureEngineeringSettings</span>,</span>
+<span>      model <span class="op">=</span> <span class="va">model</span></span>
+<span>    <span class="op">)</span></span>
+<span>  <span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span>, <span class="st">'metaData'</span><span class="op">)</span><span class="op">$</span><span class="va">featureEngineering</span> <span class="op">=</span> <span class="fu"><a href="../reference/listAppend.html">listAppend</a></span><span class="op">(</span></span>
+<span>    <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span>, <span class="st">'metaData'</span><span class="op">)</span><span class="op">$</span><span class="va">featureEngineering</span>,</span>
+<span>    <span class="va">featureEngineering</span></span>
+<span>  <span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="kw"><a href="https://rdrr.io/r/base/function.html" class="external-link">return</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">)</span></span>
+<span><span class="op">}</span></span></code></pre></div>
 </div>
 </div>
-<div id="acknowledgments" class="section level1">
-<h1 class="hasAnchor">
-<a href="#acknowledgments" class="anchor"></a>Acknowledgments</h1>
-<p>Considerable work has been dedicated to provide the <code>PatientLevelPrediction</code> package.</p>
+<div class="section level2">
+<h2 id="acknowledgments">Acknowledgments<a class="anchor" aria-label="anchor" href="#acknowledgments"></a>
+</h2>
+<p>Considerable work has been dedicated to provide the
+<code>PatientLevelPrediction</code> package.</p>
 <div class="sourceCode" id="cb3"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/utils/citation.html">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></code></pre></div>
-<pre><code>## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design
-## and implementation of a standardized framework to generate and evaluate
-## patient-level prediction models using observational healthcare data."
-## _Journal of the American Medical Informatics Association_, *25*(8),
-## 969-975. &lt;URL: https://doi.org/10.1093/jamia/ocy032&gt;.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }</code></pre>
-<p><strong>Please reference this paper if you use the PLP Package in your work:</strong></p>
-<p><a href="http://dx.doi.org/10.1093/jamia/ocy032">Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.</a></p>
-<p>This work is supported in part through the National Science Foundation grant IIS 1251151.</p>
+<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/utils/citation.html" class="external-link">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></span></code></pre></div>
+<pre><code><span><span class="co">## </span></span>
+<span><span class="co">## To cite PatientLevelPrediction in publications use:</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">##   Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design</span></span>
+<span><span class="co">##   and implementation of a standardized framework to generate and</span></span>
+<span><span class="co">##   evaluate patient-level prediction models using observational</span></span>
+<span><span class="co">##   healthcare data." _Journal of the American Medical Informatics</span></span>
+<span><span class="co">##   Association_, *25*(8), 969-975.</span></span>
+<span><span class="co">##   &lt;https://doi.org/10.1093/jamia/ocy032&gt;.</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">## A BibTeX entry for LaTeX users is</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">##   @Article{,</span></span>
+<span><span class="co">##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},</span></span>
+<span><span class="co">##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},</span></span>
+<span><span class="co">##     journal = {Journal of the American Medical Informatics Association},</span></span>
+<span><span class="co">##     volume = {25},</span></span>
+<span><span class="co">##     number = {8},</span></span>
+<span><span class="co">##     pages = {969-975},</span></span>
+<span><span class="co">##     year = {2018},</span></span>
+<span><span class="co">##     url = {https://doi.org/10.1093/jamia/ocy032},</span></span>
+<span><span class="co">##   }</span></span></code></pre>
+<p><strong>Please reference this paper if you use the PLP Package in
+your work:</strong></p>
+<p><a href="http://dx.doi.org/10.1093/jamia/ocy032" class="external-link">Reps JM, Schuemie
+MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a
+standardized framework to generate and evaluate patient-level prediction
+models using observational healthcare data. J Am Med Inform Assoc.
+2018;25(8):969-975.</a></p>
+<p>This work is supported in part through the National Science
+Foundation grant IIS 1251151.</p>
 </div>
   </div>
 
@@ -292,11 +375,13 @@ <h1 class="hasAnchor">
 
 
       <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+  <p></p>
+<p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p>
+<p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
       </footer>
@@ -305,5 +390,7 @@ <h1 class="hasAnchor">
   
 
 
+  
+
   </body>
 </html>
diff --git a/docs/articles/AddingCustomFeatureEngineering_files/header-attrs-2.11/header-attrs.js b/docs/articles/AddingCustomFeatureEngineering_files/header-attrs-2.11/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/AddingCustomFeatureEngineering_files/header-attrs-2.11/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/AddingCustomModels.html b/docs/articles/AddingCustomModels.html
index f588c269e..3cafc9a9e 100644
--- a/docs/articles/AddingCustomModels.html
+++ b/docs/articles/AddingCustomModels.html
@@ -19,6 +19,8 @@
 <![endif]-->
 </head>
 <body data-spy="scroll" data-target="#toc">
+    
+
     <div class="container template-article">
       <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
@@ -31,7 +33,7 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
@@ -53,7 +55,7 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
@@ -72,10 +74,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -84,13 +86,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -101,21 +106,30 @@
   </ul>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
       </ul>
 <ul class="nav navbar-nav navbar-right">
 <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
@@ -130,15 +144,17 @@
 
       
 
-      </header><script src="AddingCustomModels_files/header-attrs-2.11/header-attrs.js"></script><div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header toc-ignore">
-      <h1 data-toc-skip>Adding Custom Patient-Level Prediction Algorithms</h1>
-                        <h4 class="author">Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek</h4>
+      <h1 data-toc-skip>Adding Custom Patient-Level Prediction
+Algorithms</h1>
+                        <h4 data-toc-skip class="author">Jenna Reps,
+Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek</h4>
             
-            <h4 class="date">2022-03-09</h4>
+            <h4 data-toc-skip class="date">2024-09-09</h4>
       
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/AddingCustomModels.Rmd"><code>vignettes/AddingCustomModels.Rmd</code></a></small>
+      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/vignettes/AddingCustomModels.Rmd" class="external-link"><code>vignettes/AddingCustomModels.Rmd</code></a></small>
       <div class="hidden name"><code>AddingCustomModels.Rmd</code></div>
 
     </div>
@@ -146,472 +162,589 @@ <h4 class="date">2022-03-09</h4>
     
     
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Adding Custom Patient-Level Prediction Algorithms}
 -->
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
-<p>This vignette describes how you can add your own custom algorithms in the Observational Health Data Sciencs and Informatics (OHDSI) <a href="http://github.com/OHDSI/PatientLevelPrediction"><code>PatientLevelPrediction</code></a> package. This allows you to fully leverage the OHDSI PatientLevelPrediction framework for model development and validation. This vignette assumes you have read and are comfortable with building single patient level prediction models as described in the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf"><code>BuildingPredictiveModels</code> vignette</a>.</p>
-<p><strong>We invite you to share your new algorithms with the OHDSI community through our <a href="http://github.com/OHDSI/PatientLevelPrediction">GitHub repository</a>.</strong></p>
+<div class="section level2">
+<h2 id="introduction">Introduction<a class="anchor" aria-label="anchor" href="#introduction"></a>
+</h2>
+<p>This vignette describes how you can add your own custom algorithms in
+the Observational Health Data Sciencs and Informatics (OHDSI) <a href="http://github.com/OHDSI/PatientLevelPrediction" class="external-link"><code>PatientLevelPrediction</code></a>
+package. This allows you to fully leverage the OHDSI
+PatientLevelPrediction framework for model development and validation.
+This vignette assumes you have read and are comfortable with building
+single patient level prediction models as described in the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf" class="external-link"><code>BuildingPredictiveModels</code>
+vignette</a>.</p>
+<p><strong>We invite you to share your new algorithms with the OHDSI
+community through our <a href="http://github.com/OHDSI/PatientLevelPrediction" class="external-link">GitHub
+repository</a>.</strong></p>
 </div>
-<div id="algorithm-code-structure" class="section level1">
-<h1 class="hasAnchor">
-<a href="#algorithm-code-structure" class="anchor"></a>Algorithm Code Structure</h1>
-<p>Each algorithm in the package should be implemented in its own &lt;Name&gt;.R file, e.g. KNN.R, containing a set&lt;Name&gt; function, a fit&lt;Name&gt; function and a predict&lt;Name&gt; function. Occasionally the fit and prediction functions may be reused (if using an R classifier see RClassifier.R or if using a scikit-learn classifier see SklearnClassifier.R). We will now describe each of these functions in more detail below.</p>
-<div id="set" class="section level2">
-<h2 class="hasAnchor">
-<a href="#set" class="anchor"></a>Set</h2>
-<p>The set&lt;Name&gt; is a function that takes as input the different hyper-parameter values to do a grid search when training. The output of the functions needs to be a list as class <code>modelSettings</code> containing:</p>
+<div class="section level2">
+<h2 id="algorithm-code-structure">Algorithm Code Structure<a class="anchor" aria-label="anchor" href="#algorithm-code-structure"></a>
+</h2>
+<p>Each algorithm in the package should be implemented in its own
+&lt;Name&gt;.R file, e.g. KNN.R, containing a set&lt;Name&gt; function,
+a fit&lt;Name&gt; function and a predict&lt;Name&gt; function.
+Occasionally the fit and prediction functions may be reused (if using an
+R classifier see RClassifier.R or if using a scikit-learn classifier see
+SklearnClassifier.R). We will now describe each of these functions in
+more detail below.</p>
+<div class="section level3">
+<h3 id="set">Set<a class="anchor" aria-label="anchor" href="#set"></a>
+</h3>
+<p>The set&lt;Name&gt; is a function that takes as input the different
+hyper-parameter values to do a grid search when training. The output of
+the functions needs to be a list as class <code>modelSettings</code>
+containing:</p>
 <ul>
-<li>param - all the combinations of the hyper-parameter values input</li>
-<li>fitFunction - a string specifying what function to call to fit the model</li>
+<li>param - all the combinations of the hyper-parameter values
+input</li>
+<li>fitFunction - a string specifying what function to call to fit the
+model</li>
 </ul>
-<p>The param object can have a setttings attribute containing any extra settings. For example to specify the model name and the seed used for reproducibility:</p>
+<p>The param object can have a setttings attribute containing any extra
+settings. For example to specify the model name and the seed used for
+reproducibility:</p>
 <div class="sourceCode" id="cb1"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">param</span>, <span class="st">'settings'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>
-  seed <span class="op">=</span> <span class="fl">12</span>,
-  modelName <span class="op">=</span> <span class="st">"Special classifier"</span>
-  <span class="op">)</span></code></pre></div>
-<p>For example, if you were adding a model called madeUp that has two hyper-parameters then the set function should be:</p>
+<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">param</span>, <span class="st">'settings'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span></span>
+<span>  seed <span class="op">=</span> <span class="fl">12</span>,</span>
+<span>  modelName <span class="op">=</span> <span class="st">"Special classifier"</span></span>
+<span>  <span class="op">)</span></span></code></pre></div>
+<p>For example, if you were adding a model called madeUp that has two
+hyper-parameters then the set function should be:</p>
 <div class="sourceCode" id="cb2"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">setMadeUp</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">a</span><span class="op">=</span><span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="fl">1</span>,<span class="fl">4</span>,<span class="fl">10</span><span class="op">)</span>, <span class="va">b</span><span class="op">=</span><span class="fl">2</span>, <span class="va">seed</span><span class="op">=</span><span class="cn">NULL</span><span class="op">)</span><span class="op">{</span>
-  <span class="co"># add input checks here...</span>
-  
-  <span class="va">param</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/split.html">split</a></span><span class="op">(</span>
-    <span class="fu"><a href="https://rdrr.io/r/base/expand.grid.html">expand.grid</a></span><span class="op">(</span>
-      a<span class="op">=</span><span class="va">a</span>, 
-      b<span class="op">=</span><span class="va">b</span>
-    <span class="op">)</span>,
-    <span class="fl">1</span><span class="op">:</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="va">a</span><span class="op">)</span><span class="op">*</span><span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="va">b</span><span class="op">)</span><span class="op">)</span>
-    <span class="op">)</span>
-  
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">param</span>, <span class="st">'settings'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>
-    modelName <span class="op">=</span> <span class="st">"Made Up"</span>,
-    requiresDenseMatrix <span class="op">=</span> <span class="cn">TRUE</span>,
-    seed <span class="op">=</span> <span class="va">seed</span>
-    <span class="op">)</span>
-  
-  <span class="co"># now create list of all combinations:</span>
-  <span class="va">result</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>
-    fitFunction <span class="op">=</span> <span class="st">'fitMadeUp'</span>, <span class="co"># this will be called to train the made up model</span>
-    param <span class="op">=</span> <span class="va">param</span>
-  <span class="op">)</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/class.html">class</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'modelSettings'</span> 
-  
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span>
-<span class="op">}</span></code></pre></div>
+<code class="sourceCode R"><span><span class="va">setMadeUp</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">a</span><span class="op">=</span><span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">1</span>,<span class="fl">4</span>,<span class="fl">10</span><span class="op">)</span>, <span class="va">b</span><span class="op">=</span><span class="fl">2</span>, <span class="va">seed</span><span class="op">=</span><span class="cn">NULL</span><span class="op">)</span><span class="op">{</span></span>
+<span>  <span class="co"># add input checks here...</span></span>
+<span>  </span>
+<span>  <span class="va">param</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/split.html" class="external-link">split</a></span><span class="op">(</span></span>
+<span>    <span class="fu"><a href="https://rdrr.io/r/base/expand.grid.html" class="external-link">expand.grid</a></span><span class="op">(</span></span>
+<span>      a<span class="op">=</span><span class="va">a</span>, </span>
+<span>      b<span class="op">=</span><span class="va">b</span></span>
+<span>    <span class="op">)</span>,</span>
+<span>    <span class="fl">1</span><span class="op">:</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/length.html" class="external-link">length</a></span><span class="op">(</span><span class="va">a</span><span class="op">)</span><span class="op">*</span><span class="fu"><a href="https://rdrr.io/r/base/length.html" class="external-link">length</a></span><span class="op">(</span><span class="va">b</span><span class="op">)</span><span class="op">)</span></span>
+<span>    <span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">param</span>, <span class="st">'settings'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span></span>
+<span>    modelName <span class="op">=</span> <span class="st">"Made Up"</span>,</span>
+<span>    requiresDenseMatrix <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>    seed <span class="op">=</span> <span class="va">seed</span></span>
+<span>    <span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="co"># now create list of all combinations:</span></span>
+<span>  <span class="va">result</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span></span>
+<span>    fitFunction <span class="op">=</span> <span class="st">'fitMadeUp'</span>, <span class="co"># this will be called to train the made up model</span></span>
+<span>    param <span class="op">=</span> <span class="va">param</span></span>
+<span>  <span class="op">)</span></span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/class.html" class="external-link">class</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'modelSettings'</span> </span>
+<span>  </span>
+<span>  <span class="kw"><a href="https://rdrr.io/r/base/function.html" class="external-link">return</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span></span>
+<span><span class="op">}</span></span></code></pre></div>
 </div>
-<div id="fit" class="section level2">
-<h2 class="hasAnchor">
-<a href="#fit" class="anchor"></a>Fit</h2>
-<p>This function should train your custom model for each parameter entry, pick the best parameters and train a final model for that setting.</p>
+<div class="section level3">
+<h3 id="fit">Fit<a class="anchor" aria-label="anchor" href="#fit"></a>
+</h3>
+<p>This function should train your custom model for each parameter
+entry, pick the best parameters and train a final model for that
+setting.</p>
 <p>The fit&lt;Model&gt; should have as inputs:</p>
 <ul>
-<li>trainData - a list containing the covariateData, labels and folds for the training population</li>
+<li>trainData - a list containing the covariateData, labels and folds
+for the training population</li>
 <li>param - the hyper-parameters as a list of all combinations</li>
 <li>search - the type of hyper-parameter search</li>
 <li>analysisId - an identifier for the analysis</li>
 </ul>
-<p>The fit function should return a list of class <code>plpModel</code> with the following objects:</p>
+<p>The fit function should return a list of class <code>plpModel</code>
+with the following objects:</p>
+<ul>
+<li>model - a trained model (or location of the model if it is not an R
+object)</li>
+<li>prediction - a data.frame object with the trainData$labels plus an
+extra column with the name ‘value’ corresponding to the predicted risk
+of having the outcome during the time-at-risk.</li>
+<li>preprocessing - the settings required to preprocess the data when
+applying the model
 <ul>
-<li>model - a trained model (or location of the model if it is not an R object)</li>
-<li>prediction - a data.frame object with the trainData$labels plus an extra column with the name ‘value’ corresponding to the predicted risk of having the outcome during the time-at-risk.</li>
-<li>settings - a list containing:
+<li>featureEngineering - the feature engineering settings e.g.,
+attr(trainData<span class="math inline">\(covariateData,
+"metaData")\)</span>featureEngineering,</li>
+<li>tidyCovariates - the preprocessing settings e.g.,
+attr(trainData<span class="math inline">\(covariateData,
+"metaData")\)</span>tidyCovariateDataSettings,</li>
+<li>requireDenseMatrix - does the model require a dense matrix? e.g.,
+attr(param, ‘settings’)$requiresDenseMatrix,</li>
+</ul>
+</li>
+<li>modelDesign - a list containing:
 <ul>
-<li>plpDataSettings - the plpData settings e.g., attr(trainData, “metaData”)$plpDataSettings</li>
-<li>covariateSettings - the covariate settings e.g., attr(trainData, “metaData”)$covariateSettings</li>
-<li>populationSettings - the population settings e.g., attr(trainData, “metaData”)$populationSettings,</li>
-<li>featureEngineering - the feature engineering settings e.g., attr(trainData<span class="math inline">\(covariateData, "metaData")\)</span>featureEngineering,</li>
-<li>tidyCovariates - the preprocessing settings e.g., attr(trainData<span class="math inline">\(covariateData, "metaData")\)</span>tidyCovariateDataSettings,</li>
-<li>requireDenseMatrix - does the model require a dense matrix? e.g., attr(param, ‘settings’)$requiresDenseMatrix,</li>
-<li>modelSettings = a list containing: model (model name), param (the hyper-parameter search list), finalModelParameters (the final model hyper-parameters), extraSettings (any extra settings)</li>
-<li>splitSettings - the split settings e.g., attr(trainData, “metaData”)$splitSettings,</li>
-<li>sampleSettings - the sample settings e.g., attr(trainData, “metaData”)$sampleSettings</li>
+<li>targetId - the id of the target cohort</li>
+<li>outcomeId - the id of the outcome cohort</li>
+<li>plpDataSettings - the plpData settings e.g., attr(trainData,
+“metaData”)$plpDataSettings</li>
+<li>covariateSettings - the covariate settings e.g., attr(trainData,
+“metaData”)$covariateSettings</li>
+<li>populationSettings - the population settings e.g., attr(trainData,
+“metaData”)$populationSettings,</li>
+<li>featureEngineeringSettings - the feature engineering settings e.g.,
+attr(trainData<span class="math inline">\(covariateData,
+"metaData")\)</span>featureEngineeringSettings,</li>
+<li>preprocessSettings - the preprocessing settings e.g.,
+attr(trainData<span class="math inline">\(covariateData,
+"metaData")\)</span>preprocessSettings,</li>
+<li>modelSettings = a list containing: model (model name), param (the
+hyper-parameter search list), finalModelParameters (the final model
+hyper-parameters), extraSettings (any extra settings)</li>
+<li>splitSettings - the split settings e.g., attr(trainData,
+“metaData”)$splitSettings,</li>
+<li>sampleSettings - the sample settings e.g., attr(trainData,
+“metaData”)$sampleSettings</li>
 </ul>
 </li>
 <li>trainDetails - a list containing:
 <ul>
 <li>analysisId - the identifier for the analysis</li>
-<li>cdmDatabaseSchema - the database used to develop the model</li>
-<li>outcomeId - the outcome id</li>
-<li>cohortId - the target population id</li>
+<li>developmentDatabase - the database used to develop the model</li>
 <li>attrition - the attrition</li>
 <li>trainingTime - how long it took to train the model</li>
 <li>trainingDate - date of model training</li>
-<li>hyperParamSearch - the hyper-parameter search used to train the model</li>
+<li>hyperParamSearch - the hyper-parameter search used to train the
+model</li>
+<li>any other objects specific to training</li>
 </ul>
 </li>
-<li>covariateImportance - a data.frame containing the columns ‘covariateId’, ‘covariateValue’ (the variable importance) and ‘columnId’ (the column number that the variable need to be mapped to when implementing the model)</li>
+<li>covariateImportance - a data.frame containing the columns
+‘covariateId’, ‘covariateValue’ (the variable importance) and ‘columnId’
+(the column number that the variable need to be mapped to when
+implementing the model)</li>
 </ul>
 <p>In additon the plpModel requires two attributes:</p>
 <ul>
-<li>predictionFunction - the name of the function used to make predictions</li>
+<li>predictionFunction - the name of the function used to make
+predictions</li>
 <li>modelType - whether the model is ‘binary’ or ‘survival’</li>
 </ul>
-<p>For example <code>attr(result, 'predictionFunction') &lt;- 'madeupPrediction'</code> means when the model is applied to new data, the ‘madeupPrediction’ function is called to make predictions. If this doesnt exist, then the model will fail. The other attribute is the modelType <code>attr(result, 'modelType') &lt;- 'binary'</code> this is needed when evaluating the model to ensure the correct evaluation is applied. Currently the evaluation supports ‘binary’ and ‘survival’ modelType.</p>
-<p>Note: If a new modelType is desired, then the evalaution code within PatientLevelPrediction must be updated to specify how the new type is evaluated. This requires making edits to PatientLevelPrediction and then making a pull request to the PatientLevelPrediction github. The evaluation cannot have one off customization because the evaluation must be standardized to enable comparison across similar models.</p>
+<p>For example
+<code>attr(result, 'predictionFunction') &lt;- 'madeupPrediction'</code>
+means when the model is applied to new data, the ‘madeupPrediction’
+function is called to make predictions. If this doesnt exist, then the
+model will fail. The other attribute is the modelType
+<code>attr(result, 'modelType') &lt;- 'binary'</code> this is needed
+when evaluating the model to ensure the correct evaluation is applied.
+Currently the evaluation supports ‘binary’ and ‘survival’ modelType.</p>
+<p>Note: If a new modelType is desired, then the evalaution code within
+PatientLevelPrediction must be updated to specify how the new type is
+evaluated. This requires making edits to PatientLevelPrediction and then
+making a pull request to the PatientLevelPrediction github. The
+evaluation cannot have one off customization because the evaluation must
+be standardized to enable comparison across similar models.</p>
 <p>A full example of a custom ‘binary’ classifier fit function is:</p>
 <div class="sourceCode" id="cb3"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">fitMadeUp</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">trainData</span>, <span class="va">param</span>, <span class="va">search</span>, <span class="va">analysisId</span><span class="op">)</span><span class="op">{</span>
-  
-  <span class="co"># **************** code to train the model here</span>
-  <span class="co"># trainedModel &lt;- this code should apply each hyper-parameter combination   </span>
-  <span class="co"># (param[[i]]) using the specified search (e.g., cross validation)</span>
-  <span class="co">#                 then pick out the best hyper-parameter setting</span>
-  <span class="co">#                 and finally fit a model on the whole train data using the </span>
-  <span class="co">#                 optimal hyper-parameter settings</span>
-  <span class="co"># ****************</span>
-  
-  <span class="co"># **************** code to apply the model to trainData</span>
-  <span class="co"># prediction &lt;- code to apply trainedModel to trainData</span>
-  <span class="co"># ****************</span>
-  
-  <span class="co"># **************** code to get variable importance (if possible)</span>
-  <span class="co"># varImp &lt;- code to get importance of each variable in trainedModel</span>
-  <span class="co"># ****************</span>
-  
-  
-  <span class="co"># construct the standard output for a model:</span>
-  <span class="va">result</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>model <span class="op">=</span> <span class="va">trainedModel</span>,
-                 prediction <span class="op">=</span> <span class="va">prediction</span>, <span class="co"># the train and maybe the cross validation predictions for the trainData</span>
-    settings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>
-      plpDataSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">plpDataSettings</span>,
-      covariateSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">covariateSettings</span>,
-      populationSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">populationSettings</span>,
-      featureEngineering <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">featureEngineering</span>,
-      tidyCovariates <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">tidyCovariateDataSettings</span>, 
-      requireDenseMatrix <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">param</span>, <span class="st">'settings'</span><span class="op">)</span><span class="op">$</span><span class="va">requiresDenseMatrix</span>,
-      modelSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>
-        model <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">param</span>, <span class="st">'settings'</span><span class="op">)</span><span class="op">$</span><span class="va">modelName</span>, <span class="co"># the model name</span>
-        param <span class="op">=</span> <span class="va">param</span>,
-        finalModelParameters <span class="op">=</span> <span class="va">param</span><span class="op">[[</span><span class="va">bestInd</span><span class="op">]</span><span class="op">]</span>, <span class="co"># best hyper-parameters</span>
-        extraSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">param</span>, <span class="st">'settings'</span><span class="op">)</span>
-      <span class="op">)</span>,
-      splitSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">splitSettings</span>,
-      sampleSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">sampleSettings</span>
-    <span class="op">)</span>,
-    
-    trainDetails <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>
-      analysisId <span class="op">=</span> <span class="va">analysisId</span>,
-      cdmDatabaseSchema <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">cdmDatabaseSchema</span>,
-      outcomeId <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">outcomeId</span>,
-      cohortId <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">cohortId</span>,
-      attrition <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">attrition</span>, 
-      trainingTime <span class="op">=</span> <span class="va">timeToTrain</span>, <span class="co"># how long it took to train the model</span>
-      trainingDate <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/Sys.time.html">Sys.Date</a></span><span class="op">(</span><span class="op">)</span>,
-      hyperParamSearch <span class="op">=</span> <span class="va">hyperSummary</span> <span class="co"># the hyper-parameters and performance data.frame</span>
-    <span class="op">)</span>,
-    covariateImportance <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/merge.html">merge</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span><span class="op">$</span><span class="va">covariateRef</span>, <span class="va">varImp</span>, by<span class="op">=</span><span class="st">'covariateId'</span><span class="op">)</span> <span class="co"># add variable importance to covariateRef if possible</span>
-  <span class="op">)</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/class.html">class</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'plpModel'</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">result</span>, <span class="st">'predictionFunction'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'madeupPrediction'</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">result</span>, <span class="st">'modelType'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'binary'</span>
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span>
-    
-<span class="op">}</span></code></pre></div>
-<p>You could make the fitMadeUp function cleaner by adding helper function in the MadeUp.R file that are called by the fit function (for example a function to run cross validation). It is important to ensure there is a valid prediction function (the one specified by <code>attr(result, 'predictionFunction') &lt;- 'madeupPrediction'</code> is <code>madeupPrediction()</code>) as specified below.</p>
+<code class="sourceCode R"><span><span class="va">fitMadeUp</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">trainData</span>, <span class="va">modelSettings</span>, <span class="va">search</span>, <span class="va">analysisId</span><span class="op">)</span><span class="op">{</span></span>
+<span>  </span>
+<span>  <span class="va">param</span> <span class="op">&lt;-</span> <span class="va">modelSettings</span><span class="op">$</span><span class="va">param</span></span>
+<span>  </span>
+<span>  <span class="co"># **************** code to train the model here</span></span>
+<span>  <span class="co"># trainedModel &lt;- this code should apply each hyper-parameter combination   </span></span>
+<span>  <span class="co"># (param[[i]]) using the specified search (e.g., cross validation)</span></span>
+<span>  <span class="co">#                 then pick out the best hyper-parameter setting</span></span>
+<span>  <span class="co">#                 and finally fit a model on the whole train data using the </span></span>
+<span>  <span class="co">#                 optimal hyper-parameter settings</span></span>
+<span>  <span class="co"># ****************</span></span>
+<span>  </span>
+<span>  <span class="co"># **************** code to apply the model to trainData</span></span>
+<span>  <span class="co"># prediction &lt;- code to apply trainedModel to trainData</span></span>
+<span>  <span class="co"># ****************</span></span>
+<span>  </span>
+<span>  <span class="co"># **************** code to get variable importance (if possible)</span></span>
+<span>  <span class="co"># varImp &lt;- code to get importance of each variable in trainedModel</span></span>
+<span>  <span class="co"># ****************</span></span>
+<span>  </span>
+<span>  </span>
+<span>  <span class="co"># construct the standard output for a model:</span></span>
+<span>  <span class="va">result</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>model <span class="op">=</span> <span class="va">trainedModel</span>,</span>
+<span>                 prediction <span class="op">=</span> <span class="va">prediction</span>, <span class="co"># the train and maybe the cross validation predictions for the trainData</span></span>
+<span>                 preprocessing <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span></span>
+<span>                   featureEngineering <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">featureEngineering</span>,</span>
+<span>      tidyCovariates <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">tidyCovariateDataSettings</span>, </span>
+<span>      requireDenseMatrix <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">param</span>, <span class="st">'settings'</span><span class="op">)</span><span class="op">$</span><span class="va">requiresDenseMatrix</span>,</span>
+<span>      </span>
+<span>                 <span class="op">)</span>,</span>
+<span>    modelDesign <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span></span>
+<span>      outcomeId <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">outcomeId</span>,</span>
+<span>      targetId <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">targetId</span>,</span>
+<span>      plpDataSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">plpDataSettings</span>,</span>
+<span>      covariateSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">covariateSettings</span>,</span>
+<span>      populationSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">populationSettings</span>,</span>
+<span>      featureEngineeringSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">featureEngineeringSettings</span>,</span>
+<span>      prerocessSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">prerocessSettings</span>, </span>
+<span>      modelSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span></span>
+<span>        model <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">param</span>, <span class="st">'settings'</span><span class="op">)</span><span class="op">$</span><span class="va">modelName</span>, <span class="co"># the model name</span></span>
+<span>        param <span class="op">=</span> <span class="va">param</span>,</span>
+<span>        finalModelParameters <span class="op">=</span> <span class="va">param</span><span class="op">[[</span><span class="va">bestInd</span><span class="op">]</span><span class="op">]</span>, <span class="co"># best hyper-parameters</span></span>
+<span>        extraSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">param</span>, <span class="st">'settings'</span><span class="op">)</span></span>
+<span>      <span class="op">)</span>,</span>
+<span>      splitSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">splitSettings</span>,</span>
+<span>      sampleSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">sampleSettings</span></span>
+<span>    <span class="op">)</span>,</span>
+<span>    </span>
+<span>    trainDetails <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span></span>
+<span>      analysisId <span class="op">=</span> <span class="va">analysisId</span>,</span>
+<span>      developmentDatabase <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">cdmDatabaseSchema</span>,</span>
+<span>      attrition <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">attrition</span>, </span>
+<span>      trainingTime <span class="op">=</span> <span class="va">timeToTrain</span>, <span class="co"># how long it took to train the model</span></span>
+<span>      trainingDate <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/Sys.time.html" class="external-link">Sys.Date</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>      hyperParamSearch <span class="op">=</span> <span class="va">hyperSummary</span> <span class="co"># the hyper-parameters and performance data.frame</span></span>
+<span>    <span class="op">)</span>,</span>
+<span>    covariateImportance <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/merge.html" class="external-link">merge</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span><span class="op">$</span><span class="va">covariateRef</span>, <span class="va">varImp</span>, by<span class="op">=</span><span class="st">'covariateId'</span><span class="op">)</span> <span class="co"># add variable importance to covariateRef if possible</span></span>
+<span>  <span class="op">)</span></span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/class.html" class="external-link">class</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'plpModel'</span></span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">result</span>, <span class="st">'predictionFunction'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'madeupPrediction'</span></span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">result</span>, <span class="st">'modelType'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'binary'</span></span>
+<span>  <span class="kw"><a href="https://rdrr.io/r/base/function.html" class="external-link">return</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span></span>
+<span>    </span>
+<span><span class="op">}</span></span></code></pre></div>
+<p>You could make the fitMadeUp function cleaner by adding helper
+function in the MadeUp.R file that are called by the fit function (for
+example a function to run cross validation). It is important to ensure
+there is a valid prediction function (the one specified by
+<code>attr(result, 'predictionFunction') &lt;- 'madeupPrediction'</code>
+is <code>madeupPrediction()</code>) as specified below.</p>
 </div>
-<div id="predict" class="section level2">
-<h2 class="hasAnchor">
-<a href="#predict" class="anchor"></a>Predict</h2>
-<p>The prediction function takes as input the plpModel returned by fit, new data and a corresponding cohort. It returns a data.frame with the same columns as cohort but with an additional column:</p>
+<div class="section level3">
+<h3 id="predict">Predict<a class="anchor" aria-label="anchor" href="#predict"></a>
+</h3>
+<p>The prediction function takes as input the plpModel returned by fit,
+new data and a corresponding cohort. It returns a data.frame with the
+same columns as cohort but with an additional column:</p>
 <ul>
-<li>value - the predicted risk from the plpModel for each patient in the cohort</li>
+<li>value - the predicted risk from the plpModel for each patient in the
+cohort</li>
 </ul>
 <p>For example:</p>
 <div class="sourceCode" id="cb4"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">madeupPrediction</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">plpModel</span>, <span class="va">data</span>, <span class="va">cohort</span><span class="op">)</span><span class="op">{</span> 
-
-  <span class="co"># ************* code to do prediction for each rowId in cohort</span>
-  <span class="co"># predictionValues &lt;- code to do prediction here returning the predicted risk</span>
-  <span class="co">#               (value) for each rowId in cohort </span>
-  <span class="co">#**************</span>
-  
-  <span class="va">prediction</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/merge.html">merge</a></span><span class="op">(</span><span class="va">cohort</span>, <span class="va">predictionValues</span>, by<span class="op">=</span><span class="st">'rowId'</span><span class="op">)</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">prediction</span>, <span class="st">"metaData"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>modelType <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">plpModel</span>, <span class="st">'modelType'</span><span class="op">)</span><span class="op">)</span> 
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">prediction</span><span class="op">)</span>
-  
-<span class="op">}</span></code></pre></div>
+<code class="sourceCode R"><span><span class="va">madeupPrediction</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">plpModel</span>, <span class="va">data</span>, <span class="va">cohort</span><span class="op">)</span><span class="op">{</span> </span>
+<span></span>
+<span>  <span class="co"># ************* code to do prediction for each rowId in cohort</span></span>
+<span>  <span class="co"># predictionValues &lt;- code to do prediction here returning the predicted risk</span></span>
+<span>  <span class="co">#               (value) for each rowId in cohort </span></span>
+<span>  <span class="co">#**************</span></span>
+<span>  </span>
+<span>  <span class="va">prediction</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/merge.html" class="external-link">merge</a></span><span class="op">(</span><span class="va">cohort</span>, <span class="va">predictionValues</span>, by<span class="op">=</span><span class="st">'rowId'</span><span class="op">)</span></span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">prediction</span>, <span class="st">"metaData"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>modelType <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">plpModel</span>, <span class="st">'modelType'</span><span class="op">)</span><span class="op">)</span> </span>
+<span>  <span class="kw"><a href="https://rdrr.io/r/base/function.html" class="external-link">return</a></span><span class="op">(</span><span class="va">prediction</span><span class="op">)</span></span>
+<span>  </span>
+<span><span class="op">}</span></span></code></pre></div>
 </div>
 </div>
-<div id="algorithm-example" class="section level1">
-<h1 class="hasAnchor">
-<a href="#algorithm-example" class="anchor"></a>Algorithm Example</h1>
-<p>Below a fully functional algorithm example is given, however we highly recommend you to have a look at the available algorithms in the package (see GradientBoostingMachine.R for the set function, RClassifier.R for the fit and prediction function for R classifiers).</p>
-<div id="set-1" class="section level2">
-<h2 class="hasAnchor">
-<a href="#set-1" class="anchor"></a>Set</h2>
+<div class="section level2">
+<h2 id="algorithm-example">Algorithm Example<a class="anchor" aria-label="anchor" href="#algorithm-example"></a>
+</h2>
+<p>Below a fully functional algorithm example is given, however we
+highly recommend you to have a look at the available algorithms in the
+package (see GradientBoostingMachine.R for the set function,
+RClassifier.R for the fit and prediction function for R
+classifiers).</p>
+<div class="section level3">
+<h3 id="set-1">Set<a class="anchor" aria-label="anchor" href="#set-1"></a>
+</h3>
 <div class="sourceCode" id="cb5"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">setMadeUp</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">a</span><span class="op">=</span><span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="fl">1</span>,<span class="fl">4</span>,<span class="fl">6</span><span class="op">)</span>, <span class="va">b</span><span class="op">=</span><span class="fl">2</span>, <span class="va">seed</span><span class="op">=</span><span class="cn">NULL</span><span class="op">)</span><span class="op">{</span>
-  <span class="co"># add input checks here...</span>
-  
-  <span class="kw">if</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/NULL.html">is.null</a></span><span class="op">(</span><span class="va">seed</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-    <span class="va">seed</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/sample.html">sample</a></span><span class="op">(</span><span class="fl">100000</span>,<span class="fl">1</span><span class="op">)</span>
-  <span class="op">}</span>
-  
-  <span class="va">param</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/split.html">split</a></span><span class="op">(</span>
-    <span class="fu"><a href="https://rdrr.io/r/base/expand.grid.html">expand.grid</a></span><span class="op">(</span>
-      a<span class="op">=</span><span class="va">a</span>, 
-      b<span class="op">=</span><span class="va">b</span>
-    <span class="op">)</span>,
-    <span class="fl">1</span><span class="op">:</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="va">a</span><span class="op">)</span><span class="op">*</span><span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="va">b</span><span class="op">)</span><span class="op">)</span>
-    <span class="op">)</span>
-  
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">param</span>, <span class="st">'settings'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>
-    modelName <span class="op">=</span> <span class="st">"Made Up"</span>,
-    requiresDenseMatrix <span class="op">=</span> <span class="cn">TRUE</span>,
-    seed <span class="op">=</span> <span class="va">seed</span>
-    <span class="op">)</span>
-  
-  <span class="co"># now create list of all combinations:</span>
-  <span class="va">result</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>
-    fitFunction <span class="op">=</span> <span class="st">'fitMadeUp'</span>, <span class="co"># this will be called to train the made up model</span>
-    param <span class="op">=</span> <span class="va">param</span>
-  <span class="op">)</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/class.html">class</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'modelSettings'</span> 
-  
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span>
-<span class="op">}</span></code></pre></div>
+<code class="sourceCode R"><span><span class="va">setMadeUp</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">a</span><span class="op">=</span><span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">1</span>,<span class="fl">4</span>,<span class="fl">6</span><span class="op">)</span>, <span class="va">b</span><span class="op">=</span><span class="fl">2</span>, <span class="va">seed</span><span class="op">=</span><span class="cn">NULL</span><span class="op">)</span><span class="op">{</span></span>
+<span>  <span class="co"># add input checks here...</span></span>
+<span>  </span>
+<span>  <span class="kw">if</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/NULL.html" class="external-link">is.null</a></span><span class="op">(</span><span class="va">seed</span><span class="op">)</span><span class="op">)</span><span class="op">{</span></span>
+<span>    <span class="va">seed</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/sample.html" class="external-link">sample</a></span><span class="op">(</span><span class="fl">100000</span>,<span class="fl">1</span><span class="op">)</span></span>
+<span>  <span class="op">}</span></span>
+<span>  </span>
+<span>  <span class="va">param</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/split.html" class="external-link">split</a></span><span class="op">(</span></span>
+<span>    <span class="fu"><a href="https://rdrr.io/r/base/expand.grid.html" class="external-link">expand.grid</a></span><span class="op">(</span></span>
+<span>      a<span class="op">=</span><span class="va">a</span>, </span>
+<span>      b<span class="op">=</span><span class="va">b</span></span>
+<span>    <span class="op">)</span>,</span>
+<span>    <span class="fl">1</span><span class="op">:</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/length.html" class="external-link">length</a></span><span class="op">(</span><span class="va">a</span><span class="op">)</span><span class="op">*</span><span class="fu"><a href="https://rdrr.io/r/base/length.html" class="external-link">length</a></span><span class="op">(</span><span class="va">b</span><span class="op">)</span><span class="op">)</span></span>
+<span>    <span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">param</span>, <span class="st">'settings'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span></span>
+<span>    modelName <span class="op">=</span> <span class="st">"Made Up"</span>,</span>
+<span>    requiresDenseMatrix <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>    seed <span class="op">=</span> <span class="va">seed</span></span>
+<span>    <span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="co"># now create list of all combinations:</span></span>
+<span>  <span class="va">result</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span></span>
+<span>    fitFunction <span class="op">=</span> <span class="st">'fitMadeUp'</span>, <span class="co"># this will be called to train the made up model</span></span>
+<span>    param <span class="op">=</span> <span class="va">param</span></span>
+<span>  <span class="op">)</span></span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/class.html" class="external-link">class</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'modelSettings'</span> </span>
+<span>  </span>
+<span>  <span class="kw"><a href="https://rdrr.io/r/base/function.html" class="external-link">return</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span></span>
+<span><span class="op">}</span></span></code></pre></div>
 </div>
-<div id="fit-1" class="section level2">
-<h2 class="hasAnchor">
-<a href="#fit-1" class="anchor"></a>Fit</h2>
-<div class="sourceCode" id="cb6"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">fitMadeUp</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">trainData</span>, <span class="va">param</span>, <span class="va">search</span>, <span class="va">analysisId</span><span class="op">)</span><span class="op">{</span>
-
-  <span class="co"># set the seed for reproducibility</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/Random.html">set.seed</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">param</span>, <span class="st">'settings'</span><span class="op">)</span><span class="op">$</span><span class="va">seed</span><span class="op">)</span>
-  
-  <span class="co"># add folds to labels:</span>
-  <span class="va">trainData</span><span class="op">$</span><span class="va">labels</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/merge.html">merge</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">labels</span>, <span class="va">trainData</span><span class="op">$</span><span class="va">folds</span>, by<span class="op">=</span> <span class="st">'rowId'</span><span class="op">)</span>
-  <span class="co"># convert data into sparse R Matrix:</span>
-  <span class="va">mappedData</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/toSparseM.html">toSparseM</a></span><span class="op">(</span><span class="va">trainData</span>,map<span class="op">=</span><span class="cn">NULL</span><span class="op">)</span>
-  <span class="va">matrixData</span> <span class="op">&lt;-</span> <span class="va">mappedData</span><span class="op">$</span><span class="va">dataMatrix</span>
-  <span class="va">labels</span> <span class="op">&lt;-</span> <span class="va">mappedData</span><span class="op">$</span><span class="va">labels</span>
-  <span class="va">covariateRef</span> <span class="op">&lt;-</span> <span class="va">mappedData</span><span class="op">$</span><span class="va">covariateRef</span>
-
-  <span class="co">#============= STEP 1 ======================================</span>
-  <span class="co"># pick the best hyper-params and then do final training on all data...</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/writeLines.html">writeLines</a></span><span class="op">(</span><span class="st">'Cross validation'</span><span class="op">)</span>
-  <span class="va">param.sel</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/lapply.html">lapply</a></span><span class="op">(</span>
-    <span class="va">param</span>, 
-    <span class="kw">function</span><span class="op">(</span><span class="va">x</span><span class="op">)</span><span class="op">{</span>
-      <span class="fu"><a href="https://rdrr.io/r/base/do.call.html">do.call</a></span><span class="op">(</span>
-        <span class="va">made_up_model</span>, 
-        <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>
-          param <span class="op">=</span> <span class="va">x</span>, 
-          final <span class="op">=</span> <span class="cn">F</span>, 
-          data <span class="op">=</span> <span class="va">matrixData</span>, 
-          labels <span class="op">=</span> <span class="va">labels</span>
-          <span class="op">)</span>  
-      <span class="op">)</span>
-      <span class="op">}</span>
-    <span class="op">)</span>
-  <span class="va">hyperSummary</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/do.call.html">do.call</a></span><span class="op">(</span><span class="va">rbind</span>, <span class="fu"><a href="https://rdrr.io/r/base/lapply.html">lapply</a></span><span class="op">(</span><span class="va">param.sel</span>, <span class="kw">function</span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="va">x</span><span class="op">$</span><span class="va">hyperSum</span><span class="op">)</span><span class="op">)</span>
-  <span class="va">hyperSummary</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/as.data.frame.html">as.data.frame</a></span><span class="op">(</span><span class="va">hyperSummary</span><span class="op">)</span>
-  <span class="va">hyperSummary</span><span class="op">$</span><span class="va">auc</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/unlist.html">unlist</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/lapply.html">lapply</a></span><span class="op">(</span><span class="va">param.sel</span>, <span class="kw">function</span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="va">x</span><span class="op">$</span><span class="va">auc</span><span class="op">)</span><span class="op">)</span> 
-  <span class="va">param.sel</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/unlist.html">unlist</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/lapply.html">lapply</a></span><span class="op">(</span><span class="va">param.sel</span>, <span class="kw">function</span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="va">x</span><span class="op">$</span><span class="va">auc</span><span class="op">)</span><span class="op">)</span>
-  <span class="va">bestInd</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/which.min.html">which.max</a></span><span class="op">(</span><span class="va">param.sel</span><span class="op">)</span>
-  
-  <span class="co">#get cross val prediction for best hyper-parameters</span>
-  <span class="va">prediction</span> <span class="op">&lt;-</span> <span class="va">param.sel</span><span class="op">[[</span><span class="va">bestInd</span><span class="op">]</span><span class="op">]</span><span class="op">$</span><span class="va">prediction</span>
-  <span class="va">prediction</span><span class="op">$</span><span class="va">evaluationType</span> <span class="op">&lt;-</span> <span class="st">'CV'</span>
-  
-  <span class="fu"><a href="https://rdrr.io/r/base/writeLines.html">writeLines</a></span><span class="op">(</span><span class="st">'final train'</span><span class="op">)</span>
-  <span class="va">finalResult</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/do.call.html">do.call</a></span><span class="op">(</span>
-    <span class="va">made_up_model</span>, 
-    <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>
-      param <span class="op">=</span> <span class="va">param</span><span class="op">[[</span><span class="va">bestInd</span><span class="op">]</span><span class="op">]</span>, 
-      final <span class="op">=</span> <span class="cn">T</span>, 
-      data <span class="op">=</span> <span class="va">matrixData</span>, 
-      labels <span class="op">=</span> <span class="va">labels</span>
-      <span class="op">)</span>  
-    <span class="op">)</span>
-  
-  <span class="va">trainedModel</span> <span class="op">&lt;-</span> <span class="va">finalResult</span><span class="op">$</span><span class="va">model</span>
-  
-  <span class="co"># prediction risk on training data:</span>
-  <span class="va">finalResult</span><span class="op">$</span><span class="va">prediction</span><span class="op">$</span><span class="va">evaluationType</span> <span class="op">&lt;-</span> <span class="st">'Train'</span>
-  
-  <span class="co"># get CV and train prediction</span>
-  <span class="va">prediction</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/cbind.html">rbind</a></span><span class="op">(</span><span class="va">prediction</span>, <span class="va">finalResult</span><span class="op">$</span><span class="va">prediction</span><span class="op">)</span>
-  
-  <span class="va">varImp</span> <span class="op">&lt;-</span> <span class="va">covariateRef</span> <span class="op">%&gt;%</span> <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/compute.html">collect</a></span><span class="op">(</span><span class="op">)</span>
-  <span class="co"># no feature importance available</span>
-  <span class="va">vqrImp</span><span class="op">$</span><span class="va">covariateValue</span> <span class="op">&lt;-</span> <span class="fl">0</span> 
-  
- <span class="va">timeToTrain</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/Sys.time.html">Sys.time</a></span><span class="op">(</span><span class="op">)</span> <span class="op">-</span> <span class="va">start</span>
-
-  <span class="co"># construct the standard output for a model:</span>
-  <span class="va">result</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>model <span class="op">=</span> <span class="va">trainedModel</span>,
-                 prediction <span class="op">=</span> <span class="va">prediction</span>, 
-    settings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>
-      plpDataSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">plpDataSettings</span>,
-      covariateSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">covariateSettings</span>,
-      populationSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">populationSettings</span>,
-      featureEngineering <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">featureEngineering</span>,
-      tidyCovariates <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">tidyCovariateDataSettings</span>, 
-      requireDenseMatrix <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">param</span>, <span class="st">'settings'</span><span class="op">)</span><span class="op">$</span><span class="va">requiresDenseMatrix</span>,
-      modelSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>
-        model <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">param</span>, <span class="st">'settings'</span><span class="op">)</span><span class="op">$</span><span class="va">modelName</span>, <span class="co"># the model name</span>
-        param <span class="op">=</span> <span class="va">param</span>,
-        finalModelParameters <span class="op">=</span> <span class="va">param</span><span class="op">[[</span><span class="va">bestInd</span><span class="op">]</span><span class="op">]</span>, <span class="co"># best hyper-parameters</span>
-        extraSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">param</span>, <span class="st">'settings'</span><span class="op">)</span>
-      <span class="op">)</span>,
-      splitSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">splitSettings</span>,
-      sampleSettings <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">sampleSettings</span>
-    <span class="op">)</span>,
-    
-    trainDetails <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>
-      analysisId <span class="op">=</span> <span class="va">analysisId</span>,
-      cdmDatabaseSchema <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">cdmDatabaseSchema</span>,
-      outcomeId <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">outcomeId</span>,
-      cohortId <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">cohortId</span>,
-      attrition <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span>, <span class="st">"metaData"</span><span class="op">)</span><span class="op">$</span><span class="va">attrition</span>, 
-      trainingTime <span class="op">=</span> <span class="va">timeToTrain</span>, <span class="co"># how long it took to train the model</span>
-      trainingDate <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/Sys.time.html">Sys.Date</a></span><span class="op">(</span><span class="op">)</span>,
-      hyperParamSearch <span class="op">=</span> <span class="va">hyperSummary</span> <span class="co"># the hyper-parameters and performance data.frame</span>
-    <span class="op">)</span>,
-    covariateImportance <span class="op">=</span> <span class="va">varImp</span>
-  <span class="op">)</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/class.html">class</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'plpModel'</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">result</span>, <span class="st">'predictionFunction'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'madeupPrediction'</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">result</span>, <span class="st">'modelType'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'binary'</span>
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span>
-    
-<span class="op">}</span></code></pre></div>
+<div class="section level3">
+<h3 id="fit-1">Fit<a class="anchor" aria-label="anchor" href="#fit-1"></a>
+</h3>
+<div class="sourceCode" id="cb6"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb6-1"><a href="#cb6-1" tabindex="-1"></a>fitMadeUp <span class="ot">&lt;-</span> <span class="cf">function</span>(trainData, modelSettings, search, analysisId){</span>
+<span id="cb6-2"><a href="#cb6-2" tabindex="-1"></a></span>
+<span id="cb6-3"><a href="#cb6-3" tabindex="-1"></a>  <span class="co"># set the seed for reproducibility</span></span>
+<span id="cb6-4"><a href="#cb6-4" tabindex="-1"></a>  param <span class="ot">&lt;-</span> modelSettings<span class="sc">$</span>param</span>
+<span id="cb6-5"><a href="#cb6-5" tabindex="-1"></a>  <span class="fu">set.seed</span>(<span class="fu">attr</span>(param, <span class="st">'settings'</span>)<span class="sc">$</span>seed)</span>
+<span id="cb6-6"><a href="#cb6-6" tabindex="-1"></a>  </span>
+<span id="cb6-7"><a href="#cb6-7" tabindex="-1"></a>  <span class="co"># add folds to labels:</span></span>
+<span id="cb6-8"><a href="#cb6-8" tabindex="-1"></a>  trainData<span class="sc">$</span>labels <span class="ot">&lt;-</span> <span class="fu">merge</span>(trainData<span class="sc">$</span>labels, trainData<span class="sc">$</span>folds, <span class="at">by=</span> <span class="st">'rowId'</span>)</span>
+<span id="cb6-9"><a href="#cb6-9" tabindex="-1"></a>  <span class="co"># convert data into sparse R Matrix:</span></span>
+<span id="cb6-10"><a href="#cb6-10" tabindex="-1"></a>  mappedData <span class="ot">&lt;-</span> <span class="fu">toSparseM</span>(trainData,<span class="at">map=</span><span class="cn">NULL</span>)</span>
+<span id="cb6-11"><a href="#cb6-11" tabindex="-1"></a>  matrixData <span class="ot">&lt;-</span> mappedData<span class="sc">$</span>dataMatrix</span>
+<span id="cb6-12"><a href="#cb6-12" tabindex="-1"></a>  labels <span class="ot">&lt;-</span> mappedData<span class="sc">$</span>labels</span>
+<span id="cb6-13"><a href="#cb6-13" tabindex="-1"></a>  covariateRef <span class="ot">&lt;-</span> mappedData<span class="sc">$</span>covariateRef</span>
+<span id="cb6-14"><a href="#cb6-14" tabindex="-1"></a></span>
+<span id="cb6-15"><a href="#cb6-15" tabindex="-1"></a>  <span class="co">#============= STEP 1 ======================================</span></span>
+<span id="cb6-16"><a href="#cb6-16" tabindex="-1"></a>  <span class="co"># pick the best hyper-params and then do final training on all data...</span></span>
+<span id="cb6-17"><a href="#cb6-17" tabindex="-1"></a>  <span class="fu">writeLines</span>(<span class="st">'Cross validation'</span>)</span>
+<span id="cb6-18"><a href="#cb6-18" tabindex="-1"></a>  param.sel <span class="ot">&lt;-</span> <span class="fu">lapply</span>(</span>
+<span id="cb6-19"><a href="#cb6-19" tabindex="-1"></a>    param, </span>
+<span id="cb6-20"><a href="#cb6-20" tabindex="-1"></a>    <span class="cf">function</span>(x){</span>
+<span id="cb6-21"><a href="#cb6-21" tabindex="-1"></a>      <span class="fu">do.call</span>(</span>
+<span id="cb6-22"><a href="#cb6-22" tabindex="-1"></a>        made_up_model, </span>
+<span id="cb6-23"><a href="#cb6-23" tabindex="-1"></a>        <span class="fu">list</span>(</span>
+<span id="cb6-24"><a href="#cb6-24" tabindex="-1"></a>          <span class="at">param =</span> x, </span>
+<span id="cb6-25"><a href="#cb6-25" tabindex="-1"></a>          <span class="at">final =</span> F, </span>
+<span id="cb6-26"><a href="#cb6-26" tabindex="-1"></a>          <span class="at">data =</span> matrixData, </span>
+<span id="cb6-27"><a href="#cb6-27" tabindex="-1"></a>          <span class="at">labels =</span> labels</span>
+<span id="cb6-28"><a href="#cb6-28" tabindex="-1"></a>          )  </span>
+<span id="cb6-29"><a href="#cb6-29" tabindex="-1"></a>      )</span>
+<span id="cb6-30"><a href="#cb6-30" tabindex="-1"></a>      }</span>
+<span id="cb6-31"><a href="#cb6-31" tabindex="-1"></a>    )</span>
+<span id="cb6-32"><a href="#cb6-32" tabindex="-1"></a>  hyperSummary <span class="ot">&lt;-</span> <span class="fu">do.call</span>(rbind, <span class="fu">lapply</span>(param.sel, <span class="cf">function</span>(x) x<span class="sc">$</span>hyperSum))</span>
+<span id="cb6-33"><a href="#cb6-33" tabindex="-1"></a>  hyperSummary <span class="ot">&lt;-</span> <span class="fu">as.data.frame</span>(hyperSummary)</span>
+<span id="cb6-34"><a href="#cb6-34" tabindex="-1"></a>  hyperSummary<span class="sc">$</span>auc <span class="ot">&lt;-</span> <span class="fu">unlist</span>(<span class="fu">lapply</span>(param.sel, <span class="cf">function</span>(x) x<span class="sc">$</span>auc)) </span>
+<span id="cb6-35"><a href="#cb6-35" tabindex="-1"></a>  param.sel <span class="ot">&lt;-</span> <span class="fu">unlist</span>(<span class="fu">lapply</span>(param.sel, <span class="cf">function</span>(x) x<span class="sc">$</span>auc))</span>
+<span id="cb6-36"><a href="#cb6-36" tabindex="-1"></a>  bestInd <span class="ot">&lt;-</span> <span class="fu">which.max</span>(param.sel)</span>
+<span id="cb6-37"><a href="#cb6-37" tabindex="-1"></a>  </span>
+<span id="cb6-38"><a href="#cb6-38" tabindex="-1"></a>  <span class="co">#get cross val prediction for best hyper-parameters</span></span>
+<span id="cb6-39"><a href="#cb6-39" tabindex="-1"></a>  prediction <span class="ot">&lt;-</span> param.sel[[bestInd]]<span class="sc">$</span>prediction</span>
+<span id="cb6-40"><a href="#cb6-40" tabindex="-1"></a>  prediction<span class="sc">$</span>evaluationType <span class="ot">&lt;-</span> <span class="st">'CV'</span></span>
+<span id="cb6-41"><a href="#cb6-41" tabindex="-1"></a>  </span>
+<span id="cb6-42"><a href="#cb6-42" tabindex="-1"></a>  <span class="fu">writeLines</span>(<span class="st">'final train'</span>)</span>
+<span id="cb6-43"><a href="#cb6-43" tabindex="-1"></a>  finalResult <span class="ot">&lt;-</span> <span class="fu">do.call</span>(</span>
+<span id="cb6-44"><a href="#cb6-44" tabindex="-1"></a>    made_up_model, </span>
+<span id="cb6-45"><a href="#cb6-45" tabindex="-1"></a>    <span class="fu">list</span>(</span>
+<span id="cb6-46"><a href="#cb6-46" tabindex="-1"></a>      <span class="at">param =</span> param[[bestInd]], </span>
+<span id="cb6-47"><a href="#cb6-47" tabindex="-1"></a>      <span class="at">final =</span> T, </span>
+<span id="cb6-48"><a href="#cb6-48" tabindex="-1"></a>      <span class="at">data =</span> matrixData, </span>
+<span id="cb6-49"><a href="#cb6-49" tabindex="-1"></a>      <span class="at">labels =</span> labels</span>
+<span id="cb6-50"><a href="#cb6-50" tabindex="-1"></a>      )  </span>
+<span id="cb6-51"><a href="#cb6-51" tabindex="-1"></a>    )</span>
+<span id="cb6-52"><a href="#cb6-52" tabindex="-1"></a>  </span>
+<span id="cb6-53"><a href="#cb6-53" tabindex="-1"></a>  trainedModel <span class="ot">&lt;-</span> finalResult<span class="sc">$</span>model</span>
+<span id="cb6-54"><a href="#cb6-54" tabindex="-1"></a>  </span>
+<span id="cb6-55"><a href="#cb6-55" tabindex="-1"></a>  <span class="co"># prediction risk on training data:</span></span>
+<span id="cb6-56"><a href="#cb6-56" tabindex="-1"></a>  finalResult<span class="sc">$</span>prediction<span class="sc">$</span>evaluationType <span class="ot">&lt;-</span> <span class="st">'Train'</span></span>
+<span id="cb6-57"><a href="#cb6-57" tabindex="-1"></a>  </span>
+<span id="cb6-58"><a href="#cb6-58" tabindex="-1"></a>  <span class="co"># get CV and train prediction</span></span>
+<span id="cb6-59"><a href="#cb6-59" tabindex="-1"></a>  prediction <span class="ot">&lt;-</span> <span class="fu">rbind</span>(prediction, finalResult<span class="sc">$</span>prediction)</span>
+<span id="cb6-60"><a href="#cb6-60" tabindex="-1"></a>  </span>
+<span id="cb6-61"><a href="#cb6-61" tabindex="-1"></a>  varImp <span class="ot">&lt;-</span> covariateRef <span class="sc">%&gt;%</span> dplyr<span class="sc">::</span><span class="fu">collect</span>()</span>
+<span id="cb6-62"><a href="#cb6-62" tabindex="-1"></a>  <span class="co"># no feature importance available</span></span>
+<span id="cb6-63"><a href="#cb6-63" tabindex="-1"></a>  vqrImp<span class="sc">$</span>covariateValue <span class="ot">&lt;-</span> <span class="dv">0</span> </span>
+<span id="cb6-64"><a href="#cb6-64" tabindex="-1"></a>  </span>
+<span id="cb6-65"><a href="#cb6-65" tabindex="-1"></a> timeToTrain <span class="ot">&lt;-</span> <span class="fu">Sys.time</span>() <span class="sc">-</span> start</span>
+<span id="cb6-66"><a href="#cb6-66" tabindex="-1"></a></span>
+<span id="cb6-67"><a href="#cb6-67" tabindex="-1"></a>  <span class="co"># construct the standard output for a model:</span></span>
+<span id="cb6-68"><a href="#cb6-68" tabindex="-1"></a>  result <span class="ot">&lt;-</span> <span class="fu">list</span>(<span class="at">model =</span> trainedModel,</span>
+<span id="cb6-69"><a href="#cb6-69" tabindex="-1"></a>                 <span class="at">prediction =</span> prediction, </span>
+<span id="cb6-70"><a href="#cb6-70" tabindex="-1"></a>    <span class="at">preprocessing =</span> <span class="fu">list</span>(</span>
+<span id="cb6-71"><a href="#cb6-71" tabindex="-1"></a>                   <span class="at">featureEngineering =</span> <span class="fu">attr</span>(trainData<span class="sc">$</span>covariateData, <span class="st">"metaData"</span>)<span class="sc">$</span>featureEngineering,</span>
+<span id="cb6-72"><a href="#cb6-72" tabindex="-1"></a>      <span class="at">tidyCovariates =</span> <span class="fu">attr</span>(trainData<span class="sc">$</span>covariateData, <span class="st">"metaData"</span>)<span class="sc">$</span>tidyCovariateDataSettings, </span>
+<span id="cb6-73"><a href="#cb6-73" tabindex="-1"></a>      <span class="at">requireDenseMatrix =</span> <span class="fu">attr</span>(param, <span class="st">'settings'</span>)<span class="sc">$</span>requiresDenseMatrix,</span>
+<span id="cb6-74"><a href="#cb6-74" tabindex="-1"></a>      </span>
+<span id="cb6-75"><a href="#cb6-75" tabindex="-1"></a>                 ),</span>
+<span id="cb6-76"><a href="#cb6-76" tabindex="-1"></a>    <span class="at">modelDesign =</span> <span class="fu">list</span>(</span>
+<span id="cb6-77"><a href="#cb6-77" tabindex="-1"></a>      <span class="at">outcomeId =</span> <span class="fu">attr</span>(trainData, <span class="st">"metaData"</span>)<span class="sc">$</span>outcomeId,</span>
+<span id="cb6-78"><a href="#cb6-78" tabindex="-1"></a>      <span class="at">targetId =</span> <span class="fu">attr</span>(trainData, <span class="st">"metaData"</span>)<span class="sc">$</span>targetId,</span>
+<span id="cb6-79"><a href="#cb6-79" tabindex="-1"></a>      <span class="at">plpDataSettings =</span> <span class="fu">attr</span>(trainData, <span class="st">"metaData"</span>)<span class="sc">$</span>plpDataSettings,</span>
+<span id="cb6-80"><a href="#cb6-80" tabindex="-1"></a>      <span class="at">covariateSettings =</span> <span class="fu">attr</span>(trainData, <span class="st">"metaData"</span>)<span class="sc">$</span>covariateSettings,</span>
+<span id="cb6-81"><a href="#cb6-81" tabindex="-1"></a>      <span class="at">populationSettings =</span> <span class="fu">attr</span>(trainData, <span class="st">"metaData"</span>)<span class="sc">$</span>populationSettings,</span>
+<span id="cb6-82"><a href="#cb6-82" tabindex="-1"></a>      <span class="at">featureEngineeringSettings =</span> <span class="fu">attr</span>(trainData<span class="sc">$</span>covariateData, <span class="st">"metaData"</span>)<span class="sc">$</span>featureEngineeringSettings,</span>
+<span id="cb6-83"><a href="#cb6-83" tabindex="-1"></a>      <span class="at">prerocessSettings =</span> <span class="fu">attr</span>(trainData<span class="sc">$</span>covariateData, <span class="st">"metaData"</span>)<span class="sc">$</span>prerocessSettings, </span>
+<span id="cb6-84"><a href="#cb6-84" tabindex="-1"></a>      <span class="at">modelSettings =</span> <span class="fu">list</span>(</span>
+<span id="cb6-85"><a href="#cb6-85" tabindex="-1"></a>        <span class="at">model =</span> <span class="fu">attr</span>(param, <span class="st">'settings'</span>)<span class="sc">$</span>modelName, <span class="co"># the model name</span></span>
+<span id="cb6-86"><a href="#cb6-86" tabindex="-1"></a>        <span class="at">param =</span> param,</span>
+<span id="cb6-87"><a href="#cb6-87" tabindex="-1"></a>        <span class="at">finalModelParameters =</span> param[[bestInd]], <span class="co"># best hyper-parameters</span></span>
+<span id="cb6-88"><a href="#cb6-88" tabindex="-1"></a>        <span class="at">extraSettings =</span> <span class="fu">attr</span>(param, <span class="st">'settings'</span>)</span>
+<span id="cb6-89"><a href="#cb6-89" tabindex="-1"></a>      ),</span>
+<span id="cb6-90"><a href="#cb6-90" tabindex="-1"></a>      <span class="at">splitSettings =</span> <span class="fu">attr</span>(trainData, <span class="st">"metaData"</span>)<span class="sc">$</span>splitSettings,</span>
+<span id="cb6-91"><a href="#cb6-91" tabindex="-1"></a>      <span class="at">sampleSettings =</span> <span class="fu">attr</span>(trainData, <span class="st">"metaData"</span>)<span class="sc">$</span>sampleSettings</span>
+<span id="cb6-92"><a href="#cb6-92" tabindex="-1"></a>    ),</span>
+<span id="cb6-93"><a href="#cb6-93" tabindex="-1"></a>    </span>
+<span id="cb6-94"><a href="#cb6-94" tabindex="-1"></a>    <span class="at">trainDetails =</span> <span class="fu">list</span>(</span>
+<span id="cb6-95"><a href="#cb6-95" tabindex="-1"></a>      <span class="at">analysisId =</span> analysisId,</span>
+<span id="cb6-96"><a href="#cb6-96" tabindex="-1"></a>      <span class="at">developmentDatabase =</span> <span class="fu">attr</span>(trainData, <span class="st">"metaData"</span>)<span class="sc">$</span>cdmDatabaseSchema,</span>
+<span id="cb6-97"><a href="#cb6-97" tabindex="-1"></a>      <span class="at">attrition =</span> <span class="fu">attr</span>(trainData, <span class="st">"metaData"</span>)<span class="sc">$</span>attrition, </span>
+<span id="cb6-98"><a href="#cb6-98" tabindex="-1"></a>      <span class="at">trainingTime =</span> timeToTrain, <span class="co"># how long it took to train the model</span></span>
+<span id="cb6-99"><a href="#cb6-99" tabindex="-1"></a>      <span class="at">trainingDate =</span> <span class="fu">Sys.Date</span>(),</span>
+<span id="cb6-100"><a href="#cb6-100" tabindex="-1"></a>      <span class="at">hyperParamSearch =</span> hyperSummary <span class="co"># the hyper-parameters and performance data.frame</span></span>
+<span id="cb6-101"><a href="#cb6-101" tabindex="-1"></a>    ),</span>
+<span id="cb6-102"><a href="#cb6-102" tabindex="-1"></a>    <span class="at">covariateImportance =</span> <span class="fu">merge</span>(trainData<span class="sc">$</span>covariateData<span class="sc">$</span>covariateRef, varImp, <span class="at">by=</span><span class="st">'covariateId'</span>) <span class="co"># add variable importance to covariateRef if possible</span></span>
+<span id="cb6-103"><a href="#cb6-103" tabindex="-1"></a>  ),</span>
+<span id="cb6-104"><a href="#cb6-104" tabindex="-1"></a>    covariateImportance <span class="ot">=</span> varImp</span>
+<span id="cb6-105"><a href="#cb6-105" tabindex="-1"></a>  <span class="er">)</span></span>
+<span id="cb6-106"><a href="#cb6-106" tabindex="-1"></a>  <span class="fu">class</span>(result) <span class="ot">&lt;-</span> <span class="st">'plpModel'</span></span>
+<span id="cb6-107"><a href="#cb6-107" tabindex="-1"></a>  <span class="fu">attr</span>(result, <span class="st">'predictionFunction'</span>) <span class="ot">&lt;-</span> <span class="st">'madeupPrediction'</span></span>
+<span id="cb6-108"><a href="#cb6-108" tabindex="-1"></a>  <span class="fu">attr</span>(result, <span class="st">'modelType'</span>) <span class="ot">&lt;-</span> <span class="st">'binary'</span></span>
+<span id="cb6-109"><a href="#cb6-109" tabindex="-1"></a>  <span class="fu">return</span>(result)</span>
+<span id="cb6-110"><a href="#cb6-110" tabindex="-1"></a>    </span>
+<span id="cb6-111"><a href="#cb6-111" tabindex="-1"></a>}</span></code></pre></div>
 </div>
-<div id="helpers" class="section level2">
-<h2 class="hasAnchor">
-<a href="#helpers" class="anchor"></a>Helpers</h2>
-<p>In the fit model a helper function <code>made_up_model</code> is called, this is the function that trains a model given the data, labels and hyper-parameters.</p>
+<div class="section level3">
+<h3 id="helpers">Helpers<a class="anchor" aria-label="anchor" href="#helpers"></a>
+</h3>
+<p>In the fit model a helper function <code>made_up_model</code> is
+called, this is the function that trains a model given the data, labels
+and hyper-parameters.</p>
 <div class="sourceCode" id="cb7"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">made_up_model</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">param</span>, <span class="va">data</span>, <span class="va">final</span><span class="op">=</span><span class="cn">F</span>, <span class="va">labels</span><span class="op">)</span><span class="op">{</span>
-  
-  <span class="kw">if</span><span class="op">(</span><span class="va">final</span><span class="op">==</span><span class="cn">F</span><span class="op">)</span><span class="op">{</span>
-    <span class="co"># add value column to store all predictions</span>
-    <span class="va">labels</span><span class="op">$</span><span class="va">value</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/rep.html">rep</a></span><span class="op">(</span><span class="fl">0</span>, <span class="fu"><a href="https://rdrr.io/r/base/nrow.html">nrow</a></span><span class="op">(</span><span class="va">labels</span><span class="op">)</span><span class="op">)</span>
-    <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">labels</span>, <span class="st">"metaData"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>modelType <span class="op">=</span> <span class="st">"binary"</span><span class="op">)</span>
-    
-    <span class="va">foldPerm</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="op">)</span> <span class="co"># this holds CV aucs</span>
-    <span class="kw">for</span><span class="op">(</span><span class="va">index</span> <span class="kw">in</span> <span class="fl">1</span><span class="op">:</span><span class="fu"><a href="https://rdrr.io/r/base/Extremes.html">max</a></span><span class="op">(</span><span class="va">labels</span><span class="op">$</span><span class="va">index</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-      <span class="va">model</span> <span class="op">&lt;-</span> <span class="fu">madeup</span><span class="fu">::</span><span class="fu">model</span><span class="op">(</span>
-        x <span class="op">=</span> <span class="va">data</span><span class="op">[</span><span class="va">labels</span><span class="op">$</span><span class="va">index</span><span class="op">!=</span><span class="va">index</span>,<span class="op">]</span>, <span class="co"># remove left out fold</span>
-        y <span class="op">=</span> <span class="va">labels</span><span class="op">$</span><span class="va">outcomeCount</span><span class="op">[</span><span class="va">labels</span><span class="op">$</span><span class="va">index</span><span class="op">!=</span><span class="va">index</span><span class="op">]</span>,
-        a <span class="op">=</span> <span class="va">param</span><span class="op">$</span><span class="va">a</span>, 
-        b <span class="op">=</span> <span class="va">param</span><span class="op">$</span><span class="va">b</span>
-      <span class="op">)</span>
-      
-      <span class="co"># predict on left out fold</span>
-      <span class="va">pred</span> <span class="op">&lt;-</span> <span class="fu">stats</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/r/stats/predict.html">predict</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">data</span><span class="op">[</span><span class="va">labels</span><span class="op">$</span><span class="va">index</span><span class="op">==</span><span class="va">index</span>,<span class="op">]</span><span class="op">)</span>
-      <span class="va">labels</span><span class="op">$</span><span class="va">value</span><span class="op">[</span><span class="va">labels</span><span class="op">$</span><span class="va">index</span><span class="op">==</span><span class="va">index</span><span class="op">]</span> <span class="op">&lt;-</span> <span class="va">pred</span>
-      
-      <span class="co"># calculate auc on help out fold  </span>
-      <span class="va">aucVal</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/computeAuc.html">computeAuc</a></span><span class="op">(</span><span class="va">labels</span><span class="op">[</span><span class="va">labels</span><span class="op">$</span><span class="va">index</span><span class="op">==</span><span class="va">index</span>,<span class="op">]</span><span class="op">)</span>
-      <span class="va">foldPerm</span><span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="va">foldPerm</span>,<span class="va">aucVal</span><span class="op">)</span>    
-    <span class="op">}</span>
-    <span class="va">auc</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/computeAuc.html">computeAuc</a></span><span class="op">(</span><span class="va">labels</span><span class="op">)</span> <span class="co"># overal AUC</span>
-
-  <span class="op">}</span> <span class="kw">else</span> <span class="op">{</span>
-    <span class="va">model</span> <span class="op">&lt;-</span> <span class="fu">madeup</span><span class="fu">::</span><span class="fu">model</span><span class="op">(</span>
-      x <span class="op">=</span> <span class="va">data</span>, 
-      y <span class="op">=</span> <span class="va">labels</span><span class="op">$</span><span class="va">outcomeCount</span>,
-      a <span class="op">=</span> <span class="va">param</span><span class="op">$</span><span class="va">a</span>,
-      b <span class="op">=</span> <span class="va">param</span><span class="op">$</span><span class="va">b</span>
-      <span class="op">)</span>
-    
-    <span class="va">pred</span> <span class="op">&lt;-</span> <span class="fu">stats</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/r/stats/predict.html">predict</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">data</span><span class="op">)</span>
-    <span class="va">labels</span><span class="op">$</span><span class="va">value</span> <span class="op">&lt;-</span> <span class="va">pred</span>
-    <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">labels</span>, <span class="st">"metaData"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>modelType <span class="op">=</span> <span class="st">"binary"</span><span class="op">)</span> 
-    <span class="va">auc</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/computeAuc.html">computeAuc</a></span><span class="op">(</span><span class="va">labels</span><span class="op">)</span>
-    <span class="va">foldPerm</span> <span class="op">&lt;-</span> <span class="va">auc</span>
-  <span class="op">}</span>
-  
-  <span class="va">result</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>
-    model <span class="op">=</span> <span class="va">model</span>,
-    auc <span class="op">=</span> <span class="va">auc</span>,
-    prediction <span class="op">=</span> <span class="va">labels</span>,
-    hyperSum <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span>a <span class="op">=</span> <span class="va">a</span>, b <span class="op">=</span> <span class="va">b</span>, fold_auc <span class="op">=</span> <span class="va">foldPerm</span><span class="op">)</span>
-  <span class="op">)</span>
-  
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span>
-<span class="op">}</span></code></pre></div>
+<code class="sourceCode R"><span><span class="va">made_up_model</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">param</span>, <span class="va">data</span>, <span class="va">final</span><span class="op">=</span><span class="cn">F</span>, <span class="va">labels</span><span class="op">)</span><span class="op">{</span></span>
+<span>  </span>
+<span>  <span class="kw">if</span><span class="op">(</span><span class="va">final</span><span class="op">==</span><span class="cn">F</span><span class="op">)</span><span class="op">{</span></span>
+<span>    <span class="co"># add value column to store all predictions</span></span>
+<span>    <span class="va">labels</span><span class="op">$</span><span class="va">value</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/rep.html" class="external-link">rep</a></span><span class="op">(</span><span class="fl">0</span>, <span class="fu"><a href="https://rdrr.io/r/base/nrow.html" class="external-link">nrow</a></span><span class="op">(</span><span class="va">labels</span><span class="op">)</span><span class="op">)</span></span>
+<span>    <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">labels</span>, <span class="st">"metaData"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>modelType <span class="op">=</span> <span class="st">"binary"</span><span class="op">)</span></span>
+<span>    </span>
+<span>    <span class="va">foldPerm</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="op">)</span> <span class="co"># this holds CV aucs</span></span>
+<span>    <span class="kw">for</span><span class="op">(</span><span class="va">index</span> <span class="kw">in</span> <span class="fl">1</span><span class="op">:</span><span class="fu"><a href="https://rdrr.io/r/base/Extremes.html" class="external-link">max</a></span><span class="op">(</span><span class="va">labels</span><span class="op">$</span><span class="va">index</span><span class="op">)</span><span class="op">)</span><span class="op">{</span></span>
+<span>      <span class="va">model</span> <span class="op">&lt;-</span> <span class="fu">madeup</span><span class="fu">::</span><span class="fu">model</span><span class="op">(</span></span>
+<span>        x <span class="op">=</span> <span class="va">data</span><span class="op">[</span><span class="va">labels</span><span class="op">$</span><span class="va">index</span><span class="op">!=</span><span class="va">index</span>,<span class="op">]</span>, <span class="co"># remove left out fold</span></span>
+<span>        y <span class="op">=</span> <span class="va">labels</span><span class="op">$</span><span class="va">outcomeCount</span><span class="op">[</span><span class="va">labels</span><span class="op">$</span><span class="va">index</span><span class="op">!=</span><span class="va">index</span><span class="op">]</span>,</span>
+<span>        a <span class="op">=</span> <span class="va">param</span><span class="op">$</span><span class="va">a</span>, </span>
+<span>        b <span class="op">=</span> <span class="va">param</span><span class="op">$</span><span class="va">b</span></span>
+<span>      <span class="op">)</span></span>
+<span>      </span>
+<span>      <span class="co"># predict on left out fold</span></span>
+<span>      <span class="va">pred</span> <span class="op">&lt;-</span> <span class="fu">stats</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/r/stats/predict.html" class="external-link">predict</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">data</span><span class="op">[</span><span class="va">labels</span><span class="op">$</span><span class="va">index</span><span class="op">==</span><span class="va">index</span>,<span class="op">]</span><span class="op">)</span></span>
+<span>      <span class="va">labels</span><span class="op">$</span><span class="va">value</span><span class="op">[</span><span class="va">labels</span><span class="op">$</span><span class="va">index</span><span class="op">==</span><span class="va">index</span><span class="op">]</span> <span class="op">&lt;-</span> <span class="va">pred</span></span>
+<span>      </span>
+<span>      <span class="co"># calculate auc on help out fold  </span></span>
+<span>      <span class="va">aucVal</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/computeAuc.html">computeAuc</a></span><span class="op">(</span><span class="va">labels</span><span class="op">[</span><span class="va">labels</span><span class="op">$</span><span class="va">index</span><span class="op">==</span><span class="va">index</span>,<span class="op">]</span><span class="op">)</span></span>
+<span>      <span class="va">foldPerm</span><span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="va">foldPerm</span>,<span class="va">aucVal</span><span class="op">)</span>    </span>
+<span>    <span class="op">}</span></span>
+<span>    <span class="va">auc</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/computeAuc.html">computeAuc</a></span><span class="op">(</span><span class="va">labels</span><span class="op">)</span> <span class="co"># overal AUC</span></span>
+<span></span>
+<span>  <span class="op">}</span> <span class="kw">else</span> <span class="op">{</span></span>
+<span>    <span class="va">model</span> <span class="op">&lt;-</span> <span class="fu">madeup</span><span class="fu">::</span><span class="fu">model</span><span class="op">(</span></span>
+<span>      x <span class="op">=</span> <span class="va">data</span>, </span>
+<span>      y <span class="op">=</span> <span class="va">labels</span><span class="op">$</span><span class="va">outcomeCount</span>,</span>
+<span>      a <span class="op">=</span> <span class="va">param</span><span class="op">$</span><span class="va">a</span>,</span>
+<span>      b <span class="op">=</span> <span class="va">param</span><span class="op">$</span><span class="va">b</span></span>
+<span>      <span class="op">)</span></span>
+<span>    </span>
+<span>    <span class="va">pred</span> <span class="op">&lt;-</span> <span class="fu">stats</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/r/stats/predict.html" class="external-link">predict</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">data</span><span class="op">)</span></span>
+<span>    <span class="va">labels</span><span class="op">$</span><span class="va">value</span> <span class="op">&lt;-</span> <span class="va">pred</span></span>
+<span>    <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">labels</span>, <span class="st">"metaData"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>modelType <span class="op">=</span> <span class="st">"binary"</span><span class="op">)</span> </span>
+<span>    <span class="va">auc</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/computeAuc.html">computeAuc</a></span><span class="op">(</span><span class="va">labels</span><span class="op">)</span></span>
+<span>    <span class="va">foldPerm</span> <span class="op">&lt;-</span> <span class="va">auc</span></span>
+<span>  <span class="op">}</span></span>
+<span>  </span>
+<span>  <span class="va">result</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span></span>
+<span>    model <span class="op">=</span> <span class="va">model</span>,</span>
+<span>    auc <span class="op">=</span> <span class="va">auc</span>,</span>
+<span>    prediction <span class="op">=</span> <span class="va">labels</span>,</span>
+<span>    hyperSum <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span>a <span class="op">=</span> <span class="va">a</span>, b <span class="op">=</span> <span class="va">b</span>, fold_auc <span class="op">=</span> <span class="va">foldPerm</span><span class="op">)</span></span>
+<span>  <span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="kw"><a href="https://rdrr.io/r/base/function.html" class="external-link">return</a></span><span class="op">(</span><span class="va">result</span><span class="op">)</span></span>
+<span><span class="op">}</span></span></code></pre></div>
 </div>
-<div id="predict-1" class="section level2">
-<h2 class="hasAnchor">
-<a href="#predict-1" class="anchor"></a>Predict</h2>
-<p>The final step is to create a predict function for the model. In the example above the predeiction function <code>attr(result, 'predictionFunction') &lt;- 'madeupPrediction'</code> was madeupPrediction, so a <code>madeupPrediction</code> function is required when applying the model. The predict function needs to take as input the plpModel returned by the fit function, new data to apply the model on and the cohort specifying the patients of interest to make the prediction for.</p>
+<div class="section level3">
+<h3 id="predict-1">Predict<a class="anchor" aria-label="anchor" href="#predict-1"></a>
+</h3>
+<p>The final step is to create a predict function for the model. In the
+example above the predeiction function
+<code>attr(result, 'predictionFunction') &lt;- 'madeupPrediction'</code>
+was madeupPrediction, so a <code>madeupPrediction</code> function is
+required when applying the model. The predict function needs to take as
+input the plpModel returned by the fit function, new data to apply the
+model on and the cohort specifying the patients of interest to make the
+prediction for.</p>
 <div class="sourceCode" id="cb8"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">madeupPrediction</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">plpModel</span>, <span class="va">data</span> , <span class="va">cohort</span><span class="op">)</span><span class="op">{</span> 
-  
-  <span class="kw">if</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/class.html">class</a></span><span class="op">(</span><span class="va">data</span><span class="op">)</span> <span class="op">==</span> <span class="st">'plpData'</span><span class="op">)</span><span class="op">{</span>
-    <span class="co"># convert</span>
-    <span class="va">matrixObjects</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/toSparseM.html">toSparseM</a></span><span class="op">(</span>
-      plpData <span class="op">=</span> <span class="va">data</span>, 
-      cohort <span class="op">=</span> <span class="va">cohort</span>,
-      map <span class="op">=</span> <span class="va">plpModel</span><span class="op">$</span><span class="va">covariateImportance</span> <span class="op">%&gt;%</span> 
-        <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/select.html">select</a></span><span class="op">(</span><span class="va">.data</span><span class="op">$</span><span class="va">columnId</span>, <span class="va">.data</span><span class="op">$</span><span class="va">covariateId</span><span class="op">)</span>
-    <span class="op">)</span>
-    
-    <span class="va">newData</span> <span class="op">&lt;-</span> <span class="va">matrixObjects</span><span class="op">$</span><span class="va">dataMatrix</span>
-    <span class="va">cohort</span> <span class="op">&lt;-</span> <span class="va">matrixObjects</span><span class="op">$</span><span class="va">labels</span>
-    
-  <span class="op">}</span><span class="kw">else</span><span class="op">{</span>
-    <span class="va">newData</span> <span class="op">&lt;-</span> <span class="va">data</span>
-  <span class="op">}</span>
-  
-  <span class="kw">if</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/class.html">class</a></span><span class="op">(</span><span class="va">plpModel</span><span class="op">)</span> <span class="op">==</span> <span class="st">'plpModel'</span><span class="op">)</span><span class="op">{</span>
-    <span class="va">model</span> <span class="op">&lt;-</span> <span class="va">plpModel</span><span class="op">$</span><span class="va">model</span>
-  <span class="op">}</span> <span class="kw">else</span><span class="op">{</span>
-    <span class="va">model</span> <span class="op">&lt;-</span> <span class="va">plpModel</span>
-  <span class="op">}</span>
-  
-  <span class="va">cohort</span><span class="op">$</span><span class="va">value</span> <span class="op">&lt;-</span> <span class="fu">stats</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/r/stats/predict.html">predict</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">data</span><span class="op">)</span>
-  
-  <span class="co"># fix the rowIds to be the old ones</span>
-  <span class="co"># now use the originalRowId and remove the matrix rowId</span>
-  <span class="va">cohort</span> <span class="op">&lt;-</span> <span class="va">cohort</span> <span class="op">%&gt;%</span> 
-    <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/select.html">select</a></span><span class="op">(</span><span class="op">-</span><span class="va">.data</span><span class="op">$</span><span class="va">rowId</span><span class="op">)</span> <span class="op">%&gt;%</span>
-    <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/rename.html">rename</a></span><span class="op">(</span>rowId <span class="op">=</span> <span class="va">.data</span><span class="op">$</span><span class="va">originalRowId</span><span class="op">)</span>
-  
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">cohort</span>, <span class="st">"metaData"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>modelType <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">plpModel</span>, <span class="st">'modelType'</span><span class="op">)</span><span class="op">)</span> 
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">cohort</span><span class="op">)</span>
-  
-<span class="op">}</span></code></pre></div>
-<p>As the madeup model uses the standard R prediction, it has the same prediction function as xgboost, so we could have not added a new prediction function and instead made the predictionFunction of the result returned by fitMadeUpModel to <code>attr(result, 'predictionFunction') &lt;- 'predictXgboost'</code>.</p>
+<code class="sourceCode R"><span><span class="va">madeupPrediction</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">plpModel</span>, <span class="va">data</span> , <span class="va">cohort</span><span class="op">)</span><span class="op">{</span> </span>
+<span>  </span>
+<span>  <span class="kw">if</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/class.html" class="external-link">class</a></span><span class="op">(</span><span class="va">data</span><span class="op">)</span> <span class="op">==</span> <span class="st">'plpData'</span><span class="op">)</span><span class="op">{</span></span>
+<span>    <span class="co"># convert</span></span>
+<span>    <span class="va">matrixObjects</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/toSparseM.html">toSparseM</a></span><span class="op">(</span></span>
+<span>      plpData <span class="op">=</span> <span class="va">data</span>, </span>
+<span>      cohort <span class="op">=</span> <span class="va">cohort</span>,</span>
+<span>      map <span class="op">=</span> <span class="va">plpModel</span><span class="op">$</span><span class="va">covariateImportance</span> <span class="op">%&gt;%</span> </span>
+<span>        <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/select.html" class="external-link">select</a></span><span class="op">(</span><span class="st">"columnId"</span>, <span class="st">"covariateId"</span><span class="op">)</span></span>
+<span>    <span class="op">)</span></span>
+<span>    </span>
+<span>    <span class="va">newData</span> <span class="op">&lt;-</span> <span class="va">matrixObjects</span><span class="op">$</span><span class="va">dataMatrix</span></span>
+<span>    <span class="va">cohort</span> <span class="op">&lt;-</span> <span class="va">matrixObjects</span><span class="op">$</span><span class="va">labels</span></span>
+<span>    </span>
+<span>  <span class="op">}</span><span class="kw">else</span><span class="op">{</span></span>
+<span>    <span class="va">newData</span> <span class="op">&lt;-</span> <span class="va">data</span></span>
+<span>  <span class="op">}</span></span>
+<span>  </span>
+<span>  <span class="kw">if</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/class.html" class="external-link">class</a></span><span class="op">(</span><span class="va">plpModel</span><span class="op">)</span> <span class="op">==</span> <span class="st">'plpModel'</span><span class="op">)</span><span class="op">{</span></span>
+<span>    <span class="va">model</span> <span class="op">&lt;-</span> <span class="va">plpModel</span><span class="op">$</span><span class="va">model</span></span>
+<span>  <span class="op">}</span> <span class="kw">else</span><span class="op">{</span></span>
+<span>    <span class="va">model</span> <span class="op">&lt;-</span> <span class="va">plpModel</span></span>
+<span>  <span class="op">}</span></span>
+<span>  </span>
+<span>  <span class="va">cohort</span><span class="op">$</span><span class="va">value</span> <span class="op">&lt;-</span> <span class="fu">stats</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/r/stats/predict.html" class="external-link">predict</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">data</span><span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="co"># fix the rowIds to be the old ones</span></span>
+<span>  <span class="co"># now use the originalRowId and remove the matrix rowId</span></span>
+<span>  <span class="va">cohort</span> <span class="op">&lt;-</span> <span class="va">cohort</span> <span class="op">%&gt;%</span> </span>
+<span>    <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/select.html" class="external-link">select</a></span><span class="op">(</span><span class="op">-</span><span class="st">"rowId"</span><span class="op">)</span> <span class="op">%&gt;%</span></span>
+<span>    <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/rename.html" class="external-link">rename</a></span><span class="op">(</span>rowId <span class="op">=</span> <span class="st">"originalRowId"</span><span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">cohort</span>, <span class="st">"metaData"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>modelType <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">plpModel</span>, <span class="st">'modelType'</span><span class="op">)</span><span class="op">)</span> </span>
+<span>  <span class="kw"><a href="https://rdrr.io/r/base/function.html" class="external-link">return</a></span><span class="op">(</span><span class="va">cohort</span><span class="op">)</span></span>
+<span>  </span>
+<span><span class="op">}</span></span></code></pre></div>
+<p>As the madeup model uses the standard R prediction, it has the same
+prediction function as xgboost, so we could have not added a new
+prediction function and instead made the predictionFunction of the
+result returned by fitMadeUpModel to
+<code>attr(result, 'predictionFunction') &lt;- 'predictXgboost'</code>.</p>
 </div>
 </div>
-<div id="acknowledgments" class="section level1">
-<h1 class="hasAnchor">
-<a href="#acknowledgments" class="anchor"></a>Acknowledgments</h1>
-<p>Considerable work has been dedicated to provide the <code>PatientLevelPrediction</code> package.</p>
+<div class="section level2">
+<h2 id="acknowledgments">Acknowledgments<a class="anchor" aria-label="anchor" href="#acknowledgments"></a>
+</h2>
+<p>Considerable work has been dedicated to provide the
+<code>PatientLevelPrediction</code> package.</p>
 <div class="sourceCode" id="cb9"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/utils/citation.html">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></code></pre></div>
-<pre><code>## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design
-## and implementation of a standardized framework to generate and evaluate
-## patient-level prediction models using observational healthcare data."
-## _Journal of the American Medical Informatics Association_, *25*(8),
-## 969-975. &lt;URL: https://doi.org/10.1093/jamia/ocy032&gt;.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }</code></pre>
-<p><strong>Please reference this paper if you use the PLP Package in your work:</strong></p>
-<p><a href="http://dx.doi.org/10.1093/jamia/ocy032">Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.</a></p>
-<p>This work is supported in part through the National Science Foundation grant IIS 1251151.</p>
+<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/utils/citation.html" class="external-link">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></span></code></pre></div>
+<pre><code><span><span class="co">## </span></span>
+<span><span class="co">## To cite PatientLevelPrediction in publications use:</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">##   Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design</span></span>
+<span><span class="co">##   and implementation of a standardized framework to generate and</span></span>
+<span><span class="co">##   evaluate patient-level prediction models using observational</span></span>
+<span><span class="co">##   healthcare data." _Journal of the American Medical Informatics</span></span>
+<span><span class="co">##   Association_, *25*(8), 969-975.</span></span>
+<span><span class="co">##   &lt;https://doi.org/10.1093/jamia/ocy032&gt;.</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">## A BibTeX entry for LaTeX users is</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">##   @Article{,</span></span>
+<span><span class="co">##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},</span></span>
+<span><span class="co">##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},</span></span>
+<span><span class="co">##     journal = {Journal of the American Medical Informatics Association},</span></span>
+<span><span class="co">##     volume = {25},</span></span>
+<span><span class="co">##     number = {8},</span></span>
+<span><span class="co">##     pages = {969-975},</span></span>
+<span><span class="co">##     year = {2018},</span></span>
+<span><span class="co">##     url = {https://doi.org/10.1093/jamia/ocy032},</span></span>
+<span><span class="co">##   }</span></span></code></pre>
+<p><strong>Please reference this paper if you use the PLP Package in
+your work:</strong></p>
+<p><a href="http://dx.doi.org/10.1093/jamia/ocy032" class="external-link">Reps JM, Schuemie
+MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a
+standardized framework to generate and evaluate patient-level prediction
+models using observational healthcare data. J Am Med Inform Assoc.
+2018;25(8):969-975.</a></p>
+<p>This work is supported in part through the National Science
+Foundation grant IIS 1251151.</p>
 </div>
   </div>
 
@@ -626,11 +759,13 @@ <h1 class="hasAnchor">
 
 
       <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+  <p></p>
+<p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p>
+<p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
       </footer>
@@ -639,5 +774,7 @@ <h1 class="hasAnchor">
   
 
 
+  
+
   </body>
 </html>
diff --git a/docs/articles/AddingCustomModels_files/header-attrs-2.11/header-attrs.js b/docs/articles/AddingCustomModels_files/header-attrs-2.11/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/AddingCustomModels_files/header-attrs-2.11/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/AddingCustomSamples.html b/docs/articles/AddingCustomSamples.html
index c6a73fa11..84ccdde27 100644
--- a/docs/articles/AddingCustomSamples.html
+++ b/docs/articles/AddingCustomSamples.html
@@ -19,6 +19,8 @@
 <![endif]-->
 </head>
 <body data-spy="scroll" data-target="#toc">
+    
+
     <div class="container template-article">
       <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
@@ -31,7 +33,7 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
@@ -53,7 +55,7 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
@@ -72,10 +74,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -84,13 +86,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -101,21 +106,30 @@
   </ul>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
       </ul>
 <ul class="nav navbar-nav navbar-right">
 <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
@@ -130,15 +144,15 @@
 
       
 
-      </header><script src="AddingCustomSamples_files/header-attrs-2.11/header-attrs.js"></script><div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header toc-ignore">
       <h1 data-toc-skip>Adding Custom Sampling Functions</h1>
-                        <h4 class="author">Jenna Reps</h4>
+                        <h4 data-toc-skip class="author">Jenna Reps</h4>
             
-            <h4 class="date">2022-03-09</h4>
+            <h4 data-toc-skip class="date">2024-09-09</h4>
       
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/AddingCustomSamples.Rmd"><code>vignettes/AddingCustomSamples.Rmd</code></a></small>
+      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/vignettes/AddingCustomSamples.Rmd" class="external-link"><code>vignettes/AddingCustomSamples.Rmd</code></a></small>
       <div class="hidden name"><code>AddingCustomSamples.Rmd</code></div>
 
     </div>
@@ -146,137 +160,178 @@ <h4 class="date">2022-03-09</h4>
     
     
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Adding Custom Sampling}
 -->
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
-<p>This vignette describes how you can add your own custom function for sampling the target population in the Observational Health Data Sciencs and Informatics (OHDSI) <a href="http://github.com/OHDSI/PatientLevelPrediction"><code>PatientLevelPrediction</code></a> package. This vignette assumes you have read and are comfortable with building single patient level prediction models as described in the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf"><code>BuildingPredictiveModels</code> vignette</a>.</p>
-<p><strong>We invite you to share your new sample functions with the OHDSI community through our <a href="http://github.com/OHDSI/PatientLevelPrediction">GitHub repository</a>.</strong></p>
+<div class="section level2">
+<h2 id="introduction">Introduction<a class="anchor" aria-label="anchor" href="#introduction"></a>
+</h2>
+<p>This vignette describes how you can add your own custom function for
+sampling the target population in the Observational Health Data Sciencs
+and Informatics (OHDSI) <a href="http://github.com/OHDSI/PatientLevelPrediction" class="external-link"><code>PatientLevelPrediction</code></a>
+package. This vignette assumes you have read and are comfortable with
+building single patient level prediction models as described in the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf" class="external-link"><code>BuildingPredictiveModels</code>
+vignette</a>.</p>
+<p><strong>We invite you to share your new sample functions with the
+OHDSI community through our <a href="http://github.com/OHDSI/PatientLevelPrediction" class="external-link">GitHub
+repository</a>.</strong></p>
 </div>
-<div id="sample-function-code-structure" class="section level1">
-<h1 class="hasAnchor">
-<a href="#sample-function-code-structure" class="anchor"></a>Sample Function Code Structure</h1>
-<p>To make a sampling function that can be used within PatientLevelPrediction you need to write two different functions. The ‘create’ function and the ‘implement’ function.</p>
-<p>The ‘create’ function, e.g., create&lt;SampleFunctionName&gt;, takes the parameters of the sample ‘implement’ function as input, checks these are valid and outputs these as a list of class ‘sampleSettings’ with the ‘fun’ attribute specifying the ‘implement’ function to call.</p>
-<p>The ‘implement’ function, e.g., implement&lt;SampleFunctionName&gt;, must take as input: * trainData - a list containing: - covariateData: the plpData$covariateData restricted to the training patients - labels: a data frame that contain rowId (patient identifier) and outcomeCount (the class labels) - folds: a data.frame that contains rowId (patient identifier) and index (the cross validation fold) * sampleSettings - the output of your create&lt;SampleFunctionName&gt;</p>
-<p>The ‘implement’ function can then do any manipulation of the trainData (such as undersampling or oversampling) but must output a trainData object containing the covariateData, labels and folds for the new training data sample.</p>
+<div class="section level2">
+<h2 id="sample-function-code-structure">Sample Function Code Structure<a class="anchor" aria-label="anchor" href="#sample-function-code-structure"></a>
+</h2>
+<p>To make a sampling function that can be used within
+PatientLevelPrediction you need to write two different functions. The
+‘create’ function and the ‘implement’ function.</p>
+<p>The ‘create’ function, e.g., create&lt;SampleFunctionName&gt;, takes
+the parameters of the sample ‘implement’ function as input, checks these
+are valid and outputs these as a list of class ‘sampleSettings’ with the
+‘fun’ attribute specifying the ‘implement’ function to call.</p>
+<p>The ‘implement’ function, e.g., implement&lt;SampleFunctionName&gt;,
+must take as input: * trainData - a list containing: - covariateData:
+the plpData$covariateData restricted to the training patients - labels:
+a data frame that contain rowId (patient identifier) and outcomeCount
+(the class labels) - folds: a data.frame that contains rowId (patient
+identifier) and index (the cross validation fold) * sampleSettings - the
+output of your create&lt;SampleFunctionName&gt;</p>
+<p>The ‘implement’ function can then do any manipulation of the
+trainData (such as undersampling or oversampling) but must output a
+trainData object containing the covariateData, labels and folds for the
+new training data sample.</p>
 </div>
-<div id="example" class="section level1">
-<h1 class="hasAnchor">
-<a href="#example" class="anchor"></a>Example</h1>
-<p>Let’s consider the situation where we wish to take a random sample of the training data population. To make this custom sampling function we need to write the ‘create’ and ‘implement’ R functions.</p>
-<div id="create-function" class="section level2">
-<h2 class="hasAnchor">
-<a href="#create-function" class="anchor"></a>Create function</h2>
-<p>Our random sampling function will randomly sample <code>n</code> patients from the trainData. Therefore, the inputs for this are: * <code>n</code> an integer/double specifying the number of patients to sample * <code>sampleSeed</code> an integer/double specifying the seed for reproducibility</p>
+<div class="section level2">
+<h2 id="example">Example<a class="anchor" aria-label="anchor" href="#example"></a>
+</h2>
+<p>Let’s consider the situation where we wish to take a random sample of
+the training data population. To make this custom sampling function we
+need to write the ‘create’ and ‘implement’ R functions.</p>
+<div class="section level3">
+<h3 id="create-function">Create function<a class="anchor" aria-label="anchor" href="#create-function"></a>
+</h3>
+<p>Our random sampling function will randomly sample <code>n</code>
+patients from the trainData. Therefore, the inputs for this are: *
+<code>n</code> an integer/double specifying the number of patients to
+sample * <code>sampleSeed</code> an integer/double specifying the seed
+for reproducibility</p>
 <div class="sourceCode" id="cb1"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">createRandomSampleSettings</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span>
-                     <span class="va">n</span> <span class="op">=</span> <span class="fl">10000</span>,
-                     <span class="va">sampleSeed</span> <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/sample.html">sample</a></span><span class="op">(</span><span class="fl">10000</span>,<span class="fl">1</span><span class="op">)</span>
-                     <span class="op">)</span><span class="op">{</span>
-  
-  <span class="co"># add input checks</span>
-  <span class="fu">checkIsClass</span><span class="op">(</span><span class="va">n</span>, <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="st">'numeric'</span>,<span class="st">'integer'</span><span class="op">)</span><span class="op">)</span>
-  <span class="fu">checkHigher</span><span class="op">(</span><span class="va">n</span>,<span class="fl">0</span><span class="op">)</span>
-  <span class="fu">checkIsClass</span><span class="op">(</span><span class="va">sampleSeed</span>, <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="st">'numeric'</span>,<span class="st">'integer'</span><span class="op">)</span><span class="op">)</span>
-  
-  <span class="co"># create list of inputs to implement function</span>
-  <span class="va">sampleSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>
-    n <span class="op">=</span> <span class="va">n</span>,
-    sampleSeed  <span class="op">=</span> <span class="va">sampleSeed</span> 
-    <span class="op">)</span>
-  
-  <span class="co"># specify the function that will implement the sampling</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">sampleSettings</span>, <span class="st">"fun"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">"implementRandomSampleSettings"</span>
-
-  <span class="co"># make sure the object returned is of class "sampleSettings"</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/class.html">class</a></span><span class="op">(</span><span class="va">sampleSettings</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">"sampleSettings"</span>
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">sampleSettings</span><span class="op">)</span>
-  
-<span class="op">}</span></code></pre></div>
-<p>We now need to create the ‘implement’ function <code>implementRandomSampleSettings()</code></p>
+<code class="sourceCode R"><span><span class="va">createRandomSampleSettings</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span></span>
+<span>                     <span class="va">n</span> <span class="op">=</span> <span class="fl">10000</span>,</span>
+<span>                     <span class="va">sampleSeed</span> <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/sample.html" class="external-link">sample</a></span><span class="op">(</span><span class="fl">10000</span>,<span class="fl">1</span><span class="op">)</span></span>
+<span>                     <span class="op">)</span><span class="op">{</span></span>
+<span>  </span>
+<span>  <span class="co"># add input checks</span></span>
+<span>  <span class="fu">checkIsClass</span><span class="op">(</span><span class="va">n</span>, <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">'numeric'</span>,<span class="st">'integer'</span><span class="op">)</span><span class="op">)</span></span>
+<span>  <span class="fu">checkHigher</span><span class="op">(</span><span class="va">n</span>,<span class="fl">0</span><span class="op">)</span></span>
+<span>  <span class="fu">checkIsClass</span><span class="op">(</span><span class="va">sampleSeed</span>, <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">'numeric'</span>,<span class="st">'integer'</span><span class="op">)</span><span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="co"># create list of inputs to implement function</span></span>
+<span>  <span class="va">sampleSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span></span>
+<span>    n <span class="op">=</span> <span class="va">n</span>,</span>
+<span>    sampleSeed  <span class="op">=</span> <span class="va">sampleSeed</span> </span>
+<span>    <span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="co"># specify the function that will implement the sampling</span></span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">sampleSettings</span>, <span class="st">"fun"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">"implementRandomSampleSettings"</span></span>
+<span></span>
+<span>  <span class="co"># make sure the object returned is of class "sampleSettings"</span></span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/class.html" class="external-link">class</a></span><span class="op">(</span><span class="va">sampleSettings</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">"sampleSettings"</span></span>
+<span>  <span class="kw"><a href="https://rdrr.io/r/base/function.html" class="external-link">return</a></span><span class="op">(</span><span class="va">sampleSettings</span><span class="op">)</span></span>
+<span>  </span>
+<span><span class="op">}</span></span></code></pre></div>
+<p>We now need to create the ‘implement’ function
+<code>implementRandomSampleSettings()</code></p>
 </div>
-<div id="implement-function" class="section level2">
-<h2 class="hasAnchor">
-<a href="#implement-function" class="anchor"></a>Implement function</h2>
-<p>All ‘implement’ functions must take as input the trainData and the sampleSettings (this is the output of the ‘create’ function). They must return a trainData object containing the covariateData, labels and folds.</p>
-<p>In our example, the <code>createRandomSampleSettings()</code> will return a list with ‘n’ and ‘sampleSeed’. The sampleSettings therefore contains these.</p>
+<div class="section level3">
+<h3 id="implement-function">Implement function<a class="anchor" aria-label="anchor" href="#implement-function"></a>
+</h3>
+<p>All ‘implement’ functions must take as input the trainData and the
+sampleSettings (this is the output of the ‘create’ function). They must
+return a trainData object containing the covariateData, labels and
+folds.</p>
+<p>In our example, the <code>createRandomSampleSettings()</code> will
+return a list with ‘n’ and ‘sampleSeed’. The sampleSettings therefore
+contains these.</p>
 <div class="sourceCode" id="cb2"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">implementRandomSampleSettings</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">trainData</span>, <span class="va">sampleSettings</span><span class="op">)</span><span class="op">{</span>
-
-  <span class="va">n</span> <span class="op">&lt;-</span> <span class="va">sampleSetting</span><span class="op">$</span><span class="va">n</span>
-  <span class="va">sampleSeed</span> <span class="op">&lt;-</span> <span class="va">sampleSetting</span><span class="op">$</span><span class="va">sampleSeed</span>
-  
-  <span class="kw">if</span><span class="op">(</span><span class="va">n</span> <span class="op">&gt;</span> <span class="fu"><a href="https://rdrr.io/r/base/nrow.html">nrow</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">labels</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-    <span class="kw"><a href="https://rdrr.io/r/base/stop.html">stop</a></span><span class="op">(</span><span class="st">'Sample n bigger than training population'</span><span class="op">)</span>
-  <span class="op">}</span>
-  
-  <span class="co"># set the seed for the randomization</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/Random.html">set.seed</a></span><span class="op">(</span><span class="va">sampleSeed</span><span class="op">)</span>
-  
-  <span class="co"># now implement the code to do your desired sampling</span>
-  
-  <span class="va">sampleRowIds</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/sample.html">sample</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">labels</span><span class="op">$</span><span class="va">rowId</span>, <span class="va">n</span><span class="op">)</span>
-  
-  <span class="va">sampleTrainData</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span><span class="op">)</span>
-  
-  <span class="va">sampleTrainData</span><span class="op">$</span><span class="va">labels</span> <span class="op">&lt;-</span> <span class="va">trainData</span><span class="op">$</span><span class="va">labels</span> <span class="op">%&gt;%</span> 
-    <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/filter.html">filter</a></span><span class="op">(</span><span class="va">.data</span><span class="op">$</span><span class="va">rowId</span> <span class="op">%in%</span> <span class="va">sampleRowIds</span><span class="op">)</span> <span class="op">%&gt;%</span> 
-    <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/compute.html">collect</a></span><span class="op">(</span><span class="op">)</span>
-  
-  <span class="va">sampleTrainData</span><span class="op">$</span><span class="va">folds</span> <span class="op">&lt;-</span> <span class="va">trainData</span><span class="op">$</span><span class="va">folds</span> <span class="op">%&gt;%</span> 
-    <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/filter.html">filter</a></span><span class="op">(</span><span class="va">.data</span><span class="op">$</span><span class="va">rowId</span> <span class="op">%in%</span> <span class="va">sampleRowIds</span><span class="op">)</span> <span class="op">%&gt;%</span> 
-    <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/compute.html">collect</a></span><span class="op">(</span><span class="op">)</span>
-  
-  <span class="va">sampleTrainData</span><span class="op">$</span><span class="va">covariateData</span> <span class="op">&lt;-</span> <span class="fu">Andromeda</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/Andromeda/man/andromeda_constructor.html">andromeda</a></span><span class="op">(</span><span class="op">)</span>
-  <span class="va">sampleTrainData</span><span class="op">$</span><span class="va">covariateData</span><span class="op">$</span><span class="va">covariateRef</span> <span class="op">&lt;-</span><span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span><span class="op">$</span><span class="va">covariateRef</span>
-  <span class="va">sampleTrainData</span><span class="op">$</span><span class="va">covariateData</span><span class="op">$</span><span class="va">covariates</span> <span class="op">&lt;-</span> <span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span><span class="op">$</span><span class="va">covariates</span> <span class="op">%&gt;%</span> <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/filter.html">filter</a></span><span class="op">(</span><span class="va">.data</span><span class="op">$</span><span class="va">rowId</span> <span class="op">%in%</span> <span class="va">sampleRowIds</span><span class="op">)</span>
-  
-  <span class="co">#update metaData$populationSize </span>
-  <span class="va">metaData</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span>, <span class="st">'metaData'</span><span class="op">)</span>
-  <span class="va">metaData</span><span class="op">$</span><span class="va">populationSize</span> <span class="op">=</span> <span class="va">n</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">sampleTrainData</span><span class="op">$</span><span class="va">covariateData</span>, <span class="st">'metaData'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">metaData</span>
-  
-  <span class="co"># make the cocvariateData the correct class</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/class.html">class</a></span><span class="op">(</span><span class="va">sampleTrainData</span><span class="op">$</span><span class="va">covariateData</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'CovariateData'</span>
-  
-  <span class="co"># return the updated trainData</span>
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">sampleTrainData</span><span class="op">)</span>
-<span class="op">}</span></code></pre></div>
+<code class="sourceCode R"><span><span class="va">implementRandomSampleSettings</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">trainData</span>, <span class="va">sampleSettings</span><span class="op">)</span><span class="op">{</span></span>
+<span></span>
+<span>  <span class="va">n</span> <span class="op">&lt;-</span> <span class="va">sampleSetting</span><span class="op">$</span><span class="va">n</span></span>
+<span>  <span class="va">sampleSeed</span> <span class="op">&lt;-</span> <span class="va">sampleSetting</span><span class="op">$</span><span class="va">sampleSeed</span></span>
+<span>  </span>
+<span>  <span class="kw">if</span><span class="op">(</span><span class="va">n</span> <span class="op">&gt;</span> <span class="fu"><a href="https://rdrr.io/r/base/nrow.html" class="external-link">nrow</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">labels</span><span class="op">)</span><span class="op">)</span><span class="op">{</span></span>
+<span>    <span class="kw"><a href="https://rdrr.io/r/base/stop.html" class="external-link">stop</a></span><span class="op">(</span><span class="st">'Sample n bigger than training population'</span><span class="op">)</span></span>
+<span>  <span class="op">}</span></span>
+<span>  </span>
+<span>  <span class="co"># set the seed for the randomization</span></span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/Random.html" class="external-link">set.seed</a></span><span class="op">(</span><span class="va">sampleSeed</span><span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="co"># now implement the code to do your desired sampling</span></span>
+<span>  </span>
+<span>  <span class="va">sampleRowIds</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/sample.html" class="external-link">sample</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">labels</span><span class="op">$</span><span class="va">rowId</span>, <span class="va">n</span><span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="va">sampleTrainData</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="va">sampleTrainData</span><span class="op">$</span><span class="va">labels</span> <span class="op">&lt;-</span> <span class="va">trainData</span><span class="op">$</span><span class="va">labels</span> <span class="op">%&gt;%</span> </span>
+<span>    <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/filter.html" class="external-link">filter</a></span><span class="op">(</span><span class="va">.data</span><span class="op">$</span><span class="va">rowId</span> <span class="op"><a href="https://rdrr.io/r/base/match.html" class="external-link">%in%</a></span> <span class="va">sampleRowIds</span><span class="op">)</span> <span class="op">%&gt;%</span> </span>
+<span>    <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/compute.html" class="external-link">collect</a></span><span class="op">(</span><span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="va">sampleTrainData</span><span class="op">$</span><span class="va">folds</span> <span class="op">&lt;-</span> <span class="va">trainData</span><span class="op">$</span><span class="va">folds</span> <span class="op">%&gt;%</span> </span>
+<span>    <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/filter.html" class="external-link">filter</a></span><span class="op">(</span><span class="va">.data</span><span class="op">$</span><span class="va">rowId</span> <span class="op"><a href="https://rdrr.io/r/base/match.html" class="external-link">%in%</a></span> <span class="va">sampleRowIds</span><span class="op">)</span> <span class="op">%&gt;%</span> </span>
+<span>    <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/compute.html" class="external-link">collect</a></span><span class="op">(</span><span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="va">sampleTrainData</span><span class="op">$</span><span class="va">covariateData</span> <span class="op">&lt;-</span> <span class="fu">Andromeda</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/Andromeda/man/andromeda_constructor.html" class="external-link">andromeda</a></span><span class="op">(</span><span class="op">)</span></span>
+<span>  <span class="va">sampleTrainData</span><span class="op">$</span><span class="va">covariateData</span><span class="op">$</span><span class="va">covariateRef</span> <span class="op">&lt;-</span><span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span><span class="op">$</span><span class="va">covariateRef</span></span>
+<span>  <span class="va">sampleTrainData</span><span class="op">$</span><span class="va">covariateData</span><span class="op">$</span><span class="va">covariates</span> <span class="op">&lt;-</span> <span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span><span class="op">$</span><span class="va">covariates</span> <span class="op">%&gt;%</span> <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/filter.html" class="external-link">filter</a></span><span class="op">(</span><span class="va">.data</span><span class="op">$</span><span class="va">rowId</span> <span class="op"><a href="https://rdrr.io/r/base/match.html" class="external-link">%in%</a></span> <span class="va">sampleRowIds</span><span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="co">#update metaData$populationSize </span></span>
+<span>  <span class="va">metaData</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">trainData</span><span class="op">$</span><span class="va">covariateData</span>, <span class="st">'metaData'</span><span class="op">)</span></span>
+<span>  <span class="va">metaData</span><span class="op">$</span><span class="va">populationSize</span> <span class="op">=</span> <span class="va">n</span></span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">sampleTrainData</span><span class="op">$</span><span class="va">covariateData</span>, <span class="st">'metaData'</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">metaData</span></span>
+<span>  </span>
+<span>  <span class="co"># make the cocvariateData the correct class</span></span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/class.html" class="external-link">class</a></span><span class="op">(</span><span class="va">sampleTrainData</span><span class="op">$</span><span class="va">covariateData</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">'CovariateData'</span></span>
+<span>  </span>
+<span>  <span class="co"># return the updated trainData</span></span>
+<span>  <span class="kw"><a href="https://rdrr.io/r/base/function.html" class="external-link">return</a></span><span class="op">(</span><span class="va">sampleTrainData</span><span class="op">)</span></span>
+<span><span class="op">}</span></span></code></pre></div>
 </div>
 </div>
-<div id="acknowledgments" class="section level1">
-<h1 class="hasAnchor">
-<a href="#acknowledgments" class="anchor"></a>Acknowledgments</h1>
-<p>Considerable work has been dedicated to provide the <code>PatientLevelPrediction</code> package.</p>
+<div class="section level2">
+<h2 id="acknowledgments">Acknowledgments<a class="anchor" aria-label="anchor" href="#acknowledgments"></a>
+</h2>
+<p>Considerable work has been dedicated to provide the
+<code>PatientLevelPrediction</code> package.</p>
 <div class="sourceCode" id="cb3"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/utils/citation.html">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></code></pre></div>
-<pre><code>## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design
-## and implementation of a standardized framework to generate and evaluate
-## patient-level prediction models using observational healthcare data."
-## _Journal of the American Medical Informatics Association_, *25*(8),
-## 969-975. &lt;URL: https://doi.org/10.1093/jamia/ocy032&gt;.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }</code></pre>
-<p><strong>Please reference this paper if you use the PLP Package in your work:</strong></p>
-<p><a href="http://dx.doi.org/10.1093/jamia/ocy032">Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.</a></p>
-<p>This work is supported in part through the National Science Foundation grant IIS 1251151.</p>
+<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/utils/citation.html" class="external-link">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></span></code></pre></div>
+<pre><code><span><span class="co">## </span></span>
+<span><span class="co">## To cite PatientLevelPrediction in publications use:</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">##   Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design</span></span>
+<span><span class="co">##   and implementation of a standardized framework to generate and</span></span>
+<span><span class="co">##   evaluate patient-level prediction models using observational</span></span>
+<span><span class="co">##   healthcare data." _Journal of the American Medical Informatics</span></span>
+<span><span class="co">##   Association_, *25*(8), 969-975.</span></span>
+<span><span class="co">##   &lt;https://doi.org/10.1093/jamia/ocy032&gt;.</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">## A BibTeX entry for LaTeX users is</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">##   @Article{,</span></span>
+<span><span class="co">##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},</span></span>
+<span><span class="co">##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},</span></span>
+<span><span class="co">##     journal = {Journal of the American Medical Informatics Association},</span></span>
+<span><span class="co">##     volume = {25},</span></span>
+<span><span class="co">##     number = {8},</span></span>
+<span><span class="co">##     pages = {969-975},</span></span>
+<span><span class="co">##     year = {2018},</span></span>
+<span><span class="co">##     url = {https://doi.org/10.1093/jamia/ocy032},</span></span>
+<span><span class="co">##   }</span></span></code></pre>
+<p><strong>Please reference this paper if you use the PLP Package in
+your work:</strong></p>
+<p><a href="http://dx.doi.org/10.1093/jamia/ocy032" class="external-link">Reps JM, Schuemie
+MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a
+standardized framework to generate and evaluate patient-level prediction
+models using observational healthcare data. J Am Med Inform Assoc.
+2018;25(8):969-975.</a></p>
+<p>This work is supported in part through the National Science
+Foundation grant IIS 1251151.</p>
 </div>
   </div>
 
@@ -291,11 +346,13 @@ <h1 class="hasAnchor">
 
 
       <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+  <p></p>
+<p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p>
+<p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
       </footer>
@@ -304,5 +361,7 @@ <h1 class="hasAnchor">
   
 
 
+  
+
   </body>
 </html>
diff --git a/docs/articles/AddingCustomSamples_files/header-attrs-2.11/header-attrs.js b/docs/articles/AddingCustomSamples_files/header-attrs-2.11/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/AddingCustomSamples_files/header-attrs-2.11/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/AddingCustomSplitting.html b/docs/articles/AddingCustomSplitting.html
index 24683120c..6f56760db 100644
--- a/docs/articles/AddingCustomSplitting.html
+++ b/docs/articles/AddingCustomSplitting.html
@@ -19,6 +19,8 @@
 <![endif]-->
 </head>
 <body data-spy="scroll" data-target="#toc">
+    
+
     <div class="container template-article">
       <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
@@ -31,7 +33,7 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
@@ -53,7 +55,7 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
@@ -72,10 +74,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -84,13 +86,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -101,21 +106,30 @@
   </ul>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
       </ul>
 <ul class="nav navbar-nav navbar-right">
 <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
@@ -130,15 +144,15 @@
 
       
 
-      </header><script src="AddingCustomSplitting_files/header-attrs-2.11/header-attrs.js"></script><div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header toc-ignore">
       <h1 data-toc-skip>Adding Custom Data Splitting Functions</h1>
-                        <h4 class="author">Jenna Reps</h4>
+                        <h4 data-toc-skip class="author">Jenna Reps</h4>
             
-            <h4 class="date">2022-03-09</h4>
+            <h4 data-toc-skip class="date">2024-09-09</h4>
       
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/AddingCustomSplitting.Rmd"><code>vignettes/AddingCustomSplitting.Rmd</code></a></small>
+      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/vignettes/AddingCustomSplitting.Rmd" class="external-link"><code>vignettes/AddingCustomSplitting.Rmd</code></a></small>
       <div class="hidden name"><code>AddingCustomSplitting.Rmd</code></div>
 
     </div>
@@ -146,103 +160,144 @@ <h4 class="date">2022-03-09</h4>
     
     
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Adding Custom Data Splitting}
 -->
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
-<p>This vignette describes how you can add your own custom function for splitting the labelled data into training data and validation data in the Observational Health Data Sciencs and Informatics (OHDSI) <a href="http://github.com/OHDSI/PatientLevelPrediction"><code>PatientLevelPrediction</code></a> package. This vignette assumes you have read and are comfortable with building single patient level prediction models as described in the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf"><code>BuildingPredictiveModels</code> vignette</a>.</p>
-<p><strong>We invite you to share your new data splitting functions with the OHDSI community through our <a href="http://github.com/OHDSI/PatientLevelPrediction">GitHub repository</a>.</strong></p>
+<div class="section level2">
+<h2 id="introduction">Introduction<a class="anchor" aria-label="anchor" href="#introduction"></a>
+</h2>
+<p>This vignette describes how you can add your own custom function for
+splitting the labelled data into training data and validation data in
+the Observational Health Data Sciencs and Informatics (OHDSI) <a href="http://github.com/OHDSI/PatientLevelPrediction" class="external-link"><code>PatientLevelPrediction</code></a>
+package. This vignette assumes you have read and are comfortable with
+building single patient level prediction models as described in the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf" class="external-link"><code>BuildingPredictiveModels</code>
+vignette</a>.</p>
+<p><strong>We invite you to share your new data splitting functions with
+the OHDSI community through our <a href="http://github.com/OHDSI/PatientLevelPrediction" class="external-link">GitHub
+repository</a>.</strong></p>
 </div>
-<div id="data-splitting-function-code-structure" class="section level1">
-<h1 class="hasAnchor">
-<a href="#data-splitting-function-code-structure" class="anchor"></a>Data Splitting Function Code Structure</h1>
-<p>To make a custom data splitting function that can be used within PatientLevelPrediction you need to write two different functions. The ‘create’ function and the ‘implement’ function.</p>
-<p>The ‘create’ function, e.g., create&lt;DataSplittingFunction&gt;, takes the parameters of the data splitting ‘implement’ function as input, checks these are valid and outputs these as a list of class ‘splitSettings’ with the ‘fun’ attribute specifying the ‘implement’ function to call.</p>
-<p>The ‘implement’ function, e.g., implement&lt;DataSplittingFunction&gt;, must take as input: * population: a data frame that contain rowId (patient identifier), ageYear, gender and outcomeCount (the class labels) * splitSettings - the output of your create&lt;DataSplittingFunction&gt;</p>
-<p>The ‘implement’ function then needs to implement code to assign each rowId in the population to a splitId (&lt;0 means in the train data, 0 means not used and &gt;0 means in the training data with the value defining the cross validation fold).</p>
+<div class="section level2">
+<h2 id="data-splitting-function-code-structure">Data Splitting Function Code Structure<a class="anchor" aria-label="anchor" href="#data-splitting-function-code-structure"></a>
+</h2>
+<p>To make a custom data splitting function that can be used within
+PatientLevelPrediction you need to write two different functions. The
+‘create’ function and the ‘implement’ function.</p>
+<p>The ‘create’ function, e.g., create&lt;DataSplittingFunction&gt;,
+takes the parameters of the data splitting ‘implement’ function as
+input, checks these are valid and outputs these as a list of class
+‘splitSettings’ with the ‘fun’ attribute specifying the ‘implement’
+function to call.</p>
+<p>The ‘implement’ function, e.g.,
+implement&lt;DataSplittingFunction&gt;, must take as input: *
+population: a data frame that contain rowId (patient identifier),
+ageYear, gender and outcomeCount (the class labels) * splitSettings -
+the output of your create&lt;DataSplittingFunction&gt;</p>
+<p>The ‘implement’ function then needs to implement code to assign each
+rowId in the population to a splitId (&lt;0 means in the train data, 0
+means not used and &gt;0 means in the training data with the value
+defining the cross validation fold).</p>
 </div>
-<div id="example" class="section level1">
-<h1 class="hasAnchor">
-<a href="#example" class="anchor"></a>Example</h1>
-<p>Let’s consider the situation where we wish to create a split where females are used to train a model but males are used to evaluate the model.</p>
-<div id="create-function" class="section level2">
-<h2 class="hasAnchor">
-<a href="#create-function" class="anchor"></a>Create function</h2>
-<p>Our gender split function requires a single parameter, the number of folds used in cross validation. Therefore create a function with a single nfold input that returns a list of class ‘splitSettings’ with the ‘fun’ attribute specifying the ‘implement’ function we will use.</p>
+<div class="section level2">
+<h2 id="example">Example<a class="anchor" aria-label="anchor" href="#example"></a>
+</h2>
+<p>Let’s consider the situation where we wish to create a split where
+females are used to train a model but males are used to evaluate the
+model.</p>
+<div class="section level3">
+<h3 id="create-function">Create function<a class="anchor" aria-label="anchor" href="#create-function"></a>
+</h3>
+<p>Our gender split function requires a single parameter, the number of
+folds used in cross validation. Therefore create a function with a
+single nfold input that returns a list of class ‘splitSettings’ with the
+‘fun’ attribute specifying the ‘implement’ function we will use.</p>
 <div class="sourceCode" id="cb1"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">createGenderSplit</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">nfold</span><span class="op">)</span>
-  <span class="op">{</span>
-  
-  <span class="co"># create list of inputs to implement function</span>
-  <span class="va">splitSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>nfold <span class="op">=</span> <span class="va">nfold</span><span class="op">)</span>
-  
-  <span class="co"># specify the function that will implement the sampling</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/attr.html">attr</a></span><span class="op">(</span><span class="va">splitSettings</span>, <span class="st">"fun"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">"implementGenderSplit"</span>
-
-  <span class="co"># make sure the object returned is of class "sampleSettings"</span>
-  <span class="fu"><a href="https://rdrr.io/r/base/class.html">class</a></span><span class="op">(</span><span class="va">splitSettings</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">"splitSettings"</span>
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">splitSettings</span><span class="op">)</span>
-  
-<span class="op">}</span></code></pre></div>
-<p>We now need to create the ‘implement’ function <code>implementGenderSplit()</code></p>
+<code class="sourceCode R"><span><span class="va">createGenderSplit</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">nfold</span><span class="op">)</span></span>
+<span>  <span class="op">{</span></span>
+<span>  </span>
+<span>  <span class="co"># create list of inputs to implement function</span></span>
+<span>  <span class="va">splitSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>nfold <span class="op">=</span> <span class="va">nfold</span><span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="co"># specify the function that will implement the sampling</span></span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/attr.html" class="external-link">attr</a></span><span class="op">(</span><span class="va">splitSettings</span>, <span class="st">"fun"</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">"implementGenderSplit"</span></span>
+<span></span>
+<span>  <span class="co"># make sure the object returned is of class "sampleSettings"</span></span>
+<span>  <span class="fu"><a href="https://rdrr.io/r/base/class.html" class="external-link">class</a></span><span class="op">(</span><span class="va">splitSettings</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">"splitSettings"</span></span>
+<span>  <span class="kw"><a href="https://rdrr.io/r/base/function.html" class="external-link">return</a></span><span class="op">(</span><span class="va">splitSettings</span><span class="op">)</span></span>
+<span>  </span>
+<span><span class="op">}</span></span></code></pre></div>
+<p>We now need to create the ‘implement’ function
+<code>implementGenderSplit()</code></p>
 </div>
-<div id="implement-function" class="section level2">
-<h2 class="hasAnchor">
-<a href="#implement-function" class="anchor"></a>Implement function</h2>
-<p>All ‘implement’ functions for data splitting must take as input the population and the splitSettings (this is the output of the ‘create’ function). They must return a data.frame containing columns: rowId and index.</p>
-<p>The index is used to determine whether the patient (identifed by the rowId) is in the test set (index = -1) or train set (index &gt; 0). In in the train set, the value corresponds to the cross validation fold. For example, if rowId 2 is assigned index 5, then it means the patient with the rowId 2 is used to train the model and is in fold 5.</p>
+<div class="section level3">
+<h3 id="implement-function">Implement function<a class="anchor" aria-label="anchor" href="#implement-function"></a>
+</h3>
+<p>All ‘implement’ functions for data splitting must take as input the
+population and the splitSettings (this is the output of the ‘create’
+function). They must return a data.frame containing columns: rowId and
+index.</p>
+<p>The index is used to determine whether the patient (identifed by the
+rowId) is in the test set (index = -1) or train set (index &gt; 0). In
+in the train set, the value corresponds to the cross validation fold.
+For example, if rowId 2 is assigned index 5, then it means the patient
+with the rowId 2 is used to train the model and is in fold 5.</p>
 <div class="sourceCode" id="cb2"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">implementGenderSplit</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">population</span>, <span class="va">splitSettings</span><span class="op">)</span><span class="op">{</span>
-
-  <span class="co"># find the people who are male:</span>
-  <span class="va">males</span> <span class="op">&lt;-</span> <span class="va">population</span><span class="op">$</span><span class="va">rowId</span><span class="op">[</span><span class="va">population</span><span class="op">$</span><span class="va">gender</span> <span class="op">==</span> <span class="fl">8507</span><span class="op">]</span>
-  <span class="va">females</span> <span class="op">&lt;-</span> <span class="va">population</span><span class="op">$</span><span class="va">rowId</span><span class="op">[</span><span class="va">population</span><span class="op">$</span><span class="va">gender</span> <span class="op">==</span> <span class="fl">8532</span><span class="op">]</span>
-  
-  <span class="va">splitIds</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html">data.frame</a></span><span class="op">(</span>
-    rowId <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="va">males</span>, <span class="va">females</span><span class="op">)</span>,
-    index <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span>
-      <span class="fu"><a href="https://rdrr.io/r/base/rep.html">rep</a></span><span class="op">(</span><span class="op">-</span><span class="fl">1</span>, <span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="va">males</span><span class="op">)</span><span class="op">)</span>,
-      <span class="fu"><a href="https://rdrr.io/r/base/sample.html">sample</a></span><span class="op">(</span><span class="fl">1</span><span class="op">:</span><span class="va">splitSettings</span><span class="op">$</span><span class="va">nfold</span>, <span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="va">females</span><span class="op">)</span>, replace <span class="op">=</span> <span class="cn">T</span><span class="op">)</span>
-    <span class="op">)</span>
-  <span class="op">)</span>
-  
-  <span class="co"># return the updated trainData</span>
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">splitIds</span><span class="op">)</span>
-<span class="op">}</span></code></pre></div>
+<code class="sourceCode R"><span><span class="va">implementGenderSplit</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">population</span>, <span class="va">splitSettings</span><span class="op">)</span><span class="op">{</span></span>
+<span></span>
+<span>  <span class="co"># find the people who are male:</span></span>
+<span>  <span class="va">males</span> <span class="op">&lt;-</span> <span class="va">population</span><span class="op">$</span><span class="va">rowId</span><span class="op">[</span><span class="va">population</span><span class="op">$</span><span class="va">gender</span> <span class="op">==</span> <span class="fl">8507</span><span class="op">]</span></span>
+<span>  <span class="va">females</span> <span class="op">&lt;-</span> <span class="va">population</span><span class="op">$</span><span class="va">rowId</span><span class="op">[</span><span class="va">population</span><span class="op">$</span><span class="va">gender</span> <span class="op">==</span> <span class="fl">8532</span><span class="op">]</span></span>
+<span>  </span>
+<span>  <span class="va">splitIds</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span></span>
+<span>    rowId <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="va">males</span>, <span class="va">females</span><span class="op">)</span>,</span>
+<span>    index <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span></span>
+<span>      <span class="fu"><a href="https://rdrr.io/r/base/rep.html" class="external-link">rep</a></span><span class="op">(</span><span class="op">-</span><span class="fl">1</span>, <span class="fu"><a href="https://rdrr.io/r/base/length.html" class="external-link">length</a></span><span class="op">(</span><span class="va">males</span><span class="op">)</span><span class="op">)</span>,</span>
+<span>      <span class="fu"><a href="https://rdrr.io/r/base/sample.html" class="external-link">sample</a></span><span class="op">(</span><span class="fl">1</span><span class="op">:</span><span class="va">splitSettings</span><span class="op">$</span><span class="va">nfold</span>, <span class="fu"><a href="https://rdrr.io/r/base/length.html" class="external-link">length</a></span><span class="op">(</span><span class="va">females</span><span class="op">)</span>, replace <span class="op">=</span> <span class="cn">T</span><span class="op">)</span></span>
+<span>    <span class="op">)</span></span>
+<span>  <span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="co"># return the updated trainData</span></span>
+<span>  <span class="kw"><a href="https://rdrr.io/r/base/function.html" class="external-link">return</a></span><span class="op">(</span><span class="va">splitIds</span><span class="op">)</span></span>
+<span><span class="op">}</span></span></code></pre></div>
 </div>
 </div>
-<div id="acknowledgments" class="section level1">
-<h1 class="hasAnchor">
-<a href="#acknowledgments" class="anchor"></a>Acknowledgments</h1>
-<p>Considerable work has been dedicated to provide the <code>PatientLevelPrediction</code> package.</p>
+<div class="section level2">
+<h2 id="acknowledgments">Acknowledgments<a class="anchor" aria-label="anchor" href="#acknowledgments"></a>
+</h2>
+<p>Considerable work has been dedicated to provide the
+<code>PatientLevelPrediction</code> package.</p>
 <div class="sourceCode" id="cb3"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/utils/citation.html">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></code></pre></div>
-<pre><code>## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design
-## and implementation of a standardized framework to generate and evaluate
-## patient-level prediction models using observational healthcare data."
-## _Journal of the American Medical Informatics Association_, *25*(8),
-## 969-975. &lt;URL: https://doi.org/10.1093/jamia/ocy032&gt;.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }</code></pre>
-<p><strong>Please reference this paper if you use the PLP Package in your work:</strong></p>
-<p><a href="http://dx.doi.org/10.1093/jamia/ocy032">Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.</a></p>
-<p>This work is supported in part through the National Science Foundation grant IIS 1251151.</p>
+<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/utils/citation.html" class="external-link">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></span></code></pre></div>
+<pre><code><span><span class="co">## </span></span>
+<span><span class="co">## To cite PatientLevelPrediction in publications use:</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">##   Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design</span></span>
+<span><span class="co">##   and implementation of a standardized framework to generate and</span></span>
+<span><span class="co">##   evaluate patient-level prediction models using observational</span></span>
+<span><span class="co">##   healthcare data." _Journal of the American Medical Informatics</span></span>
+<span><span class="co">##   Association_, *25*(8), 969-975.</span></span>
+<span><span class="co">##   &lt;https://doi.org/10.1093/jamia/ocy032&gt;.</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">## A BibTeX entry for LaTeX users is</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">##   @Article{,</span></span>
+<span><span class="co">##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},</span></span>
+<span><span class="co">##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},</span></span>
+<span><span class="co">##     journal = {Journal of the American Medical Informatics Association},</span></span>
+<span><span class="co">##     volume = {25},</span></span>
+<span><span class="co">##     number = {8},</span></span>
+<span><span class="co">##     pages = {969-975},</span></span>
+<span><span class="co">##     year = {2018},</span></span>
+<span><span class="co">##     url = {https://doi.org/10.1093/jamia/ocy032},</span></span>
+<span><span class="co">##   }</span></span></code></pre>
+<p><strong>Please reference this paper if you use the PLP Package in
+your work:</strong></p>
+<p><a href="http://dx.doi.org/10.1093/jamia/ocy032" class="external-link">Reps JM, Schuemie
+MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a
+standardized framework to generate and evaluate patient-level prediction
+models using observational healthcare data. J Am Med Inform Assoc.
+2018;25(8):969-975.</a></p>
+<p>This work is supported in part through the National Science
+Foundation grant IIS 1251151.</p>
 </div>
   </div>
 
@@ -257,11 +312,13 @@ <h1 class="hasAnchor">
 
 
       <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+  <p></p>
+<p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p>
+<p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
       </footer>
@@ -270,5 +327,7 @@ <h1 class="hasAnchor">
   
 
 
+  
+
   </body>
 </html>
diff --git a/docs/articles/AddingCustomSplitting_files/header-attrs-2.11/header-attrs.js b/docs/articles/AddingCustomSplitting_files/header-attrs-2.11/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/AddingCustomSplitting_files/header-attrs-2.11/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/AddingExistingModels.html b/docs/articles/AddingExistingModels.html
deleted file mode 100644
index 6e493db0d..000000000
--- a/docs/articles/AddingExistingModels.html
+++ /dev/null
@@ -1,362 +0,0 @@
-<!DOCTYPE html>
-<!-- Generated by pkgdown: do not edit by hand --><html lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-<title>Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction framework • PatientLevelPrediction</title>
-<!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous">
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous">
-<!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script><!-- sticky kit --><script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script><meta property="og:title" content="Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction framework">
-<meta property="og:description" content="">
-<meta name="twitter:card" content="summary">
-<!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-</head>
-<body>
-    <div class="container template-article">
-      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.1</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-<li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-<li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/AddingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction framework</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CustomPredictionAlgorithms.html">Custom patient-level prediction algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/PlottingLearningCurves.html">Plotting learning curves</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-<ul class="nav navbar-nav navbar-right">
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-</div>
-<!--/.nav-collapse -->
-  </div>
-<!--/.container -->
-</div>
-<!--/.navbar -->
-
-      
-      </header><div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header toc-ignore">
-      <h1>Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction framework</h1>
-                        <h4 class="author">Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek</h4>
-            
-            <h4 class="date">2019-03-04</h4>
-      
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/AddingExistingModels.Rmd"><code>vignettes/AddingExistingModels.Rmd</code></a></small>
-      <div class="hidden name"><code>AddingExistingModels.Rmd</code></div>
-
-    </div>
-
-    
-    
-<!--
-%\VignetteEngine{knitr}
-%\VignetteIndexEntry{Adding existing patient-level predictive models}
--->
-
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
-<p>This vignette describes how you can implement existing logistic regression models in the <code>PatientLevelPrediction</code> framework. This allows you to for example externally validate them at scale in the OHDSI data network.</p>
-<p>As an example we are going to implement the CHADS2 model:</p>
-<p>Gage BF, Waterman AD, Shannon W, Boechler M, Rich MW, Radford MJ. Validation of clinical classification schemes for predicting stroke: results from the National Registry of Atrial Fibrillation. JAMA. 2001 Jun 13;285(22):2864-70</p>
-<p>To implement the model you need to create three tables: the model table, the covariate table, and the intercept table. The model table specifies the modelId (sequence number), the modelCovariateId (sequence number) and the covariateValue (beta for the covariate). The covariate table specifies the mapping between the covariates from the published model and the standard Patient Level Prediction framework covariates, i.e. its maps to a combination of an analysisid and a concept_id (see below). The intercept table specifies per modelId the intercept.</p>
-</div>
-<div id="model-implementation" class="section level1">
-<h1 class="hasAnchor">
-<a href="#model-implementation" class="anchor"></a>Model implementation</h1>
-<div id="define-the-model" class="section level2">
-<h2 class="hasAnchor">
-<a href="#define-the-model" class="anchor"></a>Define the model</h2>
-<p>The CHADS2 is a score based model with:</p>
-<pre><code>##   Points                        Covariate
-## 1      1         Congestive heart failure
-## 2      1                     Hypertension
-## 3      1                  Age &gt;= 75 years
-## 4      1                Diabetes mellitus
-## 5      2 Stroke/transient ischemic attack</code></pre>
-<p>The model table should therefore be defined as:</p>
-<pre><code>##   modelId modelCovariateId covariateValue
-## 1       1                1              1
-## 2       1                2              1
-## 3       1                3              1
-## 4       1                4              1
-## 5       1                5              2</code></pre>
-<p>The covariateTable will then specify what standard covariates need to be included in the model.</p>
-<p>In this case we choose the following Standard SNOMED concept_ids: 319835 for congestive heart failure, 316866 for hypertensive disorder, 201820 for diabetes, and 381591 for cerebrovascular disease. It is allowed to add multiple concept_ids as seperate rows for the same modelCovariateId if concept sets are needed. These concept_ids can be found using the vocabulary search in ATLAS.</p>
-<p>The Patient Level Prediction standard covariates are of the form: conceptid*1000 + analysisid. The analysisid specifies the domain of the covariate and its lookback window. Examples can be found here: <a href="https://github.com/OHDSI/FeatureExtraction/blob/master/inst/csv/PrespecAnalyses.csv" class="uri">https://github.com/OHDSI/FeatureExtraction/blob/master/inst/csv/PrespecAnalyses.csv</a></p>
-<p>Our example of CHADS2 uses agegroup and conditions in the full history. Therefore we need to define the standard covariates using the FeatureExtraction::createCovariateSettings as follows:</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/library">library</a></span>(PatientLevelPrediction)
-covSet &lt;-<span class="st"> </span>FeatureExtraction<span class="op">::</span><span class="kw"><a href="https://www.rdocumentation.org/packages/FeatureExtraction/topics/createCovariateSettings">createCovariateSettings</a></span>(<span class="dt">useDemographicsAgeGroup =</span> T,                             
-                                                     <span class="dt">useConditionOccurrenceLongTerm =</span> T,
-                                                     <span class="dt">includedCovariateIds =</span> <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/c">c</a></span>(),
-                                                     <span class="dt">longTermStartDays =</span> <span class="op">-</span><span class="dv">9999</span>, 
-                                                     <span class="dt">endDays =</span> <span class="dv">0</span>)</code></pre></div>
-<p>In the above code we used the useConditionOccurrenceLongTerm (these have an analysis id of 102) and we defined the longTermStartDays to be -9999 days relative to index (so we get the full history). We include the index date in our lookback period by specifying endDays = 0. The includeCovariateIds is set to 0, but this will be updated when you run the next code to pick out the standard covariates of interest. As we picked analysis id 102, the standard covariate for anytime prior congestive heart failure is 319835102, the same logic follows for the other conditions, so the covariate table will be:</p>
-<pre><code>##   modelCovariateId covariateId
-## 1                1   319835102
-## 2                2   316866102
-## 3                3       15003
-## 4                3       16003
-## 5                3       17003
-## 6                3       18003
-## 7                3       19003
-## 8                4   201820102
-## 9                5   381591102</code></pre>
-<p>modelCovariateId 3 was age&gt;= 75, as the standard covariate age groups are in 5 year groups, we needed to add the age groups 75-80, 80-85, 85-90, 90-95 and 95-100, these correspond to the covaraiteIds 15003, 16003, 17003, 18003 and 19003 respectively.</p>
-<p>To create the tables in R for CHADS2 you need to make the following dataframes:</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">model_table &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/data.frame">data.frame</a></span>(<span class="dt">modelId =</span> <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/c">c</a></span>(<span class="dv">1</span>,<span class="dv">1</span>,<span class="dv">1</span>,<span class="dv">1</span>,<span class="dv">1</span>),
-                          <span class="dt">modelCovariateId =</span> <span class="dv">1</span><span class="op">:</span><span class="dv">5</span>, 
-                          <span class="dt">coefficientValue =</span> <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/c">c</a></span>(<span class="dv">1</span>, <span class="dv">1</span>, <span class="dv">1</span>, <span class="dv">1</span>, <span class="dv">2</span>)
-                          )
-
-covariate_table &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/data.frame">data.frame</a></span>(<span class="dt">modelCovariateId =</span> <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/c">c</a></span>(<span class="dv">1</span>,<span class="dv">2</span>,<span class="dv">3</span>,<span class="dv">3</span>,<span class="dv">3</span>,<span class="dv">3</span>,<span class="dv">3</span>,<span class="dv">4</span>,<span class="dv">5</span>),
-                              <span class="dt">covariateId =</span> <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/c">c</a></span>(<span class="dv">319835102</span>, <span class="dv">316866102</span>, 
-                                            <span class="dv">15003</span>, <span class="dv">16003</span>, <span class="dv">17003</span>, <span class="dv">18003</span>, <span class="dv">19003</span>, 
-                                            <span class="dv">201820102</span>, <span class="dv">381591102</span>)
-                              )
-
-interceptTable &lt;-<span class="st">  </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/data.frame">data.frame</a></span>(<span class="dt">modelId =</span> <span class="dv">1</span>, 
-                              <span class="dt">interceptValue =</span> <span class="dv">0</span>)</code></pre></div>
-</div>
-<div id="create-the-model" class="section level2">
-<h2 class="hasAnchor">
-<a href="#create-the-model" class="anchor"></a>Create the model</h2>
-<p>Now you have everything in place actually create the existing model. First specify the current environment as executing createExistingModelSql creates two functions for running the existing model into the specificed environment. Next a few additional settings are needed: as some models require an intercept, there is an option for this (set it to 0 if an intercept isn’t needed), also the type specifies the final mapping (either logistic or linear/score), in our example we are calculating a score. We finally need to specify the analysisId for the newly created CHADS2 covariate.</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">e &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/environment">environment</a></span>()
-PatientLevelPrediction<span class="op">::</span><span class="kw"><a href="https://www.rdocumentation.org/packages/PatientLevelPrediction/topics/createExistingModelSql">createExistingModelSql</a></span>(<span class="dt">modelTable =</span> model_table, 
-                       <span class="dt">modelNames =</span> <span class="st">'CHADS2'</span>, 
-                       <span class="dt">interceptTable =</span> <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/data.frame">data.frame</a></span>(<span class="dt">modelId =</span> <span class="dv">1</span>, <span class="dt">interceptValue =</span> <span class="dv">0</span>),
-                       <span class="dt">covariateTable =</span> covariate_table, 
-                       <span class="dt">type =</span> <span class="st">'score'</span>,
-                       <span class="dt">analysisId =</span> <span class="dv">112</span>, <span class="dt">covariateSettings =</span> covSettings, <span class="dt">e =</span> e)</code></pre></div>
-<p>Once run you will find two new functions in your environment:</p>
-<ul>
-<li>createExistingmodelsCovariateSettings()</li>
-<li>getExistingmodelsCovariateSettings()</li>
-</ul>
-</div>
-</div>
-<div id="run-the-model" class="section level1">
-<h1 class="hasAnchor">
-<a href="#run-the-model" class="anchor"></a>Run the model</h1>
-<p>Now you can use the functions you previously created to extract the existing model risk scores for a target population:</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">plpData &lt;-<span class="st"> </span>PatientLevelPrediction<span class="op">::</span><span class="kw"><a href="https://www.rdocumentation.org/packages/PatientLevelPrediction/topics/getPlpData">getPlpData</a></span>(connectionDetails, 
-                      <span class="dt">cdmDatabaseSchema =</span> <span class="st">'databasename.dbo'</span>,
-                      <span class="dt">cohortId =</span> <span class="dv">1</span>,
-                      <span class="dt">outcomeIds =</span> <span class="dv">2</span>, 
-                      <span class="dt">cohortDatabaseSchema =</span> <span class="st">'databasename.dbo'</span>, 
-                      <span class="dt">cohortTable =</span>  <span class="st">'cohort'</span> , 
-                      <span class="dt">outcomeDatabaseSchema =</span> <span class="st">'databasename.dbo'</span>, 
-                      <span class="dt">outcomeTable =</span> <span class="st">'cohort'</span>, 
-                      <span class="dt">covariateSettings =</span>  <span class="kw">createExistingmodelsCovariateSettings</span>(),
-                      <span class="dt">sampleSize =</span> <span class="dv">20000</span>
-                      )</code></pre></div>
-<p>To implement and evaluate an existing model you can use the function:</p>
-<p><code><a href="../reference/evaluateExistingModel.html">PatientLevelPrediction::evaluateExistingModel()</a></code></p>
-<p>with the following parameters:</p>
-<ul>
-<li>modelTable - a data.frame containing the columns: modelId, covariateId and coefficientValue</li>
-<li>covariateTable - a data.frame containing the columns: covariateId and standardCovariateId - this provides a set of standardCovariateId to define each model covariate.</li>
-<li>interceptTable - a data.frame containing the columns modelId and interceptValue or NULL if the model doesn’t have an intercept (equal to zero).</li>
-<li>type - the type of model (either: score or logistic)</li>
-<li>covariateSettings - this is used to determine the startDay and endDay for the standard covariates</li>
-<li>customCovariates - a data.frame with the covariateId and sql to generate the covariate value.</li>
-<li>riskWindowStart - the time at risk starts at target cohort start date + riskWindowStart</li>
-<li>addExposureDaysToEnd - if true then the time at risk window ends a the cohort end date + riskWindowEnd rather than cohort start date + riskWindowEnd</li>
-<li>riskWindowEnd - the time at risk ends at target cohort start/end date + riskWindowStart</li>
-<li>requireTimeAtRisk - whether to add a constraint to the number of days observed during the time at risk period in including people into the study</li>
-<li>minTimeAtRisk - the minimum number of days observation during the time at risk a target population person needs to be included</li>
-<li>includeAllOutcomes - Include outcomes even if they do not satisfy the minTimeAtRisk? (useful if the outcome is associated to death or rare)</li>
-<li>removeSubjectsWithPriorOutcome - remove target population people who have the outcome prior to the time at tisk period?</li>
-<li>connectionDetails - the connection to the CDM database</li>
-</ul>
-<p>Finally you need to add the settings for downloading the new data:</p>
-<ul>
-<li>cdmDatabaseSchema</li>
-<li>cohortDatabaseSchema</li>
-<li>cohortTable</li>
-<li>cohortId</li>
-<li>outcomeDatabaseSchema</li>
-<li>outcomeTable</li>
-<li>outcomeId</li>
-<li>oracleTempSchema</li>
-</ul>
-<p>To run the external validation of an existing model where the target population are those in the cohort table with id 1 and the outcome is those in the cohort table with id 2 and we are looking to predict first time occurrance of the outcome 1 day to 365 days after the target cohort start date (asusming you have the modelTable, covariateTable and interceptTable in the format explained above):</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="co"># if the existing model uses gender and condition groups looking back 200 days:</span>
-covSet &lt;-<span class="st"> </span>FeatureExtraction<span class="op">::</span><span class="kw"><a href="https://www.rdocumentation.org/packages/FeatureExtraction/topics/createCovariateSettings">createCovariateSettings</a></span>(<span class="dt">useDemographicsGender =</span> T,
-                                                     <span class="dt">useConditionGroupEraMediumTerm =</span> T, 
-                                                     <span class="dt">mediumTermStartDays =</span> <span class="op">-</span><span class="dv">200</span>)
-result &lt;-<span class="st"> </span><span class="kw"><a href="../reference/evaluateExistingModel.html">evaluateExistingModel</a></span>(<span class="dt">modelTable =</span> modelTable,
-                                <span class="dt">covariateTable =</span> covariateTable,
-                                <span class="dt">interceptTable =</span> <span class="ot">NULL</span>,
-                                <span class="dt">type =</span> <span class="st">'score'</span>, 
-                                <span class="dt">covariateSettings =</span>  covSet,
-                                <span class="dt">riskWindowStart =</span> <span class="dv">1</span>, 
-                                <span class="dt">addExposureDaysToEnd =</span> F, 
-                                <span class="dt">riskWindowEnd =</span> <span class="dv">365</span>, 
-                                <span class="dt">requireTimeAtRisk =</span> T, 
-                                <span class="dt">minTimeAtRisk =</span> <span class="dv">364</span>, 
-                                <span class="dt">includeAllOutcomes =</span> T, 
-                                <span class="dt">removeSubjectsWithPriorOutcome =</span> T, 
-                                <span class="dt">connectionDetails =</span> connectionDetails, 
-                                <span class="dt">cdmDatabaseSchema =</span> <span class="st">'databasename.dbo'</span>,
-                                <span class="dt">cohortId =</span> <span class="dv">1</span>,
-                                <span class="dt">outcomeId =</span> <span class="dv">2</span>, 
-                                <span class="dt">cohortDatabaseSchema =</span> <span class="st">'databasename.dbo'</span>, 
-                                <span class="dt">cohortTable =</span>  <span class="st">'cohort'</span> , 
-                                <span class="dt">outcomeDatabaseSchema =</span> <span class="st">'databasename.dbo'</span>, 
-                                <span class="dt">outcomeTable =</span> <span class="st">'cohort'</span>
-                      )</code></pre></div>
-</div>
-<div id="acknowledgments" class="section level1">
-<h1 class="hasAnchor">
-<a href="#acknowledgments" class="anchor"></a>Acknowledgments</h1>
-<p>Considerable work has been dedicated to provide the <code>PatientLevelPrediction</code> package.</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw"><a href="https://www.rdocumentation.org/packages/utils/topics/citation">citation</a></span>(<span class="st">"PatientLevelPrediction"</span>)</code></pre></div>
-<pre><code>## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018).
-## "Design and implementation of a standardized framework to generate
-## and evaluate patient-level prediction models using observational
-## healthcare data." _Journal of the American Medical Informatics
-## Association_, *25*(8), 969-975. &lt;URL:
-## https://doi.org/10.1093/jamia/ocy032&gt;.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }</code></pre>
-<p>This work is supported in part through the National Science Foundation grant IIS 1251151.</p>
-</div>
-  </div>
-
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-        <div id="tocnav">
-      <h2 class="hasAnchor">
-<a href="#tocnav" class="anchor"></a>Contents</h2>
-      <ul class="nav nav-pills nav-stacked">
-<li><a href="#introduction">Introduction</a></li>
-      <li>
-<a href="#model-implementation">Model implementation</a><ul class="nav nav-pills nav-stacked">
-<li><a href="#define-the-model">Define the model</a></li>
-      <li><a href="#create-the-model">Create the model</a></li>
-      </ul>
-</li>
-      <li><a href="#run-the-model">Run the model</a></li>
-      <li><a href="#acknowledgments">Acknowledgments</a></li>
-      </ul>
-</div>
-      </div>
-
-</div>
-
-
-      <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-</div>
-
-  
-
-  </body>
-</html>
diff --git a/docs/articles/BenchmarkTasks.html b/docs/articles/BenchmarkTasks.html
new file mode 100644
index 000000000..423651309
--- /dev/null
+++ b/docs/articles/BenchmarkTasks.html
@@ -0,0 +1,343 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<meta charset="utf-8">
+<meta http-equiv="X-UA-Compatible" content="IE=edge">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>Benchmark Tasks • PatientLevelPrediction</title>
+<!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous">
+<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css">
+<script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous">
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous">
+<!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet">
+<script src="../pkgdown.js"></script><meta property="og:title" content="Benchmark Tasks">
+<meta property="og:description" content="PatientLevelPrediction">
+<!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]-->
+</head>
+<body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-article">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav">
+<li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu">
+<li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul>
+</li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul>
+<ul class="nav navbar-nav navbar-right">
+<li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul>
+</div>
+<!--/.nav-collapse -->
+  </div>
+<!--/.container -->
+</div>
+<!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header toc-ignore">
+      <h1 data-toc-skip>Benchmark Tasks</h1>
+                        <h4 data-toc-skip class="author">Jenna Reps,
+Ross Williams, Peter R. Rijnbeek</h4>
+            
+            <h4 data-toc-skip class="date">2024-09-09</h4>
+      
+      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/vignettes/BenchmarkTasks.Rmd" class="external-link"><code>vignettes/BenchmarkTasks.Rmd</code></a></small>
+      <div class="hidden name"><code>BenchmarkTasks.Rmd</code></div>
+
+    </div>
+
+    
+    
+<!--
+%\VignetteEngine{knitr::knitr}
+%\VignetteIndexEntry{Benchmark Tasks}
+-->
+<div class="section level2">
+<h2 id="benchmark-tasks-for-large-scale-empirical-analyses">Benchmark Tasks For Large-Scale Empirical Analyses<a class="anchor" aria-label="anchor" href="#benchmark-tasks-for-large-scale-empirical-analyses"></a>
+</h2>
+<p>Here we provide a set of diverse prediction tasks that can be used
+when evaluating the impact of the model design choice when developing
+models using observational data.</p>
+<table style="width:100%;" class="table">
+<colgroup>
+<col width="71%">
+<col width="22%">
+<col width="4%">
+<col width="1%">
+</colgroup>
+<thead><tr class="header">
+<th>Target Cohort (index)</th>
+<th>Outcome</th>
+<th>Time-at-risk</th>
+<th>Link</th>
+</tr></thead>
+<tbody>
+<tr class="odd">
+<td>Patients with an outpatient visit in 2017 with no prior cancer
+(first visit in 2017)</td>
+<td>Lung cancer</td>
+<td>1 day - 3 years after index</td>
+<td></td>
+</tr>
+<tr class="even">
+<td>Patients newly diagnosed with major depressive disorder (date of
+first record)</td>
+<td>Bipolar</td>
+<td>1 day - 365 day after index</td>
+<td></td>
+</tr>
+<tr class="odd">
+<td>Patients with an outpatient visit in 2019</td>
+<td>Dementia</td>
+<td>1 day - 3 years after index</td>
+<td></td>
+</tr>
+<tr class="even">
+<td>Patients with an outpatient visit and a positive COVID test</td>
+<td>Hospitalization with pneumonia</td>
+<td>1 day - 30 days after index</td>
+<td></td>
+</tr>
+<tr class="odd">
+<td>Patients with an outpatient visit and a positive COVID test</td>
+<td>Hospitalization with pneumonia that required intensive services
+(ventilation, intubation, tracheotomy, or extracorporeal membrane
+oxygenation) or death</td>
+<td>1 day - 30 days after index</td>
+<td></td>
+</tr>
+<tr class="even">
+<td>Patients with an outpatient visit and a positive COVID test</td>
+<td>Death</td>
+<td>1 day - 30 days after index</td>
+<td></td>
+</tr>
+<tr class="odd">
+<td>Patients with T2DM who were treated with metformin and who became
+new adult users of one of sulfonylureas, thiazolidinediones, dipeptidyl
+peptidase-4 inhibitors, glucagon-like peptide-1 receptor agonists, or
+sodium-glucose co-transporter-2 inhibitors (date of secondary drug).
+Patients with HF or patients treated with insulin on or prior to the
+index date were excluded from the analysis. Patients were required to
+have been enrolled for at least 365 days before cohort entry.</td>
+<td>Heart Failure</td>
+<td>1 to 365 days</td>
+<td></td>
+</tr>
+<tr class="even">
+<td>Patients newly diagnosed with atrial fibrilation (date of initial
+afib record)</td>
+<td>Ischemic stroke</td>
+<td>1 to 365 days</td>
+<td></td>
+</tr>
+<tr class="odd">
+<td>Patients undergoing elective major non-cardiac surgery (date of
+surgery). Patients were required to have been enrolled for at least 365
+days before cohort entry.</td>
+<td>Earliest of AMI cardiac arrest or death (MACE)</td>
+<td>O to 30 days</td>
+<td></td>
+</tr>
+<tr class="even">
+<td>Patients starting intravitreal Anti-VEGF (date of
+administration)</td>
+<td>Kidney Failure</td>
+<td>1 to 365 days</td>
+<td></td>
+</tr>
+<tr class="odd">
+<td>Pregnancy women (start of pregnancy)</td>
+<td>Preeclampsia</td>
+<td>During pregnancy</td>
+<td></td>
+</tr>
+<tr class="even">
+<td>Pregnancy women (start of pregnancy)</td>
+<td>Still birth</td>
+<td>During pregnancy</td>
+<td></td>
+</tr>
+<tr class="odd">
+<td>Patients with COPD (first record)</td>
+<td>Cardiovascular event and death</td>
+<td>1-30 days and 1-90 days</td>
+<td></td>
+</tr>
+<tr class="even">
+<td>Patients starting menopause (first record)</td>
+<td>Depression</td>
+<td>1 day - 3-years</td>
+<td></td>
+</tr>
+<tr class="odd">
+<td>Patients with anemia (date of first anemia record)</td>
+<td>Colorectal cancer</td>
+<td>1 day - 1-year</td>
+<td></td>
+</tr>
+<tr class="even">
+<td>Patients with quadriplegia (date of first quadriplegia record)</td>
+<td>Death</td>
+<td>1 day - 1-year</td>
+<td></td>
+</tr>
+<tr class="odd">
+<td>Patient undergoing</td>
+<td></td>
+<td></td>
+<td></td>
+</tr>
+<tr class="even">
+<td></td>
+<td></td>
+<td></td>
+<td></td>
+</tr>
+</tbody>
+</table>
+</div>
+  </div>
+
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+
+        <nav id="toc" data-toggle="toc"><h2 data-toc-skip>Contents</h2>
+    </nav>
+</div>
+
+</div>
+
+
+
+      <footer><div class="copyright">
+  <p></p>
+<p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p>
+<p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer>
+</div>
+
+  
+
+
+  
+
+  </body>
+</html>
diff --git a/docs/articles/BestPractices.html b/docs/articles/BestPractices.html
index c0490c754..49fc4e02f 100644
--- a/docs/articles/BestPractices.html
+++ b/docs/articles/BestPractices.html
@@ -19,6 +19,8 @@
 <![endif]-->
 </head>
 <body data-spy="scroll" data-target="#toc">
+    
+
     <div class="container template-article">
       <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
@@ -31,7 +33,7 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
@@ -53,7 +55,7 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
@@ -72,10 +74,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -84,13 +86,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -101,21 +106,30 @@
   </ul>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
       </ul>
 <ul class="nav navbar-nav navbar-right">
 <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
@@ -130,15 +144,16 @@
 
       
 
-      </header><script src="BestPractices_files/header-attrs-2.11/header-attrs.js"></script><div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header toc-ignore">
       <h1 data-toc-skip>Best Practice Research</h1>
-                        <h4 class="author">Jenna Reps, Peter R. Rijnbeek</h4>
+                        <h4 data-toc-skip class="author">Jenna Reps,
+Peter R. Rijnbeek</h4>
             
-            <h4 class="date">2022-03-09</h4>
+            <h4 data-toc-skip class="date">2024-09-09</h4>
       
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/BestPractices.rmd"><code>vignettes/BestPractices.rmd</code></a></small>
+      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/vignettes/BestPractices.rmd" class="external-link"><code>vignettes/BestPractices.rmd</code></a></small>
       <div class="hidden name"><code>BestPractices.rmd</code></div>
 
     </div>
@@ -149,9 +164,10 @@ <h4 class="date">2022-03-09</h4>
 %\VignetteEngine{knitr}
 %\VignetteIndexEntry{Best Practices}
 -->
-<div id="best-practice-publications-using-the-ohdsi-patientlevelprediction-framework" class="section level2">
-<h2 class="hasAnchor">
-<a href="#best-practice-publications-using-the-ohdsi-patientlevelprediction-framework" class="anchor"></a>Best practice publications using the OHDSI PatientLevelPrediction framework</h2>
+<div class="section level2">
+<h2 id="best-practice-publications-using-the-ohdsi-patientlevelprediction-framework">Best practice publications using the OHDSI PatientLevelPrediction
+framework<a class="anchor" aria-label="anchor" href="#best-practice-publications-using-the-ohdsi-patientlevelprediction-framework"></a>
+</h2>
 <table class="table">
 <tr>
 <th>
@@ -183,7 +199,8 @@ <h2 class="hasAnchor">
 Comparison of cohort vs case-control design
 </td>
 <td>
-<a href="https://journalofbigdata.springeropen.com/articles/10.1186/s40537-021-00501-2">Journal of Big Data</a>
+<a href="https://journalofbigdata.springeropen.com/articles/10.1186/s40537-021-00501-2" class="external-link">Journal
+of Big Data</a>
 </td>
 </tr>
 <tr>
@@ -194,7 +211,8 @@ <h2 class="hasAnchor">
 Addressing loss to follow-up (right censoring)
 </td>
 <td>
-<a href="https://link.springer.com/article/10.1186/s12911-021-01408-x">BMC medical informatics and decision makingk</a>
+<a href="https://link.springer.com/article/10.1186/s12911-021-01408-x" class="external-link">BMC
+medical informatics and decision makingk</a>
 </td>
 </tr>
 <tr>
@@ -205,7 +223,8 @@ <h2 class="hasAnchor">
 Investigating how to address left censoring in features construction
 </td>
 <td>
-<a href="https://bmcmedresmethodol.biomedcentral.com/articles/10.1186/s12874-021-01370-2">BMC Medical Research Methodology</a>
+<a href="https://bmcmedresmethodol.biomedcentral.com/articles/10.1186/s12874-021-01370-2" class="external-link">BMC
+Medical Research Methodology</a>
 </td>
 </tr>
 <tr>
@@ -216,7 +235,8 @@ <h2 class="hasAnchor">
 Impact of over/under-sampling
 </td>
 <td>
-Study being developed
+<a href="https://link.springer.com/article/10.1186/s40537-023-00857-7" class="external-link">
+Journal of big data</a>
 </td>
 </tr>
 <tr>
@@ -238,7 +258,8 @@ <h2 class="hasAnchor">
 How much data do we need for prediction - Learning curves at scale
 </td>
 <td>
-<a href="https://arxiv.org/abs/2008.07361">Preprint link</a>
+<a href="https://www.sciencedirect.com/science/article/pii/S1386505622000764" class="external-link">International
+Journal of Medical Informatics </a>
 </td>
 </tr>
 <tr>
@@ -249,7 +270,7 @@ <h2 class="hasAnchor">
 What impact does test/train/validation design have on model performance
 </td>
 <td>
-<a href="https://bmjopen.bmj.com/content/11/12/e050146">BMJ Open </a>
+<a href="https://bmjopen.bmj.com/content/11/12/e050146" class="external-link">BMJ Open </a>
 </td>
 </tr>
 <tr>
@@ -260,7 +281,7 @@ <h2 class="hasAnchor">
 What is the impact of the classifier
 </td>
 <td>
-<a href="https://academic.oup.com/jamia/article/25/8/969/4989437?login=true">JAMIA</a>
+<a href="https://academic.oup.com/jamia/article/25/8/969/4989437?login=true" class="external-link">JAMIA</a>
 </td>
 </tr>
 <tr>
@@ -268,7 +289,8 @@ <h2 class="hasAnchor">
 Model development
 </td>
 <td>
-Can we find hyper-parameter combinations per classifier that consistently lead to good performing models when using claims/EHR data?
+Can we find hyper-parameter combinations per classifier that
+consistently lead to good performing models when using claims/EHR data?
 </td>
 <td>
 Study needs to be done
@@ -279,10 +301,73 @@ <h2 class="hasAnchor">
 Model development
 </td>
 <td>
-Can we use ensembles to combine models developed using different databases to improve models transportability?
+Can we use ensembles to combine different algorithm models within a
+database to improve models transportability?
 </td>
 <td>
-<a href=""> Paper under review at BMC </a>
+<a href="https://ebooks.iospress.nl/doi/10.3233/SHTI230080" class="external-link"> Caring is
+Sharing–Exploiting the Value in Data for Health and Innovation </a>
+</td>
+</tr>
+<tr>
+<td>
+Model development
+</td>
+<td>
+Can we use ensembles to combine models developed using different
+databases to improve models transportability?
+</td>
+<td>
+<a href="https://link.springer.com/article/10.1186/s12911-022-01879-6" class="external-link">
+BMC Medical Informatics and Decision Making </a>
+</td>
+</tr>
+<tr>
+<td>
+Model development
+</td>
+<td>
+Impact of regularization method
+</td>
+<td>
+<a href="https://academic.oup.com/jamia/article/31/7/1514/7676584" class="external-link">
+JAMIA </a>
+</td>
+</tr>
+<tr>
+<td>
+Evaluation
+</td>
+<td>
+Why prediction is not suitable for risk factor identification
+</td>
+<td>
+<a href="https://proceedings.mlr.press/v182/markus22a.html" class="external-link"> Machine
+Learning for Healthcare Conference </a>
+</td>
+</tr>
+<tr>
+<td>
+Evaluation
+</td>
+<td>
+Iterative pairwise external validation to put validation into context
+</td>
+<td>
+<a href="https://link.springer.com/article/10.1007/s40264-022-01161-8" class="external-link">
+Drug Safety </a>
+</td>
+</tr>
+<tr>
+<td>
+Evaluation
+</td>
+<td>
+A novel method to estimate external validation using aggregate
+statistics
+</td>
+<td>
+<a href=""> Study under review </a>
 </td>
 </tr>
 <tr>
@@ -293,7 +378,8 @@ <h2 class="hasAnchor">
 How should we present model performance? (e.g., new visualizations)
 </td>
 <td>
-<a href="https://academic.oup.com/jamiaopen/article/4/1/ooab017/6168493?searchresult=1">JAMIA Open</a>
+<a href="https://academic.oup.com/jamiaopen/article/4/1/ooab017/6168493?searchresult=1" class="external-link">JAMIA
+Open</a>
 </td>
 </tr>
 <tr>
@@ -301,7 +387,8 @@ <h2 class="hasAnchor">
 Evaluation
 </td>
 <td>
-How to interpret external validation performance (can we figure out why the performance drops or stays consistent)?
+How to interpret external validation performance (can we figure out why
+the performance drops or stays consistent)?
 </td>
 <td>
 Study needs to be done
@@ -326,7 +413,8 @@ <h2 class="hasAnchor">
 Is there a way to automatically simplify models?
 </td>
 <td>
-<a href="https://ohdsi-studies.github.io/FeatureSelectionComparison/docs/Protocol.html">Study protocol under development </a>
+<a href="https://ohdsi-studies.github.io/FeatureSelectionComparison/docs/Protocol.html" class="external-link">Study
+protocol under development </a>
 </td>
 </tr>
 </table>
@@ -344,11 +432,13 @@ <h2 class="hasAnchor">
 
 
       <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+  <p></p>
+<p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p>
+<p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
       </footer>
@@ -357,5 +447,7 @@ <h2 class="hasAnchor">
   
 
 
+  
+
   </body>
 </html>
diff --git a/docs/articles/BestPractices_files/header-attrs-2.11/header-attrs.js b/docs/articles/BestPractices_files/header-attrs-2.11/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/BestPractices_files/header-attrs-2.11/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/BestPractices_files/header-attrs-2.7/header-attrs.js b/docs/articles/BestPractices_files/header-attrs-2.7/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/BestPractices_files/header-attrs-2.7/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/BuildingDeepLearningModels.html b/docs/articles/BuildingDeepLearningModels.html
deleted file mode 100644
index 062ff4614..000000000
--- a/docs/articles/BuildingDeepLearningModels.html
+++ /dev/null
@@ -1,597 +0,0 @@
-<!DOCTYPE html>
-<!-- Generated by pkgdown: do not edit by hand --><html lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-<title>Building Deep Learning Models • PatientLevelPrediction</title>
-<!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous">
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous">
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous">
-<!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script><meta property="og:title" content="Building Deep Learning Models">
-<meta property="og:description" content="PatientLevelPrediction">
-<!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-</head>
-<body data-spy="scroll" data-target="#toc">
-    <div class="container template-article">
-      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-<li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-<li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-<ul class="nav navbar-nav navbar-right">
-<li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-</div>
-<!--/.nav-collapse -->
-  </div>
-<!--/.container -->
-</div>
-<!--/.navbar -->
-
-      
-
-      </header><script src="BuildingDeepLearningModels_files/header-attrs-2.7/header-attrs.js"></script><div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header toc-ignore">
-      <h1 data-toc-skip>Building Deep Learning Models</h1>
-                        <h4 class="author">Peter R. Rijnbeek, Seng Chan You, Xiaoyong Pan, Jenna Reps</h4>
-            
-            <h4 class="date">2021-09-29</h4>
-      
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/BuildingDeepLearningModels.Rmd"><code>vignettes/BuildingDeepLearningModels.Rmd</code></a></small>
-      <div class="hidden name"><code>BuildingDeepLearningModels.Rmd</code></div>
-
-    </div>
-
-    
-    
-<!--
-%\VignetteEngine{knitr}
-%\VignetteIndexEntry{Building Deep Learning Models}
--->
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
-<p>Electronic Health Records (EHR) data is high dimensional, heterogeneous, and sparse, which makes predictive modelling a challenge. In the early days, the machine learning community mainly focused on algorithm development, currently there is a shift to more powerful feature engineering. Deep Learning models are widely used to automatically learn high-level feature representations from the data, and have achieved remarkable results in image processing, speech recognition and computational biology. Recently, interesting results have been shown using EHRs, but more extensive research is needed to assess the power of Deep Learning in this domain.</p>
-<p>This vignette describes how you can use the Observational Health Data Sciences and Informatics (OHDSI) <a href="http://github.com/OHDSI/PatientLevelPrediction"><code>PatientLevelPrediction</code></a> package to build Deep Learning models. This vignette assumes you have read and are comfortable with building patient level prediction models as described in the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf"><code>BuildingPredictiveModels</code> vignette</a>. Furthermore, this vignette assumes you are familiar with Deep Learning methods.</p>
-</div>
-<div id="background" class="section level1">
-<h1 class="hasAnchor">
-<a href="#background" class="anchor"></a>Background</h1>
-<p>Deep Learning models are build by stacking an often large number of neural network layers that perform feature engineering steps, e.g embedding, and are collapsed in a final softmax layer (basically a logistic regression layer). These algorithms need a lot of data to converge to a good representation, but currently the sizes of the EHR databases are growing fast which would make Deep Learning an interesting approach to test within OHDSI’s <a href="https://academic.oup.com/jamia/article/25/8/969/4989437">Patient-Level Prediction Framework</a>. The current implementation allows us to perform research at scale on the value and limitations of Deep Learning using EHR data. For relatively small Target and Outcome cohorts, Deep Learning is most probably not the best choice.</p>
-<p>Most current Deep Learning research is performed in python and we have developed a pipeline to interact with python. Multiple Deep Learning backends have been developed, e.g. Tensorflow, PyTorch, Keras (recently also available in R) etc. In the package we have implemented interaction with Keras in R and PyTorch in Python but we invite the community to add other backends.</p>
-<p>Many network architectures have recently been proposed and we have implemented a number of them, however, this list will grow in the near future. It is important to understand that some of these architectures require a 2D data matrix, i.e. |patient|x|feature|, and others use a 3D data matrix |patient|x|feature|x|time|. The <a href="www.github.com%5Cohdsi%5CFeatureExtraction">FeatureExtraction Package</a> has been extended to enable the extraction of both data formats as will be described with examples below.</p>
-<p>Note that training Deep Learning models is computationally intensive, our implementation therefore supports both GPU and CPU. It will automatically check whether there is GPU or not in your computer. A GPU is highly recommended for Deep Learning!</p>
-</div>
-<div id="non-temporal-architectures" class="section level1">
-<h1 class="hasAnchor">
-<a href="#non-temporal-architectures" class="anchor"></a>Non-Temporal Architectures</h1>
-<p>We implemented the following non-temporal (2D data matrix) architectures using PyTorch:</p>
-<pre><code>1) Logistics regression (LRTorch)
-   A simple softmax layer with l2 regularization
-
-2) Feed forward network (MLPTorch) 
-   Supports multilayer perceptron (mlp_type = MLP) and 
-   Self-Normalizing Neural Networks (mlp_type = SNN)
-   Reference: https://arxiv.org/abs/1706.02515</code></pre>
-<p>For the above two methods, we implemented support for a stacked autoencoder and a variational autoencoder to reduce the feature dimension as a first step. These autoencoders learn efficient data encodings in an unsupervised manner by stacking multiple layers in a neural network. Compared to the standard implementations of LR and MLP these implementations can use the GPU power to speed up the gradient descent approach in the back propagation to optimize the weights of the classifier.</p>
-<p>Table 1: Non-Temporal Deep Learning Models Hyper-Parameters</p>
-<table class="table">
-<colgroup>
-<col width="11%">
-<col width="37%">
-<col width="51%">
-</colgroup>
-<thead><tr class="header">
-<th>Name</th>
-<th>Description</th>
-<th>Hyper-parameters</th>
-</tr></thead>
-<tbody>
-<tr class="odd">
-<td>LRTorch</td>
-<td>Logistic Regression Model</td>
-<td>w_decay (l2 regularization), epochs (number of epochs), class_weight (0 = inverse ratio between number of positive and negative examples, -1 = focal loss (<a href="https://arxiv.org/abs/1708.02002" class="uri">https://arxiv.org/abs/1708.02002</a>), or other), autoencoder (apply stacked autoencoder?, vae (apply variational autoencoder)</td>
-</tr>
-<tr class="even">
-<td>MLPTorch</td>
-<td>Multi-Layer Perceptron Model</td>
-<td>mlp_type (MLP = default, SNN = self-normalizing neural network), size (number of hidden nodes), w_decay (l2 regularization), epochs (number of epochs), class_weight(0 = inverse ratio between number of positive and negative examples, -1 = focal loss, or other), autoencoder (apply stacked autoencoder), vae (apply variational autoencoder?)</td>
-</tr>
-</tbody>
-</table>
-<p>##Example The approach for logistic regression (LRTorch) and the Multi-Layer Perceptron (MLPTorch) is identical. Here we will take LRTorch as an example.</p>
-<p>You need to generate a <code>population</code> and <code>plpData</code> object as described in more detail in <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf"><code>BuildingPredictiveModels</code> vignette</a>.</p>
-<p>Alternatively, you can make use of the data simulator. The following code snippet creates a population of 12000 patients.</p>
-<div class="sourceCode" id="cb2"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/base/Random.html">set.seed</a></span><span class="op">(</span><span class="fl">1234</span><span class="op">)</span>
-<span class="fu"><a href="https://rdrr.io/r/utils/data.html">data</a></span><span class="op">(</span><span class="va">plpDataSimulationProfile</span><span class="op">)</span>
-<span class="va">sampleSize</span> <span class="op">&lt;-</span> <span class="fl">12000</span>
-<span class="va">plpData</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/simulatePlpData.html">simulatePlpData</a></span><span class="op">(</span>
-  <span class="va">plpDataSimulationProfile</span>,
-  n <span class="op">=</span> <span class="va">sampleSize</span>
-<span class="op">)</span>
-
-<span class="va">population</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createStudyPopulation.html">createStudyPopulation</a></span><span class="op">(</span>
-  <span class="va">plpData</span>,
-  outcomeId <span class="op">=</span> <span class="fl">2</span>,
-  binary <span class="op">=</span> <span class="cn">TRUE</span>,
-  firstExposureOnly <span class="op">=</span> <span class="cn">FALSE</span>,
-  washoutPeriod <span class="op">=</span> <span class="fl">0</span>,
-  removeSubjectsWithPriorOutcome <span class="op">=</span> <span class="cn">FALSE</span>,
-  priorOutcomeLookback <span class="op">=</span> <span class="fl">99999</span>,
-  requireTimeAtRisk <span class="op">=</span> <span class="cn">FALSE</span>,
-  minTimeAtRisk <span class="op">=</span> <span class="fl">0</span>,
-  riskWindowStart <span class="op">=</span> <span class="fl">0</span>,
-  addExposureDaysToStart <span class="op">=</span> <span class="cn">FALSE</span>,
-  riskWindowEnd <span class="op">=</span> <span class="fl">365</span>,
-  addExposureDaysToEnd <span class="op">=</span> <span class="cn">FALSE</span>,
-  verbosity <span class="op">=</span> <span class="st">"INFO"</span>
-<span class="op">)</span></code></pre></div>
-<p>As an example we will build a LRTorch model. We could specify the stacked autoencoder or the variational autoencoder to be used for reducing the feature dimension as an initial layer, but for this example we do not.</p>
-<div class="sourceCode" id="cb3"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">autoencoder</span> <span class="op">&lt;-</span> <span class="cn">FALSE</span>
-<span class="va">vae</span> <span class="op">&lt;-</span> <span class="cn">FALSE</span></code></pre></div>
-<p>We added a class_weight for imbalanced data, the default value 0 is the inverse ratio between negatives and positives,-1 applies focal loss.</p>
-<div class="sourceCode" id="cb4"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">class_weight</span> <span class="op">&lt;-</span> <span class="fl">0</span></code></pre></div>
-<div class="sourceCode" id="cb5"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="co"># Specify the settings for Logistics regression model using Torch in Python</span>
-<span class="va">model</span> <span class="op">&lt;-</span> <span class="fu">setLRTorch</span><span class="op">(</span>autoencoder<span class="op">=</span><span class="va">autoencoder</span>, vae<span class="op">=</span><span class="va">vae</span>,  class_weight<span class="op">=</span><span class="va">class_weight</span><span class="op">)</span></code></pre></div>
-<p>No we define our modelling parameters.</p>
-<div class="sourceCode" id="cb6"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">testFraction</span> <span class="op">&lt;-</span> <span class="fl">0.2</span>
-<span class="va">testSplit</span> <span class="op">&lt;-</span> <span class="st">'person'</span>
-<span class="va">nfold</span> <span class="op">&lt;-</span> <span class="fl">3</span>
-<span class="va">splitSeed</span> <span class="op">&lt;-</span> <span class="fl">1000</span></code></pre></div>
-<p>And we train and internally validate the model.</p>
-<div class="sourceCode" id="cb7"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">results</span> <span class="op">&lt;-</span> <span class="fu">PatientLevelPrediction</span><span class="fu">::</span><span class="fu"><a href="../reference/runPlp.html">runPlp</a></span><span class="op">(</span>population <span class="op">=</span> <span class="va">population</span>, 
-                                          plpData <span class="op">=</span> <span class="va">plpData</span>, 
-                                          modelSettings <span class="op">=</span> <span class="va">model</span>,
-                                          testSplit<span class="op">=</span><span class="va">testSplit</span>,
-                                          testFraction<span class="op">=</span><span class="va">testFraction</span>,
-                                          nfold<span class="op">=</span><span class="va">nfold</span>, 
-                                          splitSeed<span class="op">=</span><span class="va">splitSeed</span><span class="op">)</span> </code></pre></div>
-</div>
-<div id="temporal-architectures" class="section level1">
-<h1 class="hasAnchor">
-<a href="#temporal-architectures" class="anchor"></a>Temporal Architectures</h1>
-<p>Several architectures are implemented that can handle temporal data in PyTorch and R Keras.</p>
-<div id="pytorch-cnn" class="section level2">
-<h2 class="hasAnchor">
-<a href="#pytorch-cnn" class="anchor"></a>PyTorch CNN</h2>
-<p>We implemented the following <strong>convolutional</strong> models described in <a href="https://github.com/clinicalml/deepDiagnosis" class="uri">https://github.com/clinicalml/deepDiagnosis</a> in CNNTorch:</p>
-<ol style="list-style-type: decimal">
-<li><dl>
-<dt>
-Temporal Convolutional neural network over a backward window (type = cnn)
-</dt>
-<dd>
-<img src="arch1.png">
-</dd>
-</dl></li>
-<li><dl>
-<dt>
-Convolutional neural network over input and time dimension (type = mix)
-</dt>
-<dd>
-<img src="conv_arch2.png">
-</dd>
-</dl></li>
-<li><dl>
-<dt>
-Multi-resolution temporal convolutional neural network (type = multi)
-</dt>
-<dd>
-<img src="conv_arch1.png">
-</dd>
-</dl></li>
-</ol>
-<p>Furthermore, we added the following achitectures:</p>
-<ol start="4" style="list-style-type: decimal">
-<li><dl>
-<dt>
-CNN with filters with three different parallel kernel sizes (3,4,5) and a fully connected layers (type = mlf)
-</dt>
-<dd>
-<img src="cnn_mlf2.png">
-</dd>
-</dl></li>
-<li>
-<dt>
-LSTM network over the backward window (type = lstm)
-</dt>
-<dd>
-<img src="cnn_lstm.png">
-</dd>
-</li>
-<li>
-<dt>
-Residual Learning Network as described in: <a href="https://arxiv.org/abs/1512.03385" class="uri">https://arxiv.org/abs/1512.03385</a> (type = resnet)
-</dt>
-<dd>
-This a very big network, see the paper for the topology.
-</dd>
-</li>
-</ol>
-<table class="table">
-<colgroup>
-<col width="27%">
-<col width="72%">
-</colgroup>
-<thead><tr class="header">
-<th>parameter</th>
-<th>description</th>
-</tr></thead>
-<tbody>
-<tr class="odd">
-<td>nbfilters</td>
-<td>The number of convolution filters</td>
-</tr>
-<tr class="even">
-<td>epochs</td>
-<td>The number of epochs</td>
-</tr>
-<tr class="odd">
-<td>seed</td>
-<td>Random seed</td>
-</tr>
-<tr class="even">
-<td>class_weight</td>
-<td>The class weight used for imbalanced data <br> (0: Inverse ratio between positives and negatives, -1: Focal loss, or number)</td>
-</tr>
-</tbody>
-</table>
-</div>
-<div id="pytorch-rnn" class="section level2">
-<h2 class="hasAnchor">
-<a href="#pytorch-rnn" class="anchor"></a>PyTorch RNN</h2>
-<p>The following <strong>recurrent neural network</strong> models are implemented in RNNTorch:</p>
-<ol style="list-style-type: decimal">
-<li><dl>
-<dt>
-RNN with one LSTM layer fed into one fully connected layer (type = RNN)
-</dt>
-<dd>
-<dd>
-<img src="lstm_last.png">
-</dd>
-</dd>
-</dl></li>
-<li><dl>
-<dt>
-RNN with one bidirectional LSTM layer fed into one fully connected layer (type = BiRNN)
-</dt>
-<dd>
-This network looks the same as above but then as a bi-directional version
-</dd>
-</dl></li>
-<li><dl>
-<dt>
-One Gated Recurrent Unit layer fed into one fully connected layers (type = GRU)
-</dt>
-<dd>
-This network looks the same as above but then implemented as GRU
-</dd>
-</dl></li>
-</ol>
-<p>The following hyper-parameters can be set for these PyTorch models:</p>
-<table class="table">
-<colgroup>
-<col width="27%">
-<col width="72%">
-</colgroup>
-<thead><tr class="header">
-<th>parameter</th>
-<th>description</th>
-</tr></thead>
-<tbody>
-<tr class="odd">
-<td>hidden_size</td>
-<td>The number of features in hidden state</td>
-</tr>
-<tr class="even">
-<td>epochs</td>
-<td>The number of epochs</td>
-</tr>
-<tr class="odd">
-<td>seed</td>
-<td>Random seed</td>
-</tr>
-<tr class="even">
-<td>class_weight</td>
-<td>The class weight used for imbalanced data <br> (0: Inverse ratio between positives and negatives, -1: Focal loss, or number)</td>
-</tr>
-</tbody>
-</table>
-<div style="page-break-after: always;"></div>
-</div>
-<div id="r-keras-cnn" class="section level2">
-<h2 class="hasAnchor">
-<a href="#r-keras-cnn" class="anchor"></a>R Keras CNN</h2>
-<p>The following temporal architectures as described in <a href="https://arxiv.org/pdf/1608.00647.pdf" class="uri">https://arxiv.org/pdf/1608.00647.pdf</a> were implemented using R Keras:</p>
-<ol style="list-style-type: decimal">
-<li><dl>
-<dt>
-Multi-resolution CovNN model (CovNN.R)
-</dt>
-<dd>
-<img src="CovCNN.png">
-</dd>
-</dl></li>
-<li>
-<dl>
-<dt>
-Convolution across data and time according(CovNN2.R)
-</dt>
-<dd>
-<img src="covcnn2.png">
-</dd>
-</dl>
-<div style="page-break-after: always;"></div>
-</li>
-</ol>
-<p>Furthermore, a custom build RNN is added that uses a variational autoencoder.</p>
-<ol start="3" style="list-style-type: decimal">
-<li><dl>
-<dt>
-Clinically Informing application based on Recurrent Neural Network (CIReNN.R)
-</dt>
-<dd>
-<img src="cirenn.png">
-</dd>
-</dl></li>
-</ol>
-<p>Table 2: Temporal Deep Learning Models</p>
-<table class="table">
-<colgroup>
-<col width="11%">
-<col width="88%">
-</colgroup>
-<thead><tr class="header">
-<th>Model</th>
-<th>Hyper-parameters</th>
-</tr></thead>
-<tbody>
-<tr class="odd">
-<td>CovNN</td>
-<td>batchSize (The number of samples to used in each batch during model training), outcomeWeight (The weight assigned to the outcome), lr (The learning rate), decay (The decay of the learning rate), dropout ([currently not used] the dropout rate for regularization), epochs (The number of times data is used to train the model, e.g., epoches=1 means data only used once to train), filters (The number of columns output by each convolution), kernelSize (The number of time dimensions used for each convolution), loss (The loss function implemented), seed (The random seed)</td>
-</tr>
-<tr class="even">
-<td>CovNN2</td>
-<td>batchSize (The number of samples to used in each batch during model training), outcomeWeight (The weight assigned to the outcome), lr (The learning rate), decay (The decay of the learning rate), dropout ([currently not used] the dropout rate for regularization), epochs (The number of times data is used to train the model, e.g., epoches=1 means data only used once to train), filters (The number of columns output by each convolution), kernelSize (The number of time dimensions used for each convolution), loss (The loss function implemented), seed (The random seed)</td>
-</tr>
-<tr class="odd">
-<td>CIReNN</td>
-<td>units (The number of units of RNN layer - as a list of vectors), recurrentDropout (The reccurrent dropout rate), layerDropout (The layer dropout rate), lr (Learning rate), decay (Learning rate decay over each update), outcomeWeight (The weight of the outcome class in the loss function), batchSize (The number of data points to use per training batch), epochs (Number of times to iterate over data set), earlyStoppingMinDelta (Minimum change in the monitored quantity to qualify as an improvement for early stopping, i.e. an absolute change of less than min_delta in loss of validation data, will count as no improvement), earlyStoppingPatience (Number of epochs with no improvement after which training will be stopped), seed (Random seed used by Deep Learning model)</td>
-</tr>
-</tbody>
-</table>
-</div>
-<div id="example" class="section level2">
-<h2 class="hasAnchor">
-<a href="#example" class="anchor"></a>Example</h2>
-<p>We will now show how to use the temporal models by using CNNTorch as an example.</p>
-<p>You need to generate a <code>population</code> and <code>plpData</code> object as described in more detail in <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf"><code>BuildingPredictiveModels</code> vignette</a>.</p>
-<p>Note that for these algorithms you need to extracted temporal data as described in the [FeatureExtraction vignette] (<a href="https://github.com/OHDSI/FeatureExtraction/blob/master/inst/doc/UsingFeatureExtraction.pdf" class="uri">https://github.com/OHDSI/FeatureExtraction/blob/master/inst/doc/UsingFeatureExtraction.pdf</a>) as follows:</p>
-<div class="sourceCode" id="cb8"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">settings</span> <span class="op">&lt;-</span> <span class="fu">createTemporalCovariateSettings</span><span class="op">(</span>useConditionEraStart <span class="op">=</span> <span class="cn">FALSE</span>,
-                                            useConditionEraOverlap <span class="op">=</span> <span class="cn">FALSE</span>,
-                                            useConditionOccurrence <span class="op">=</span> <span class="cn">FALSE</span>,
-                                            useConditionEraGroupStart <span class="op">=</span> <span class="cn">FALSE</span>,
-                                            useConditionEraGroupOverlap <span class="op">=</span> <span class="cn">FALSE</span>,
-                                            useDrugExposure <span class="op">=</span> <span class="cn">FALSE</span>,
-                                            useDrugEraStart <span class="op">=</span> <span class="cn">FALSE</span>,
-                                            useDrugEraOverlap <span class="op">=</span> <span class="cn">FALSE</span>,
-                                            useMeasurement <span class="op">=</span> <span class="cn">FALSE</span>,
-                                            useMeasurementValue <span class="op">=</span> <span class="cn">TRUE</span>,
-                                            useMeasurementRangeGroup <span class="op">=</span> <span class="cn">FALSE</span>,
-                                            useProcedureOccurrence <span class="op">=</span> <span class="cn">FALSE</span>,
-                                            useDeviceExposure <span class="op">=</span> <span class="cn">FALSE</span>,
-                                            useObservation <span class="op">=</span> <span class="cn">FALSE</span>,
-                                            excludedCovariateConceptIds <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="fl">316866</span><span class="op">)</span>,
-                                            addDescendantsToExclude <span class="op">=</span> <span class="cn">TRUE</span>,
-                                            temporalStartDays <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/seq.html">seq</a></span><span class="op">(</span>from <span class="op">=</span> <span class="op">-</span><span class="fl">365</span>, 
-                                                                    to <span class="op">=</span> <span class="op">-</span><span class="fl">1</span>, by <span class="op">=</span> <span class="fl">12</span><span class="op">)</span>, 
-                                            temporalEndDays <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/seq.html">seq</a></span><span class="op">(</span>from <span class="op">=</span> <span class="op">-</span><span class="fl">353</span>, 
-                                                                    to <span class="op">=</span> <span class="fl">0</span>, by <span class="op">=</span> <span class="fl">12</span><span class="op">)</span>, <span class="fl">0</span><span class="op">)</span><span class="op">)</span>
-
-<span class="va">plpData</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/getPlpData.html">getPlpData</a></span><span class="op">(</span>connectionDetails <span class="op">=</span> <span class="va">connectionDetails</span>,
-                        cdmDatabaseSchema <span class="op">=</span> <span class="va">cdmDatabaseSchema</span>,
-                        cohortDatabaseSchema <span class="op">=</span> <span class="st">"results"</span>,
-                        cohortTable <span class="op">=</span> <span class="st">"cohort"</span>,
-                        cohortId <span class="op">=</span> <span class="fl">11</span>,
-                        covariateSettings <span class="op">=</span> <span class="va">settings</span>,
-                        outcomeDatabaseSchema <span class="op">=</span> <span class="va">resultsDatabaseSchema</span>,
-                        outcomeTable <span class="op">=</span> <span class="st">"cohort"</span>,
-                        outcomeIds <span class="op">=</span> <span class="fl">25</span>,
-                        cdmVersion <span class="op">=</span> <span class="fl">5</span><span class="op">)</span></code></pre></div>
-<p>Each CNN/RNN has several hyper-parameters that can be set as shown in the Tables above, but for this example we take the defaults.</p>
-<div class="sourceCode" id="cb9"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="co"># specify the the CNN</span>
-<span class="va">model</span> <span class="op">&lt;-</span> <span class="fu">setCNNTorch</span><span class="op">(</span>cnn_type<span class="op">=</span><span class="st">'CNN'</span><span class="op">)</span></code></pre></div>
-<p>Run the model training, for example with a testFraction = 0.2 and a split by person:</p>
-<div class="sourceCode" id="cb10"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">results</span> <span class="op">&lt;-</span> <span class="fu">PatientLevelPrediction</span><span class="fu">::</span><span class="fu"><a href="../reference/runPlp.html">runPlp</a></span><span class="op">(</span><span class="va">population</span>, <span class="va">plpData</span>, <span class="va">model</span>,
-                                          testSplit<span class="op">=</span><span class="st">'person'</span>,
-                                          testFraction<span class="op">=</span><span class="fl">0.2</span>,
-                                          nfold<span class="op">=</span><span class="fl">3</span>, 
-                                          splitSeed<span class="op">=</span><span class="fl">1000</span><span class="op">)</span> </code></pre></div>
-</div>
-</div>
-<div id="apply-the-trained-deep-learning-model" class="section level1">
-<h1 class="hasAnchor">
-<a href="#apply-the-trained-deep-learning-model" class="anchor"></a>Apply the trained Deep Learning model</h1>
-<p>Applying a Deep Learning is identical to the other models in the package:</p>
-<div class="sourceCode" id="cb11"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="co"># load the trained model</span>
-<span class="va">plpModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/loadPlpModel.html">loadPlpModel</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"&lt;your model&gt;"</span><span class="op">)</span>
-
-<span class="co"># load the new plpData (should have the same temporal features!) and create the population</span>
-<span class="va">plpData</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/loadPlpData.html">loadPlpData</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"&lt;your data&gt;"</span><span class="op">)</span>
-
-<span class="va">populationSettings</span> <span class="op">&lt;-</span> <span class="va">plpModel</span><span class="op">$</span><span class="va">populationSettings</span>
-<span class="va">populationSettings</span><span class="op">$</span><span class="va">plpData</span> <span class="op">&lt;-</span> <span class="va">plpData</span>
-<span class="va">population</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/do.call.html">do.call</a></span><span class="op">(</span><span class="va">createStudyPopulation</span>, <span class="va">populationSettings</span><span class="op">)</span>  
-
-<span class="co"># apply the trained model on the new data</span>
-<span class="va">validationResults</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/applyModel.html">applyModel</a></span><span class="op">(</span><span class="va">population</span>, <span class="va">plpData</span>, <span class="va">plpModel</span><span class="op">)</span></code></pre></div>
-</div>
-<div id="adding-new-architectures" class="section level1">
-<h1 class="hasAnchor">
-<a href="#adding-new-architectures" class="anchor"></a>Adding new architectures</h1>
-<p>It is possible to add new architectures in our framework using PyTorch or R Keras. We are happy to help you with this, please post your questions on the <a href="www.github.com/OHDSI/PatientLevelPrediction/issues">issue tracker</a> of the package.</p>
-</div>
-<div id="acknowledgments" class="section level1">
-<h1 class="hasAnchor">
-<a href="#acknowledgments" class="anchor"></a>Acknowledgments</h1>
-<p>Considerable work has been dedicated to provide the <code>PatientLevelPrediction</code> package.</p>
-<div class="sourceCode" id="cb12"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/utils/citation.html">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></code></pre></div>
-<pre><code>## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design
-## and implementation of a standardized framework to generate and evaluate
-## patient-level prediction models using observational healthcare data."
-## _Journal of the American Medical Informatics Association_, *25*(8),
-## 969-975. &lt;URL: https://doi.org/10.1093/jamia/ocy032&gt;.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }</code></pre>
-<p><strong>Please reference this paper if you use the PLP Package in your work:</strong></p>
-<p><a href="http://dx.doi.org/10.1093/jamia/ocy032">Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.</a></p>
-</div>
-  </div>
-
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-
-        <nav id="toc" data-toggle="toc"><h2 data-toc-skip>Contents</h2>
-    </nav>
-</div>
-
-</div>
-
-
-
-      <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-</div>
-
-  
-
-
-  </body>
-</html>
diff --git a/docs/articles/BuildingDeepLearningModels_files/header-attrs-2.7/header-attrs.js b/docs/articles/BuildingDeepLearningModels_files/header-attrs-2.7/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/BuildingDeepLearningModels_files/header-attrs-2.7/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/BuildingEnsembleModels.html b/docs/articles/BuildingEnsembleModels.html
deleted file mode 100644
index 8cbfd2eb8..000000000
--- a/docs/articles/BuildingEnsembleModels.html
+++ /dev/null
@@ -1,316 +0,0 @@
-<!DOCTYPE html>
-<!-- Generated by pkgdown: do not edit by hand --><html lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-<title>Building Ensemble Models • PatientLevelPrediction</title>
-<!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous">
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous">
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous">
-<!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script><meta property="og:title" content="Building Ensemble Models">
-<meta property="og:description" content="PatientLevelPrediction">
-<!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-</head>
-<body data-spy="scroll" data-target="#toc">
-    <div class="container template-article">
-      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-<li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/InstallationGuide.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-<li>
-      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
-    </li>
-    <li>
-      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
-    </li>
-    <li>
-      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-<ul class="nav navbar-nav navbar-right">
-<li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-</div>
-<!--/.nav-collapse -->
-  </div>
-<!--/.container -->
-</div>
-<!--/.navbar -->
-
-      
-
-      </header><script src="BuildingEnsembleModels_files/header-attrs-2.11/header-attrs.js"></script><div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header toc-ignore">
-      <h1 data-toc-skip>Building Ensemble Models</h1>
-                        <h4 class="author">Xiaoyong Pan, Jenna Reps, Peter R. Rijnbeek</h4>
-            
-            <h4 class="date">2022-03-09</h4>
-      
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/BuildingEnsembleModels.Rmd"><code>vignettes/BuildingEnsembleModels.Rmd</code></a></small>
-      <div class="hidden name"><code>BuildingEnsembleModels.Rmd</code></div>
-
-    </div>
-
-    
-    
-<!--
-%\VignetteEngine{knitr}
-%\VignetteIndexEntry{Building Ensemble Models}
--->
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
-<p>Ensemble models combine several models to improve the overall performance. Traditionally, weak learners were combined to boost performance but recent results show that combining several strong approaches can also result in a better performance. There are many examples in literature where ensemble models outperform individual models using stacking, i.e. a final logistic regresssion layer accross the individual model outputs, but other approaches like weigthing has also shown promising results.</p>
-<p>This vignette describes how you can use the Observational Health Data Sciencs and Informatics (OHDSI) <a href="http://github.com/OHDSI/PatientLevelPrediction"><code>PatientLevelPrediction</code></a> package to build ensemble models. This vignette assumes you have read and are comfortable with building single patient level prediction models as described in the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf"><code>BuildingPredictiveModels</code> vignette</a>.</p>
-<p>This will enable studying ensemble methods at scale in the OHDSI data network.</p>
-<div class="figure">
-<img src="ensemble.png" alt=""><p class="caption">Ensemble model</p>
-</div>
-<p>In PatientLevelPrediction package, four ensemble strategies have been implemented:</p>
-<ol style="list-style-type: decimal">
-<li>average ensemble: Calculate the average probability from individual models</li>
-<li>product ensemble: Calculate the product of probabilites from individual models.</li>
-<li>weighted ensemble: Calculate the weighted average probability from individual models using train AUC as weights.</li>
-<li>stacked ensemble: Train a logistics regression on outputs from individual models</li>
-</ol>
-</div>
-<div id="usage" class="section level1">
-<h1 class="hasAnchor">
-<a href="#usage" class="anchor"></a>Usage</h1>
-<p>Use the <a href="http://github.com/OHDSI/PatientLevelPrediction"><code>PatientLevelPrediction</code></a> package to generate a <code>population</code> and <code>plpData</code> object. Alternatively, you can make use of the data simulator. The following code snippet creates a population of 12000 patients.</p>
-<div class="sourceCode" id="cb1"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/utils/data.html">data</a></span><span class="op">(</span><span class="va">plpDataSimulationProfile</span><span class="op">)</span>
-<span class="fu"><a href="https://rdrr.io/r/base/Random.html">set.seed</a></span><span class="op">(</span><span class="fl">1234</span><span class="op">)</span>
-<span class="va">sampleSize</span> <span class="op">&lt;-</span> <span class="fl">2000</span>
-<span class="va">plpData</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/simulatePlpData.html">simulatePlpData</a></span><span class="op">(</span>
-  <span class="va">plpDataSimulationProfile</span>,
-  n <span class="op">=</span> <span class="va">sampleSize</span>
-<span class="op">)</span>
-
-<span class="va">population</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createStudyPopulation.html">createStudyPopulation</a></span><span class="op">(</span>
-  <span class="va">plpData</span>,
-  outcomeId <span class="op">=</span> <span class="fl">2</span>,
-  binary <span class="op">=</span> <span class="cn">TRUE</span>,
-  firstExposureOnly <span class="op">=</span> <span class="cn">FALSE</span>,
-  washoutPeriod <span class="op">=</span> <span class="fl">0</span>,
-  removeSubjectsWithPriorOutcome <span class="op">=</span> <span class="cn">FALSE</span>,
-  priorOutcomeLookback <span class="op">=</span> <span class="fl">99999</span>,
-  requireTimeAtRisk <span class="op">=</span> <span class="cn">FALSE</span>,
-  minTimeAtRisk <span class="op">=</span> <span class="fl">0</span>,
-  riskWindowStart <span class="op">=</span> <span class="fl">0</span>,
-  addExposureDaysToStart <span class="op">=</span> <span class="cn">FALSE</span>,
-  riskWindowEnd <span class="op">=</span> <span class="fl">365</span>,
-  addExposureDaysToEnd <span class="op">=</span> <span class="cn">FALSE</span>,
-  verbosity <span class="op">=</span> <span class="st">"INFO"</span>
-<span class="op">)</span></code></pre></div>
-<p>Specify the prediction algorithms to be combined.</p>
-<div class="sourceCode" id="cb2"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="co"># Use LASSO logistic regression and Random Forest as base predictors</span>
-<span class="va">model1</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span>
-<span class="va">model2</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/setRandomForest.html">setRandomForest</a></span><span class="op">(</span><span class="op">)</span></code></pre></div>
-<p>Specify a test fraction and a sequence of training set fractions.</p>
-<div class="sourceCode" id="cb3"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">testFraction</span> <span class="op">&lt;-</span> <span class="fl">0.2</span></code></pre></div>
-<p>Specify an ensembleStrategy to combine multiple predictors. The strategy used for ensembling the outputs from different models, it can be ‘mean’, ‘product’, ‘weighted’ and ‘stacked’: ‘mean’ the average probability from differnt models ‘product’ the product rule ‘weighted’ the weighted average probability from different models using train AUC as weights. ‘stacked’ the stakced ensemble trains a logistics regression on different models.</p>
-<div class="sourceCode" id="cb4"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">ensembleStrategy</span> <span class="op">&lt;-</span> <span class="st">'stacked'</span></code></pre></div>
-<p>Specify the test split to be used.</p>
-<div class="sourceCode" id="cb5"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="co"># Use a split by person, alterantively a time split is possible</span>
-<span class="va">testSplit</span> <span class="op">&lt;-</span> <span class="st">'person'</span></code></pre></div>
-<p>Run the ensemble learning to combine model1 and model2. You can also use different plpData for different models.</p>
-<div class="sourceCode" id="cb6"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">ensembleResults</span> <span class="op">&lt;-</span> <span class="fu">PatientLevelPrediction</span><span class="fu">::</span><span class="fu">runEnsembleModel</span><span class="op">(</span><span class="va">population</span>, 
-                                   dataList <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span><span class="va">plpData</span>, <span class="va">plpData</span><span class="op">)</span>, 
-                                   modelList <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span><span class="va">model1</span>, <span class="va">model2</span><span class="op">)</span>,
-                                   testSplit<span class="op">=</span><span class="va">testSplit</span>,
-                                   testFraction<span class="op">=</span><span class="va">testFraction</span>,
-                                   nfold<span class="op">=</span><span class="fl">3</span>, splitSeed<span class="op">=</span><span class="fl">1000</span>, 
-                                   ensembleStrategy <span class="op">=</span> <span class="va">ensembleStrategy</span><span class="op">)</span> </code></pre></div>
-<div id="saving-and-loading-the-ensemble-model" class="section level2">
-<h2 class="hasAnchor">
-<a href="#saving-and-loading-the-ensemble-model" class="anchor"></a>Saving and loading the ensemble model</h2>
-<p>You can save and load the model using:</p>
-<div class="sourceCode" id="cb7"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu">saveEnsemblePlpModel</span><span class="op">(</span><span class="va">ensembleResults</span><span class="op">$</span><span class="va">model</span>, dirPath <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"model"</span><span class="op">)</span><span class="op">)</span>
-<span class="va">ensembleModel</span> <span class="op">&lt;-</span> <span class="fu">loadEnsemblePlpModel</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"model"</span><span class="op">)</span></code></pre></div>
-</div>
-</div>
-<div id="apply-ensemble-model" class="section level1">
-<h1 class="hasAnchor">
-<a href="#apply-ensemble-model" class="anchor"></a>Apply Ensemble model</h1>
-<div class="sourceCode" id="cb8"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">plpData</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/loadPlpData.html">loadPlpData</a></span><span class="op">(</span><span class="st">"&lt;data file&gt;"</span><span class="op">)</span>
-<span class="va">populationSettings</span> <span class="op">&lt;-</span> <span class="va">ensembleModel</span><span class="op">$</span><span class="va">populationSettings</span>
-<span class="va">populationSettings</span><span class="op">$</span><span class="va">plpData</span> <span class="op">&lt;-</span> <span class="va">plpData</span>
-<span class="va">population</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/do.call.html">do.call</a></span><span class="op">(</span><span class="va">createStudyPopulation</span>, <span class="va">populationSettings</span><span class="op">)</span></code></pre></div>
-<p>Load the model.</p>
-<div class="sourceCode" id="cb9"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">ensembleModel</span> <span class="op">&lt;-</span> <span class="fu">loadEnsemblePlpModel</span><span class="op">(</span><span class="st">"&lt;model folder&gt;"</span><span class="op">)</span></code></pre></div>
-<p>Get the predictions by applying the model:</p>
-<div class="sourceCode" id="cb10"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">prediction</span> <span class="op">&lt;-</span> <span class="fu">applyEnsembleModel</span><span class="op">(</span><span class="va">population</span>,
-                                  dataList <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span><span class="va">plpData</span>, <span class="va">plpData</span><span class="op">)</span>,
-                                  ensembleModel <span class="op">=</span> <span class="va">ensembleModel</span><span class="op">)</span><span class="op">$</span><span class="va">prediction</span></code></pre></div>
-</div>
-<div id="demo" class="section level1">
-<h1 class="hasAnchor">
-<a href="#demo" class="anchor"></a>Demo</h1>
-<p>We have added a demo of the ensemble training:</p>
-<div class="sourceCode" id="cb11"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="co"># Show all demos in our package: </span>
- <span class="fu"><a href="https://rdrr.io/r/utils/demo.html">demo</a></span><span class="op">(</span>package <span class="op">=</span> <span class="st">"PatientLevelPrediction"</span><span class="op">)</span>
-
-<span class="co"># Run the learning curve</span>
- <span class="fu"><a href="https://rdrr.io/r/utils/demo.html">demo</a></span><span class="op">(</span><span class="st">"EnsembleModelDemo"</span>, package <span class="op">=</span> <span class="st">"PatientLevelPrediction"</span><span class="op">)</span></code></pre></div>
-</div>
-<div id="acknowledgments" class="section level1">
-<h1 class="hasAnchor">
-<a href="#acknowledgments" class="anchor"></a>Acknowledgments</h1>
-<p>Considerable work has been dedicated to provide the <code>PatientLevelPrediction</code> package.</p>
-<div class="sourceCode" id="cb12"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/utils/citation.html">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></code></pre></div>
-<pre><code>## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design
-## and implementation of a standardized framework to generate and evaluate
-## patient-level prediction models using observational healthcare data."
-## _Journal of the American Medical Informatics Association_, *25*(8),
-## 969-975. &lt;URL: https://doi.org/10.1093/jamia/ocy032&gt;.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }</code></pre>
-<p><strong>Please reference this paper if you use the PLP Package in your work:</strong></p>
-<p><a href="http://dx.doi.org/10.1093/jamia/ocy032">Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.</a></p>
-</div>
-  </div>
-
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-
-        <nav id="toc" data-toggle="toc"><h2 data-toc-skip>Contents</h2>
-    </nav>
-</div>
-
-</div>
-
-
-
-      <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-</div>
-
-  
-
-
-  </body>
-</html>
diff --git a/docs/articles/BuildingEnsembleModels_files/header-attrs-2.11/header-attrs.js b/docs/articles/BuildingEnsembleModels_files/header-attrs-2.11/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/BuildingEnsembleModels_files/header-attrs-2.11/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/BuildingEnsembleModels_files/header-attrs-2.7/header-attrs.js b/docs/articles/BuildingEnsembleModels_files/header-attrs-2.7/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/BuildingEnsembleModels_files/header-attrs-2.7/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/BuildingMultiplePredictiveModels.html b/docs/articles/BuildingMultiplePredictiveModels.html
index e8e402763..45d404d03 100644
--- a/docs/articles/BuildingMultiplePredictiveModels.html
+++ b/docs/articles/BuildingMultiplePredictiveModels.html
@@ -19,6 +19,8 @@
 <![endif]-->
 </head>
 <body data-spy="scroll" data-target="#toc">
+    
+
     <div class="container template-article">
       <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
@@ -31,7 +33,7 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
@@ -53,7 +55,7 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
@@ -72,10 +74,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -84,13 +86,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -101,21 +106,30 @@
   </ul>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
       </ul>
 <ul class="nav navbar-nav navbar-right">
 <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
@@ -130,15 +144,17 @@
 
       
 
-      </header><script src="BuildingMultiplePredictiveModels_files/header-attrs-2.11/header-attrs.js"></script><div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header toc-ignore">
-      <h1 data-toc-skip>Automatically Build Multiple Patient-Level Predictive Models</h1>
-                        <h4 class="author">Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek</h4>
+      <h1 data-toc-skip>Automatically Build Multiple Patient-Level
+Predictive Models</h1>
+                        <h4 data-toc-skip class="author">Jenna Reps,
+Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek</h4>
             
-            <h4 class="date">2022-03-09</h4>
+            <h4 data-toc-skip class="date">2024-09-09</h4>
       
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/BuildingMultiplePredictiveModels.Rmd"><code>vignettes/BuildingMultiplePredictiveModels.Rmd</code></a></small>
+      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/vignettes/BuildingMultiplePredictiveModels.Rmd" class="external-link"><code>vignettes/BuildingMultiplePredictiveModels.Rmd</code></a></small>
       <div class="hidden name"><code>BuildingMultiplePredictiveModels.Rmd</code></div>
 
     </div>
@@ -146,20 +162,39 @@ <h4 class="date">2022-03-09</h4>
     
     
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Building multiple patient-level predictive models}
 -->
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
-<p>In our <a href="https://academic.oup.com/jamia/article/25/8/969/4989437"><code>paper</code></a>, we propose a standardised framework for patient-level prediction that utilizes the OMOP CDM and standardized vocabularies, and describe the open-source software that we developed implementing the framework’s pipeline. The framework is the first to enforce existing best practice guidelines and will enable open dissemination of models that can be extensively validated across the network of OHDSI collaborators.</p>
-<p>One our best practices is that we see the selection of models and all study setting as an emperical question, i.e. we should use a data-driven approach in which we try many settings. This vignette describes how you can use the Observational Health Data Sciencs and Informatics (OHDSI) <a href="http://github.com/OHDSI/PatientLevelPrediction"><code>PatientLevelPrediction</code></a> package to automatically build multiple patient-level predictive models, e.g. different population settings, covariate settings, and modelsetting. This vignette assumes you have read and are comfortable with building single patient level prediction models as described in the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf"><code>BuildingPredictiveModels</code> vignette</a>.</p>
-<p>Note that it is also possible to generate a Study Package directly in Atlas that allows for multiple patient-level prediction analyses this is out-of-scope for this vignette.</p>
+<div class="section level2">
+<h2 id="introduction">Introduction<a class="anchor" aria-label="anchor" href="#introduction"></a>
+</h2>
+<p>In our <a href="https://academic.oup.com/jamia/article/25/8/969/4989437" class="external-link"><code>paper</code></a>,
+we propose a standardised framework for patient-level prediction that
+utilizes the OMOP CDM and standardized vocabularies, and describe the
+open-source software that we developed implementing the framework’s
+pipeline. The framework is the first to enforce existing best practice
+guidelines and will enable open dissemination of models that can be
+extensively validated across the network of OHDSI collaborators.</p>
+<p>One our best practices is that we see the selection of models and all
+study setting as an emperical question, i.e. we should use a data-driven
+approach in which we try many settings. This vignette describes how you
+can use the Observational Health Data Sciencs and Informatics (OHDSI) <a href="http://github.com/OHDSI/PatientLevelPrediction" class="external-link"><code>PatientLevelPrediction</code></a>
+package to automatically build multiple patient-level predictive models,
+e.g. different population settings, covariate settings, and
+modelsetting. This vignette assumes you have read and are comfortable
+with building single patient level prediction models as described in the
+<a href="https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingPredictiveModels.pdf" class="external-link"><code>BuildingPredictiveModels</code>
+vignette</a>.</p>
+<p>Note that it is also possible to generate a Study Package directly in
+Atlas that allows for multiple patient-level prediction analyses this is
+out-of-scope for this vignette.</p>
 </div>
-<div id="creating-a-model-design" class="section level1">
-<h1 class="hasAnchor">
-<a href="#creating-a-model-design" class="anchor"></a>Creating a model design</h1>
-<p>The first step is to specify each model you wish to develop by using the <code>createModelDesign</code> function. This function requires the following:</p>
+<div class="section level2">
+<h2 id="creating-a-model-design">Creating a model design<a class="anchor" aria-label="anchor" href="#creating-a-model-design"></a>
+</h2>
+<p>The first step is to specify each model you wish to develop by using
+the <code>createModelDesign</code> function. This function requires the
+following:</p>
 <table class="table">
 <caption>The inputs for the model design</caption>
 <colgroup>
@@ -181,275 +216,322 @@ <h1 class="hasAnchor">
 </tr>
 <tr class="odd">
 <td align="left">restrictPlpDataSettings</td>
-<td align="left">The settings used to restrict the target population, created with createRestrictPlpDataSettings()</td>
+<td align="left">The settings used to restrict the target population,
+created with createRestrictPlpDataSettings()</td>
 </tr>
 <tr class="even">
 <td align="left">populationSettings</td>
-<td align="left">The settings used to restrict the target population and create the outcome labels, created with createStudyPopulationSettings()</td>
+<td align="left">The settings used to restrict the target population and
+create the outcome labels, created with
+createStudyPopulationSettings()</td>
 </tr>
 <tr class="odd">
 <td align="left">covariateSettings</td>
-<td align="left">The settings used to define the covariates, created with FeatureExtraction::createDefaultCovariateSettings()</td>
+<td align="left">The settings used to define the covariates, created
+with FeatureExtraction::createDefaultCovariateSettings()</td>
 </tr>
 <tr class="even">
 <td align="left">sampleSettings</td>
-<td align="left">The settings used to define any under/over sampling, created with createSampleSettings()</td>
+<td align="left">The settings used to define any under/over sampling,
+created with createSampleSettings()</td>
 </tr>
 <tr class="odd">
 <td align="left">featureEngineeringSettings</td>
-<td align="left">The settings used to define any feature engineering, created with createFeatureEngineeringSettings()</td>
+<td align="left">The settings used to define any feature engineering,
+created with createFeatureEngineeringSettings()</td>
 </tr>
 <tr class="even">
 <td align="left">preprocessSettings</td>
-<td align="left">The settings used to define any preprocessing, created with createPreprocessSettings()</td>
+<td align="left">The settings used to define any preprocessing, created
+with createPreprocessSettings()</td>
 </tr>
 <tr class="odd">
 <td align="left">modelSettings</td>
-<td align="left">The settings used to define the model fitting settings, such as setLassoLogisticRegression()</td>
+<td align="left">The settings used to define the model fitting settings,
+such as setLassoLogisticRegression()</td>
 </tr>
 </tbody>
 </table>
-<div id="model-design-example-1" class="section level2">
-<h2 class="hasAnchor">
-<a href="#model-design-example-1" class="anchor"></a>Model design example 1</h2>
-<p>For example, if we wanted to predict the outcome (id 2) occuring for the first time within 180 days of the the target population index date (id 1). We are only interested in index dates betwrrn 2018-2020. Finally, we only want to use age, gender in 5 year buckets and conditions as features. The model can be specified by:</p>
+<div class="section level3">
+<h3 id="model-design-example-1">Model design example 1<a class="anchor" aria-label="anchor" href="#model-design-example-1"></a>
+</h3>
+<p>For example, if we wanted to predict the outcome (id 2) occuring for
+the first time within 180 days of the the target population index date
+(id 1). We are only interested in index dates betwrrn 2018-2020.
+Finally, we only want to use age, gender in 5 year buckets and
+conditions as features. The model can be specified by:</p>
 <div class="sourceCode" id="cb1"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="co"># Model 1 is only using data between 2018-2020:</span>
-<span class="va">restrictPlpDataSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createRestrictPlpDataSettings.html">createRestrictPlpDataSettings</a></span><span class="op">(</span>
-  studyStartDate <span class="op">=</span> <span class="st">'20180101'</span>, 
-  studyEndDate <span class="op">=</span> <span class="st">'20191231'</span>
-  <span class="op">)</span>
-
-<span class="co"># predict outcome within 1 to 180 days after index</span>
-<span class="co"># remove people with outcome prior and with &lt; 365 days observation</span>
-<span class="va">populationSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createStudyPopulationSettings.html">createStudyPopulationSettings</a></span><span class="op">(</span>
-  binary <span class="op">=</span> <span class="cn">T</span>, 
-  firstExposureOnly <span class="op">=</span> <span class="cn">T</span>, 
-  washoutPeriod <span class="op">=</span> <span class="fl">365</span>, 
-  removeSubjectsWithPriorOutcome <span class="op">=</span> <span class="cn">T</span>,
-  priorOutcomeLookback <span class="op">=</span> <span class="fl">9999</span>,
-  requireTimeAtRisk <span class="op">=</span> <span class="cn">F</span>, 
-  riskWindowStart <span class="op">=</span> <span class="fl">1</span>, 
-  riskWindowEnd <span class="op">=</span> <span class="fl">180</span>
-<span class="op">)</span>
-
-<span class="co"># use age/gender in groups and condition groups as features</span>
-<span class="va">covariateSettings</span> <span class="op">&lt;-</span> <span class="fu">FeatureExtraction</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/FeatureExtraction/man/createCovariateSettings.html">createCovariateSettings</a></span><span class="op">(</span>
-  useDemographicsGender <span class="op">=</span> <span class="cn">T</span>, 
-  useDemographicsAgeGroup <span class="op">=</span> <span class="cn">T</span>, 
-  useConditionGroupEraAnyTimePrior <span class="op">=</span> <span class="cn">T</span>
-<span class="op">)</span>
-
-<span class="va">modelDesign1</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createModelDesign.html">createModelDesign</a></span><span class="op">(</span>
-  targetId <span class="op">=</span> <span class="fl">1</span>, 
-  outcomeId <span class="op">=</span> <span class="fl">2</span>, 
-  restrictPlpDataSettings <span class="op">=</span> <span class="va">restrictPlpDataSettings</span>, 
-  populationSettings <span class="op">=</span> <span class="va">populationSettings</span>, 
-  covariateSettings <span class="op">=</span> <span class="va">covariateSettings</span>, 
-  featureEngineeringSettings <span class="op">=</span> <span class="fu"><a href="../reference/createFeatureEngineeringSettings.html">createFeatureEngineeringSettings</a></span><span class="op">(</span><span class="op">)</span>,
-  sampleSettings <span class="op">=</span> <span class="fu"><a href="../reference/createSampleSettings.html">createSampleSettings</a></span><span class="op">(</span><span class="op">)</span>, 
-  preprocessSettings <span class="op">=</span> <span class="fu"><a href="../reference/createPreprocessSettings.html">createPreprocessSettings</a></span><span class="op">(</span><span class="op">)</span>, 
-  modelSettings <span class="op">=</span> <span class="fu"><a href="../reference/setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span>
-  <span class="op">)</span></code></pre></div>
+<code class="sourceCode R"><span><span class="co"># Model 1 is only using data between 2018-2020:</span></span>
+<span><span class="va">restrictPlpDataSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createRestrictPlpDataSettings.html">createRestrictPlpDataSettings</a></span><span class="op">(</span></span>
+<span>  studyStartDate <span class="op">=</span> <span class="st">'20180101'</span>, </span>
+<span>  studyEndDate <span class="op">=</span> <span class="st">'20191231'</span></span>
+<span>  <span class="op">)</span></span>
+<span></span>
+<span><span class="co"># predict outcome within 1 to 180 days after index</span></span>
+<span><span class="co"># remove people with outcome prior and with &lt; 365 days observation</span></span>
+<span><span class="va">populationSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createStudyPopulationSettings.html">createStudyPopulationSettings</a></span><span class="op">(</span></span>
+<span>  binary <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>  firstExposureOnly <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>  washoutPeriod <span class="op">=</span> <span class="fl">365</span>, </span>
+<span>  removeSubjectsWithPriorOutcome <span class="op">=</span> <span class="cn">T</span>,</span>
+<span>  priorOutcomeLookback <span class="op">=</span> <span class="fl">9999</span>,</span>
+<span>  requireTimeAtRisk <span class="op">=</span> <span class="cn">F</span>, </span>
+<span>  riskWindowStart <span class="op">=</span> <span class="fl">1</span>, </span>
+<span>  riskWindowEnd <span class="op">=</span> <span class="fl">180</span></span>
+<span><span class="op">)</span></span>
+<span></span>
+<span><span class="co"># use age/gender in groups and condition groups as features</span></span>
+<span><span class="va">covariateSettings</span> <span class="op">&lt;-</span> <span class="fu">FeatureExtraction</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/FeatureExtraction/man/createCovariateSettings.html" class="external-link">createCovariateSettings</a></span><span class="op">(</span></span>
+<span>  useDemographicsGender <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>  useDemographicsAgeGroup <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>  useConditionGroupEraAnyTimePrior <span class="op">=</span> <span class="cn">T</span></span>
+<span><span class="op">)</span></span>
+<span></span>
+<span><span class="va">modelDesign1</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createModelDesign.html">createModelDesign</a></span><span class="op">(</span></span>
+<span>  targetId <span class="op">=</span> <span class="fl">1</span>, </span>
+<span>  outcomeId <span class="op">=</span> <span class="fl">2</span>, </span>
+<span>  restrictPlpDataSettings <span class="op">=</span> <span class="va">restrictPlpDataSettings</span>, </span>
+<span>  populationSettings <span class="op">=</span> <span class="va">populationSettings</span>, </span>
+<span>  covariateSettings <span class="op">=</span> <span class="va">covariateSettings</span>, </span>
+<span>  featureEngineeringSettings <span class="op">=</span> <span class="fu"><a href="../reference/createFeatureEngineeringSettings.html">createFeatureEngineeringSettings</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  sampleSettings <span class="op">=</span> <span class="fu"><a href="../reference/createSampleSettings.html">createSampleSettings</a></span><span class="op">(</span><span class="op">)</span>, </span>
+<span>  splitSettings <span class="op">=</span> <span class="fu"><a href="../reference/createDefaultSplitSetting.html">createDefaultSplitSetting</a></span><span class="op">(</span><span class="op">)</span>, </span>
+<span>  preprocessSettings <span class="op">=</span> <span class="fu"><a href="../reference/createPreprocessSettings.html">createPreprocessSettings</a></span><span class="op">(</span><span class="op">)</span>, </span>
+<span>  modelSettings <span class="op">=</span> <span class="fu"><a href="../reference/setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span></span>
+<span>  <span class="op">)</span></span></code></pre></div>
 </div>
-<div id="model-design-example-2" class="section level2">
-<h2 class="hasAnchor">
-<a href="#model-design-example-2" class="anchor"></a>Model design example 2</h2>
-<p>For the second example, we want to predict the outcome (id 2) occuring for the first time within 730 days of the the target population index date (id 1). We want to train a random forest classifier. Finally, we want to use age, gender in 5 year buckets, drug ingredients (and groups) and conditions as features. The model can be specified by:</p>
+<div class="section level3">
+<h3 id="model-design-example-2">Model design example 2<a class="anchor" aria-label="anchor" href="#model-design-example-2"></a>
+</h3>
+<p>For the second example, we want to predict the outcome (id 2)
+occuring for the first time within 730 days of the the target population
+index date (id 1). We want to train a random forest classifier. Finally,
+we want to use age, gender in 5 year buckets, drug ingredients (and
+groups) and conditions as features. The model can be specified by:</p>
 <div class="sourceCode" id="cb2"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="co"># Model 2 has no restrictions when extracting data</span>
-<span class="va">restrictPlpDataSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createRestrictPlpDataSettings.html">createRestrictPlpDataSettings</a></span><span class="op">(</span>
-  <span class="op">)</span>
-
-<span class="co"># predict outcome within 1 to 730 days after index</span>
-<span class="co"># remove people with outcome prior and with &lt; 365 days observation</span>
-<span class="va">populationSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createStudyPopulationSettings.html">createStudyPopulationSettings</a></span><span class="op">(</span>
-  binary <span class="op">=</span> <span class="cn">T</span>, 
-  firstExposureOnly <span class="op">=</span> <span class="cn">T</span>, 
-  washoutPeriod <span class="op">=</span> <span class="fl">365</span>, 
-  removeSubjectsWithPriorOutcome <span class="op">=</span> <span class="cn">T</span>,
-  priorOutcomeLookback <span class="op">=</span> <span class="fl">9999</span>,
-  requireTimeAtRisk <span class="op">=</span> <span class="cn">F</span>, 
-  riskWindowStart <span class="op">=</span> <span class="fl">1</span>, 
-  riskWindowEnd <span class="op">=</span> <span class="fl">730</span>
-<span class="op">)</span>
-
-<span class="co"># use age/gender in groups and condition/drug groups as features</span>
-<span class="va">covariateSettings</span> <span class="op">&lt;-</span> <span class="fu">FeatureExtraction</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/FeatureExtraction/man/createCovariateSettings.html">createCovariateSettings</a></span><span class="op">(</span>
-  useDemographicsGender <span class="op">=</span> <span class="cn">T</span>, 
-  useDemographicsAgeGroup <span class="op">=</span> <span class="cn">T</span>, 
-  useConditionGroupEraAnyTimePrior <span class="op">=</span> <span class="cn">T</span>, 
-  useDrugGroupEraAnyTimePrior <span class="op">=</span> <span class="cn">T</span> 
-<span class="op">)</span>
-
-<span class="va">modelDesign2</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createModelDesign.html">createModelDesign</a></span><span class="op">(</span>
-  targetId <span class="op">=</span> <span class="fl">1</span>, 
-  outcomeId <span class="op">=</span> <span class="fl">2</span>, 
-  restrictPlpDataSettings <span class="op">=</span> <span class="va">restrictPlpDataSettings</span>, 
-  populationSettings <span class="op">=</span> <span class="va">populationSettings</span>, 
-  covariateSettings <span class="op">=</span> <span class="va">covariateSettings</span>, 
-  featureEngineeringSettings <span class="op">=</span> <span class="fu"><a href="../reference/createRandomForestFeatureSelection.html">createRandomForestFeatureSelection</a></span><span class="op">(</span>ntrees <span class="op">=</span> <span class="fl">500</span>, maxDepth <span class="op">=</span> <span class="fl">7</span><span class="op">)</span>,
-  sampleSettings <span class="op">=</span> <span class="fu"><a href="../reference/createSampleSettings.html">createSampleSettings</a></span><span class="op">(</span><span class="op">)</span>, 
-  preprocessSettings <span class="op">=</span> <span class="fu"><a href="../reference/createPreprocessSettings.html">createPreprocessSettings</a></span><span class="op">(</span><span class="op">)</span>, 
-  modelSettings <span class="op">=</span> <span class="fu"><a href="../reference/setRandomForest.html">setRandomForest</a></span><span class="op">(</span><span class="op">)</span>
-  <span class="op">)</span></code></pre></div>
+<code class="sourceCode R"><span><span class="co"># Model 2 has no restrictions when extracting data</span></span>
+<span><span class="va">restrictPlpDataSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createRestrictPlpDataSettings.html">createRestrictPlpDataSettings</a></span><span class="op">(</span></span>
+<span>  <span class="op">)</span></span>
+<span></span>
+<span><span class="co"># predict outcome within 1 to 730 days after index</span></span>
+<span><span class="co"># remove people with outcome prior and with &lt; 365 days observation</span></span>
+<span><span class="va">populationSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createStudyPopulationSettings.html">createStudyPopulationSettings</a></span><span class="op">(</span></span>
+<span>  binary <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>  firstExposureOnly <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>  washoutPeriod <span class="op">=</span> <span class="fl">365</span>, </span>
+<span>  removeSubjectsWithPriorOutcome <span class="op">=</span> <span class="cn">T</span>,</span>
+<span>  priorOutcomeLookback <span class="op">=</span> <span class="fl">9999</span>,</span>
+<span>  requireTimeAtRisk <span class="op">=</span> <span class="cn">F</span>, </span>
+<span>  riskWindowStart <span class="op">=</span> <span class="fl">1</span>, </span>
+<span>  riskWindowEnd <span class="op">=</span> <span class="fl">730</span></span>
+<span><span class="op">)</span></span>
+<span></span>
+<span><span class="co"># use age/gender in groups and condition/drug groups as features</span></span>
+<span><span class="va">covariateSettings</span> <span class="op">&lt;-</span> <span class="fu">FeatureExtraction</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/FeatureExtraction/man/createCovariateSettings.html" class="external-link">createCovariateSettings</a></span><span class="op">(</span></span>
+<span>  useDemographicsGender <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>  useDemographicsAgeGroup <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>  useConditionGroupEraAnyTimePrior <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>  useDrugGroupEraAnyTimePrior <span class="op">=</span> <span class="cn">T</span> </span>
+<span><span class="op">)</span></span>
+<span></span>
+<span><span class="va">modelDesign2</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createModelDesign.html">createModelDesign</a></span><span class="op">(</span></span>
+<span>  targetId <span class="op">=</span> <span class="fl">1</span>, </span>
+<span>  outcomeId <span class="op">=</span> <span class="fl">2</span>, </span>
+<span>  restrictPlpDataSettings <span class="op">=</span> <span class="va">restrictPlpDataSettings</span>, </span>
+<span>  populationSettings <span class="op">=</span> <span class="va">populationSettings</span>, </span>
+<span>  covariateSettings <span class="op">=</span> <span class="va">covariateSettings</span>, </span>
+<span>  featureEngineeringSettings <span class="op">=</span> <span class="fu"><a href="../reference/createRandomForestFeatureSelection.html">createRandomForestFeatureSelection</a></span><span class="op">(</span>ntrees <span class="op">=</span> <span class="fl">500</span>, maxDepth <span class="op">=</span> <span class="fl">7</span><span class="op">)</span>,</span>
+<span>  sampleSettings <span class="op">=</span> <span class="fu"><a href="../reference/createSampleSettings.html">createSampleSettings</a></span><span class="op">(</span><span class="op">)</span>, </span>
+<span>  splitSettings <span class="op">=</span> <span class="fu"><a href="../reference/createDefaultSplitSetting.html">createDefaultSplitSetting</a></span><span class="op">(</span><span class="op">)</span>, </span>
+<span>  preprocessSettings <span class="op">=</span> <span class="fu"><a href="../reference/createPreprocessSettings.html">createPreprocessSettings</a></span><span class="op">(</span><span class="op">)</span>, </span>
+<span>  modelSettings <span class="op">=</span> <span class="fu"><a href="../reference/setRandomForest.html">setRandomForest</a></span><span class="op">(</span><span class="op">)</span></span>
+<span>  <span class="op">)</span></span></code></pre></div>
 </div>
-<div id="model-design-example-3" class="section level2">
-<h2 class="hasAnchor">
-<a href="#model-design-example-3" class="anchor"></a>Model design example 3</h2>
-<p>For the third example, we want to predict the outcome (id 5) occuring during the cohort exposure of the the target population (id 1). We want to train a gradient boosting machine. Finally, we want to use age, gender in 5 year buckets and indications of measurements taken as features. The model can be specified by:</p>
+<div class="section level3">
+<h3 id="model-design-example-3">Model design example 3<a class="anchor" aria-label="anchor" href="#model-design-example-3"></a>
+</h3>
+<p>For the third example, we want to predict the outcome (id 5) occuring
+during the cohort exposure of the the target population (id 1). We want
+to train a gradient boosting machine. Finally, we want to use age,
+gender in 5 year buckets and indications of measurements taken as
+features. The model can be specified by:</p>
 <div class="sourceCode" id="cb3"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="co"># Model 3 has no restrictions when extracting data</span>
-<span class="va">restrictPlpDataSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createRestrictPlpDataSettings.html">createRestrictPlpDataSettings</a></span><span class="op">(</span>
-  <span class="op">)</span>
-
-<span class="co"># predict outcome during target cohort start/end </span>
-<span class="co"># remove people with  &lt; 365 days observation</span>
-<span class="va">populationSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createStudyPopulationSettings.html">createStudyPopulationSettings</a></span><span class="op">(</span>
-  binary <span class="op">=</span> <span class="cn">T</span>, 
-  firstExposureOnly <span class="op">=</span> <span class="cn">T</span>, 
-  washoutPeriod <span class="op">=</span> <span class="fl">365</span>, 
-  removeSubjectsWithPriorOutcome <span class="op">=</span> <span class="cn">F</span>,
-  requireTimeAtRisk <span class="op">=</span> <span class="cn">F</span>, 
-  riskWindowStart <span class="op">=</span> <span class="fl">0</span>,
-  startAnchor <span class="op">=</span>  <span class="st">'cohort start'</span>,
-  riskWindowEnd <span class="op">=</span> <span class="fl">0</span>, 
-  endAnchor <span class="op">=</span> <span class="st">'cohort end'</span>
-<span class="op">)</span>
-
-<span class="co"># use age/gender in groups and measurement indicators as features</span>
-<span class="va">covariateSettings</span> <span class="op">&lt;-</span> <span class="fu">FeatureExtraction</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/FeatureExtraction/man/createCovariateSettings.html">createCovariateSettings</a></span><span class="op">(</span>
-  useDemographicsGender <span class="op">=</span> <span class="cn">T</span>, 
-  useDemographicsAgeGroup <span class="op">=</span> <span class="cn">T</span>, 
-  useMeasurementAnyTimePrior <span class="op">=</span> <span class="cn">T</span>,
-  endDays <span class="op">=</span> <span class="op">-</span><span class="fl">1</span>
-<span class="op">)</span>
-
-<span class="va">modelDesign3</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createModelDesign.html">createModelDesign</a></span><span class="op">(</span>
-  targetId <span class="op">=</span> <span class="fl">1</span>, 
-  outcomeId <span class="op">=</span> <span class="fl">5</span>, 
-  restrictPlpDataSettings <span class="op">=</span> <span class="va">restrictPlpDataSettings</span>, 
-  populationSettings <span class="op">=</span> <span class="va">populationSettings</span>, 
-  covariateSettings <span class="op">=</span> <span class="va">covariateSettings</span>, 
-  featureEngineeringSettings <span class="op">=</span> <span class="fu"><a href="../reference/createFeatureEngineeringSettings.html">createFeatureEngineeringSettings</a></span><span class="op">(</span><span class="op">)</span>,
-  sampleSettings <span class="op">=</span> <span class="fu"><a href="../reference/createSampleSettings.html">createSampleSettings</a></span><span class="op">(</span><span class="op">)</span>, 
-  preprocessSettings <span class="op">=</span> <span class="fu"><a href="../reference/createPreprocessSettings.html">createPreprocessSettings</a></span><span class="op">(</span><span class="op">)</span>, 
-  modelSettings <span class="op">=</span> <span class="fu"><a href="../reference/setGradientBoostingMachine.html">setGradientBoostingMachine</a></span><span class="op">(</span><span class="op">)</span>
-  <span class="op">)</span></code></pre></div>
+<code class="sourceCode R"><span><span class="co"># Model 3 has no restrictions when extracting data</span></span>
+<span><span class="va">restrictPlpDataSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createRestrictPlpDataSettings.html">createRestrictPlpDataSettings</a></span><span class="op">(</span></span>
+<span>  <span class="op">)</span></span>
+<span></span>
+<span><span class="co"># predict outcome during target cohort start/end </span></span>
+<span><span class="co"># remove people with  &lt; 365 days observation</span></span>
+<span><span class="va">populationSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createStudyPopulationSettings.html">createStudyPopulationSettings</a></span><span class="op">(</span></span>
+<span>  binary <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>  firstExposureOnly <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>  washoutPeriod <span class="op">=</span> <span class="fl">365</span>, </span>
+<span>  removeSubjectsWithPriorOutcome <span class="op">=</span> <span class="cn">F</span>,</span>
+<span>  requireTimeAtRisk <span class="op">=</span> <span class="cn">F</span>, </span>
+<span>  riskWindowStart <span class="op">=</span> <span class="fl">0</span>,</span>
+<span>  startAnchor <span class="op">=</span>  <span class="st">'cohort start'</span>,</span>
+<span>  riskWindowEnd <span class="op">=</span> <span class="fl">0</span>, </span>
+<span>  endAnchor <span class="op">=</span> <span class="st">'cohort end'</span></span>
+<span><span class="op">)</span></span>
+<span></span>
+<span><span class="co"># use age/gender in groups and measurement indicators as features</span></span>
+<span><span class="va">covariateSettings</span> <span class="op">&lt;-</span> <span class="fu">FeatureExtraction</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/FeatureExtraction/man/createCovariateSettings.html" class="external-link">createCovariateSettings</a></span><span class="op">(</span></span>
+<span>  useDemographicsGender <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>  useDemographicsAgeGroup <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>  useMeasurementAnyTimePrior <span class="op">=</span> <span class="cn">T</span>,</span>
+<span>  endDays <span class="op">=</span> <span class="op">-</span><span class="fl">1</span></span>
+<span><span class="op">)</span></span>
+<span></span>
+<span><span class="va">modelDesign3</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createModelDesign.html">createModelDesign</a></span><span class="op">(</span></span>
+<span>  targetId <span class="op">=</span> <span class="fl">1</span>, </span>
+<span>  outcomeId <span class="op">=</span> <span class="fl">5</span>, </span>
+<span>  restrictPlpDataSettings <span class="op">=</span> <span class="va">restrictPlpDataSettings</span>, </span>
+<span>  populationSettings <span class="op">=</span> <span class="va">populationSettings</span>, </span>
+<span>  covariateSettings <span class="op">=</span> <span class="va">covariateSettings</span>, </span>
+<span>  featureEngineeringSettings <span class="op">=</span> <span class="fu"><a href="../reference/createFeatureEngineeringSettings.html">createFeatureEngineeringSettings</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  sampleSettings <span class="op">=</span> <span class="fu"><a href="../reference/createSampleSettings.html">createSampleSettings</a></span><span class="op">(</span><span class="op">)</span>, </span>
+<span>  splitSettings <span class="op">=</span> <span class="fu"><a href="../reference/createDefaultSplitSetting.html">createDefaultSplitSetting</a></span><span class="op">(</span><span class="op">)</span>, </span>
+<span>  preprocessSettings <span class="op">=</span> <span class="fu"><a href="../reference/createPreprocessSettings.html">createPreprocessSettings</a></span><span class="op">(</span><span class="op">)</span>, </span>
+<span>  modelSettings <span class="op">=</span> <span class="fu"><a href="../reference/setGradientBoostingMachine.html">setGradientBoostingMachine</a></span><span class="op">(</span><span class="op">)</span></span>
+<span>  <span class="op">)</span></span></code></pre></div>
 </div>
 </div>
-<div id="running-multiple-models" class="section level1">
-<h1 class="hasAnchor">
-<a href="#running-multiple-models" class="anchor"></a>Running multiple models</h1>
-<p>As we will be downloading loads of data in the multiple plp analysis it is useful to set the Andromeda temp folder to a directory with write access and plenty of space. <code><a href="https://rdrr.io/r/base/options.html">options(andromedaTempFolder = "c:/andromedaTemp")</a></code></p>
+<div class="section level2">
+<h2 id="running-multiple-models">Running multiple models<a class="anchor" aria-label="anchor" href="#running-multiple-models"></a>
+</h2>
+<p>As we will be downloading loads of data in the multiple plp analysis
+it is useful to set the Andromeda temp folder to a directory with write
+access and plenty of space.
+<code>options(andromedaTempFolder = "c:/andromedaTemp")</code></p>
 <p>To run the study requires setting up a connectionDetails object</p>
 <div class="sourceCode" id="cb4"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">dbms</span> <span class="op">&lt;-</span> <span class="st">"your dbms"</span>
-<span class="va">user</span> <span class="op">&lt;-</span> <span class="st">"your username"</span>
-<span class="va">pw</span> <span class="op">&lt;-</span> <span class="st">"your password"</span>
-<span class="va">server</span> <span class="op">&lt;-</span> <span class="st">"your server"</span>
-<span class="va">port</span> <span class="op">&lt;-</span> <span class="st">"your port"</span>
-
-<span class="va">connectionDetails</span> <span class="op">&lt;-</span> <span class="fu">DatabaseConnector</span><span class="fu">::</span><span class="fu"><a href="https://ohdsi.github.io/DatabaseConnector/reference/createConnectionDetails.html">createConnectionDetails</a></span><span class="op">(</span>dbms <span class="op">=</span> <span class="va">dbms</span>,
-                                                                server <span class="op">=</span> <span class="va">server</span>,
-                                                                user <span class="op">=</span> <span class="va">user</span>,
-                                                                password <span class="op">=</span> <span class="va">pw</span>,
-                                                                port <span class="op">=</span> <span class="va">port</span><span class="op">)</span></code></pre></div>
-<p>Next you need to specify the cdmDatabaseSchema where your cdm database is found and workDatabaseSchema where your target population and outcome cohorts are and you need to specify a label for the database name: a string with a shareable name of the database (this will be shown to OHDSI researchers if the results get transported).</p>
-<div class="sourceCode" id="cb5"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a>cdmDatabaseSchema <span class="ot">&lt;-</span> <span class="st">"your cdmDatabaseSchema"</span></span>
-<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a>workDatabaseSchema <span class="ot">&lt;-</span> <span class="st">"your workDatabaseSchema"</span></span>
-<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a>cdmDatabaseName <span class="ot">&lt;-</span> <span class="st">"your cdmDatabaseName"</span></span>
-<span id="cb5-4"><a href="#cb5-4" aria-hidden="true" tabindex="-1"></a>cohortTable <span class="ot">&lt;-</span> <span class="st">"your cohort table"</span>,</span>
-<span id="cb5-5"><a href="#cb5-5" aria-hidden="true" tabindex="-1"></a></span>
-<span id="cb5-6"><a href="#cb5-6" aria-hidden="true" tabindex="-1"></a>databaseDetails <span class="ot">&lt;-</span> <span class="fu">createDatabaseDetails</span>(</span>
-<span id="cb5-7"><a href="#cb5-7" aria-hidden="true" tabindex="-1"></a>  <span class="at">connectionDetails =</span> connectionDetails, </span>
-<span id="cb5-8"><a href="#cb5-8" aria-hidden="true" tabindex="-1"></a>  <span class="at">cdmDatabaseSchema =</span> cdmDatabaseSchema, </span>
-<span id="cb5-9"><a href="#cb5-9" aria-hidden="true" tabindex="-1"></a>  <span class="at">cdmDatabaseName =</span> cdmDatabaseName , </span>
-<span id="cb5-10"><a href="#cb5-10" aria-hidden="true" tabindex="-1"></a>  <span class="at">cohortDatabaseSchema =</span> workDatabaseSchema, </span>
-<span id="cb5-11"><a href="#cb5-11" aria-hidden="true" tabindex="-1"></a>  <span class="at">cohortTable =</span> cohortTable, </span>
-<span id="cb5-12"><a href="#cb5-12" aria-hidden="true" tabindex="-1"></a>  <span class="at">outcomeDatabaseSchema =</span> workDatabaseSchema, </span>
-<span id="cb5-13"><a href="#cb5-13" aria-hidden="true" tabindex="-1"></a>  <span class="at">outcomeTable =</span> cohortTable </span>
-<span id="cb5-14"><a href="#cb5-14" aria-hidden="true" tabindex="-1"></a>  <span class="at">cdmVersion =</span> <span class="dv">5</span></span>
-<span id="cb5-15"><a href="#cb5-15" aria-hidden="true" tabindex="-1"></a>    )</span></code></pre></div>
+<code class="sourceCode R"><span><span class="va">dbms</span> <span class="op">&lt;-</span> <span class="st">"your dbms"</span></span>
+<span><span class="va">user</span> <span class="op">&lt;-</span> <span class="st">"your username"</span></span>
+<span><span class="va">pw</span> <span class="op">&lt;-</span> <span class="st">"your password"</span></span>
+<span><span class="va">server</span> <span class="op">&lt;-</span> <span class="st">"your server"</span></span>
+<span><span class="va">port</span> <span class="op">&lt;-</span> <span class="st">"your port"</span></span>
+<span></span>
+<span><span class="va">connectionDetails</span> <span class="op">&lt;-</span> <span class="fu">DatabaseConnector</span><span class="fu">::</span><span class="fu"><a href="https://ohdsi.github.io/DatabaseConnector/reference/createConnectionDetails.html" class="external-link">createConnectionDetails</a></span><span class="op">(</span>dbms <span class="op">=</span> <span class="va">dbms</span>,</span>
+<span>                                                                server <span class="op">=</span> <span class="va">server</span>,</span>
+<span>                                                                user <span class="op">=</span> <span class="va">user</span>,</span>
+<span>                                                                password <span class="op">=</span> <span class="va">pw</span>,</span>
+<span>                                                                port <span class="op">=</span> <span class="va">port</span><span class="op">)</span></span></code></pre></div>
+<p>Next you need to specify the cdmDatabaseSchema where your cdm
+database is found and workDatabaseSchema where your target population
+and outcome cohorts are and you need to specify a label for the database
+name: a string with a shareable name of the database (this will be shown
+to OHDSI researchers if the results get transported).</p>
+<div class="sourceCode" id="cb5"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb5-1"><a href="#cb5-1" tabindex="-1"></a>cdmDatabaseSchema <span class="ot">&lt;-</span> <span class="st">"your cdmDatabaseSchema"</span></span>
+<span id="cb5-2"><a href="#cb5-2" tabindex="-1"></a>workDatabaseSchema <span class="ot">&lt;-</span> <span class="st">"your workDatabaseSchema"</span></span>
+<span id="cb5-3"><a href="#cb5-3" tabindex="-1"></a>cdmDatabaseName <span class="ot">&lt;-</span> <span class="st">"your cdmDatabaseName"</span></span>
+<span id="cb5-4"><a href="#cb5-4" tabindex="-1"></a>cohortTable <span class="ot">&lt;-</span> <span class="st">"your cohort table"</span>,</span>
+<span id="cb5-5"><a href="#cb5-5" tabindex="-1"></a></span>
+<span id="cb5-6"><a href="#cb5-6" tabindex="-1"></a>databaseDetails <span class="ot">&lt;-</span> <span class="fu">createDatabaseDetails</span>(</span>
+<span id="cb5-7"><a href="#cb5-7" tabindex="-1"></a>  <span class="at">connectionDetails =</span> connectionDetails, </span>
+<span id="cb5-8"><a href="#cb5-8" tabindex="-1"></a>  <span class="at">cdmDatabaseSchema =</span> cdmDatabaseSchema, </span>
+<span id="cb5-9"><a href="#cb5-9" tabindex="-1"></a>  <span class="at">cdmDatabaseName =</span> cdmDatabaseName , </span>
+<span id="cb5-10"><a href="#cb5-10" tabindex="-1"></a>  <span class="at">cohortDatabaseSchema =</span> workDatabaseSchema, </span>
+<span id="cb5-11"><a href="#cb5-11" tabindex="-1"></a>  <span class="at">cohortTable =</span> cohortTable, </span>
+<span id="cb5-12"><a href="#cb5-12" tabindex="-1"></a>  <span class="at">outcomeDatabaseSchema =</span> workDatabaseSchema, </span>
+<span id="cb5-13"><a href="#cb5-13" tabindex="-1"></a>  <span class="at">outcomeTable =</span> cohortTable </span>
+<span id="cb5-14"><a href="#cb5-14" tabindex="-1"></a>  <span class="at">cdmVersion =</span> <span class="dv">5</span></span>
+<span id="cb5-15"><a href="#cb5-15" tabindex="-1"></a>    )</span></code></pre></div>
 <p>Now you can run the multiple patient-level prediction analysis:</p>
 <div class="sourceCode" id="cb6"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">results</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/runMultiplePlp.html">runMultiplePlp</a></span><span class="op">(</span>
-  databaseDetails <span class="op">=</span> <span class="va">databaseDetails</span>, 
-  modelDesignList <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html">list</a></span><span class="op">(</span>
-    <span class="va">modelDesign1</span>, 
-    <span class="va">modelDesign2</span>, 
-    <span class="va">modelDesign3</span>
-    <span class="op">)</span>, 
-  onlyFetchData <span class="op">=</span> <span class="cn">F</span>, 
-  splitSettings <span class="op">=</span> <span class="fu"><a href="../reference/createDefaultSplitSetting.html">createDefaultSplitSetting</a></span><span class="op">(</span><span class="op">)</span>, 
-  logSettings <span class="op">=</span> <span class="fu"><a href="../reference/createLogSettings.html">createLogSettings</a></span><span class="op">(</span><span class="op">)</span>, 
-  saveDirectory <span class="op">=</span>  <span class="st">"./PlpMultiOutput"</span>
-  <span class="op">)</span></code></pre></div>
-<p>This will then save all the plpData objects from the study into “./PlpMultiOutput/plpData_T1_L<i>” and the results into “./PlpMultiOutput/Analysis_<i>”. The csv file named settings.csv found in “./PlpMultiOutput” has a row for each prediction model developed and points to the plpData and settings used for the model development, it also has descriptions of the cohorts if these are input by the user.</i></i></p>
-<p>Note that if for some reason the run is interrupted, e.g. because of an error, a new call to <code>runMultiplePlp</code> will continue and not restart until you remove the output folder.</p>
+<code class="sourceCode R"><span><span class="va">results</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/runMultiplePlp.html">runMultiplePlp</a></span><span class="op">(</span></span>
+<span>  databaseDetails <span class="op">=</span> <span class="va">databaseDetails</span>, </span>
+<span>  modelDesignList <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span></span>
+<span>    <span class="va">modelDesign1</span>, </span>
+<span>    <span class="va">modelDesign2</span>, </span>
+<span>    <span class="va">modelDesign3</span></span>
+<span>    <span class="op">)</span>, </span>
+<span>  onlyFetchData <span class="op">=</span> <span class="cn">F</span>, </span>
+<span>  logSettings <span class="op">=</span> <span class="fu"><a href="../reference/createLogSettings.html">createLogSettings</a></span><span class="op">(</span><span class="op">)</span>, </span>
+<span>  saveDirectory <span class="op">=</span>  <span class="st">"./PlpMultiOutput"</span></span>
+<span>  <span class="op">)</span></span></code></pre></div>
+<p>This will then save all the plpData objects from the study into
+“./PlpMultiOutput/plpData_T1_L<i>” and the results into
+“./PlpMultiOutput/Analysis_<i>”. The csv file named settings.csv found
+in “./PlpMultiOutput” has a row for each prediction model developed and
+points to the plpData and settings used for the model development, it
+also has descriptions of the cohorts if these are input by the user.</i></i></p>
+<p>Note that if for some reason the run is interrupted, e.g. because of
+an error, a new call to <code>runMultiplePlp</code> will continue and
+not restart until you remove the output folder.</p>
 </div>
-<div id="validating-multiple-models" class="section level1">
-<h1 class="hasAnchor">
-<a href="#validating-multiple-models" class="anchor"></a>Validating multiple models</h1>
-<p>If you have access to multiple databases on the same server in different schemas you could evaluate accross these using this call:</p>
+<div class="section level2">
+<h2 id="validating-multiple-models">Validating multiple models<a class="anchor" aria-label="anchor" href="#validating-multiple-models"></a>
+</h2>
+<p>If you have access to multiple databases on the same server in
+different schemas you could evaluate accross these using this call:</p>
 <div class="sourceCode" id="cb7"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">validationDatabaseDetails</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDatabaseDetails.html">createDatabaseDetails</a></span><span class="op">(</span>
-  connectionDetails <span class="op">=</span> <span class="va">connectionDetails</span>, 
-  cdmDatabaseSchema <span class="op">=</span> <span class="st">'new cdm schema'</span>, 
-  cdmDatabaseName <span class="op">=</span> <span class="st">'validation database'</span>, 
-  cohortDatabaseSchema <span class="op">=</span> <span class="va">workDatabaseSchema</span>, 
-  cohortTable <span class="op">=</span> <span class="va">cohortTable</span>, 
-  outcomeDatabaseSchema <span class="op">=</span> <span class="va">workDatabaseSchema</span>, 
-  outcomeTable <span class="op">=</span> <span class="va">cohortTable</span>, 
-  cdmVersion <span class="op">=</span> <span class="fl">5</span>
-  <span class="op">)</span>
-
-<span class="va">val</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/validateMultiplePlp.html">validateMultiplePlp</a></span><span class="op">(</span>
-  analysesLocation <span class="op">=</span> <span class="st">"./PlpMultiOutput"</span>,
-  valdiationDatabaseDetails <span class="op">=</span> <span class="va">validationDatabaseDetails</span>,
-  validationRestrictPlpDataSettings <span class="op">=</span> <span class="fu"><a href="../reference/createRestrictPlpDataSettings.html">createRestrictPlpDataSettings</a></span><span class="op">(</span><span class="op">)</span>,
-  recalibrate <span class="op">=</span> <span class="cn">NULL</span>,
-  saveDirectory <span class="op">=</span> <span class="st">"./PlpMultiOutput/validation"</span>
-  <span class="op">)</span></code></pre></div>
-<p>This then saves the external validation results in the validation folder of the main study (the outputLocation you used in runPlpAnalyses).</p>
+<code class="sourceCode R"><span><span class="va">validationDatabaseDetails</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDatabaseDetails.html">createDatabaseDetails</a></span><span class="op">(</span></span>
+<span>  connectionDetails <span class="op">=</span> <span class="va">connectionDetails</span>, </span>
+<span>  cdmDatabaseSchema <span class="op">=</span> <span class="st">'new cdm schema'</span>, </span>
+<span>  cdmDatabaseName <span class="op">=</span> <span class="st">'validation database'</span>, </span>
+<span>  cohortDatabaseSchema <span class="op">=</span> <span class="va">workDatabaseSchema</span>, </span>
+<span>  cohortTable <span class="op">=</span> <span class="va">cohortTable</span>, </span>
+<span>  outcomeDatabaseSchema <span class="op">=</span> <span class="va">workDatabaseSchema</span>, </span>
+<span>  outcomeTable <span class="op">=</span> <span class="va">cohortTable</span>, </span>
+<span>  cdmVersion <span class="op">=</span> <span class="fl">5</span></span>
+<span>  <span class="op">)</span></span>
+<span></span>
+<span><span class="va">val</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/validateMultiplePlp.html">validateMultiplePlp</a></span><span class="op">(</span></span>
+<span>  analysesLocation <span class="op">=</span> <span class="st">"./PlpMultiOutput"</span>,</span>
+<span>  valdiationDatabaseDetails <span class="op">=</span> <span class="va">validationDatabaseDetails</span>,</span>
+<span>  validationRestrictPlpDataSettings <span class="op">=</span> <span class="fu"><a href="../reference/createRestrictPlpDataSettings.html">createRestrictPlpDataSettings</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  recalibrate <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  saveDirectory <span class="op">=</span> <span class="st">"./PlpMultiOutput/Validation"</span></span>
+<span>  <span class="op">)</span></span></code></pre></div>
+<p>This then saves the external validation results in the
+<code>Validation</code> folder of the main study (the outputLocation you
+used in runPlpAnalyses).</p>
 </div>
-<div id="viewing-the-results" class="section level1">
-<h1 class="hasAnchor">
-<a href="#viewing-the-results" class="anchor"></a>Viewing the results</h1>
+<div class="section level2">
+<h2 id="viewing-the-results">Viewing the results<a class="anchor" aria-label="anchor" href="#viewing-the-results"></a>
+</h2>
 <p>To view the results for the multiple prediction analysis:</p>
 <div class="sourceCode" id="cb8"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="../reference/viewMultiplePlp.html">viewMultiplePlp</a></span><span class="op">(</span>analysesLocation<span class="op">=</span><span class="st">"./PlpMultiOutput"</span><span class="op">)</span></code></pre></div>
-<p>If the validation directory in “./PlpMultiOutput” has results, the external validation will also be displayed.</p>
+<code class="sourceCode R"><span><span class="fu"><a href="../reference/viewMultiplePlp.html">viewMultiplePlp</a></span><span class="op">(</span>analysesLocation<span class="op">=</span><span class="st">"./PlpMultiOutput"</span><span class="op">)</span></span></code></pre></div>
+<p>If the validation directory in “./PlpMultiOutput” has a sqlite
+results database, the external validation will also be displayed.</p>
 </div>
-<div id="acknowledgments" class="section level1">
-<h1 class="hasAnchor">
-<a href="#acknowledgments" class="anchor"></a>Acknowledgments</h1>
-<p>Considerable work has been dedicated to provide the <code>PatientLevelPrediction</code> package.</p>
+<div class="section level2">
+<h2 id="acknowledgments">Acknowledgments<a class="anchor" aria-label="anchor" href="#acknowledgments"></a>
+</h2>
+<p>Considerable work has been dedicated to provide the
+<code>PatientLevelPrediction</code> package.</p>
 <div class="sourceCode" id="cb9"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/utils/citation.html">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></code></pre></div>
-<pre><code>## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design
-## and implementation of a standardized framework to generate and evaluate
-## patient-level prediction models using observational healthcare data."
-## _Journal of the American Medical Informatics Association_, *25*(8),
-## 969-975. &lt;URL: https://doi.org/10.1093/jamia/ocy032&gt;.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }</code></pre>
-<p><strong>Please reference this paper if you use the PLP Package in your work:</strong></p>
-<p><a href="http://dx.doi.org/10.1093/jamia/ocy032">Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.</a></p>
+<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/utils/citation.html" class="external-link">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></span></code></pre></div>
+<pre><code><span><span class="co">## </span></span>
+<span><span class="co">## To cite PatientLevelPrediction in publications use:</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">##   Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design</span></span>
+<span><span class="co">##   and implementation of a standardized framework to generate and</span></span>
+<span><span class="co">##   evaluate patient-level prediction models using observational</span></span>
+<span><span class="co">##   healthcare data." _Journal of the American Medical Informatics</span></span>
+<span><span class="co">##   Association_, *25*(8), 969-975.</span></span>
+<span><span class="co">##   &lt;https://doi.org/10.1093/jamia/ocy032&gt;.</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">## A BibTeX entry for LaTeX users is</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">##   @Article{,</span></span>
+<span><span class="co">##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},</span></span>
+<span><span class="co">##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},</span></span>
+<span><span class="co">##     journal = {Journal of the American Medical Informatics Association},</span></span>
+<span><span class="co">##     volume = {25},</span></span>
+<span><span class="co">##     number = {8},</span></span>
+<span><span class="co">##     pages = {969-975},</span></span>
+<span><span class="co">##     year = {2018},</span></span>
+<span><span class="co">##     url = {https://doi.org/10.1093/jamia/ocy032},</span></span>
+<span><span class="co">##   }</span></span></code></pre>
+<p><strong>Please reference this paper if you use the PLP Package in
+your work:</strong></p>
+<p><a href="http://dx.doi.org/10.1093/jamia/ocy032" class="external-link">Reps JM, Schuemie
+MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a
+standardized framework to generate and evaluate patient-level prediction
+models using observational healthcare data. J Am Med Inform Assoc.
+2018;25(8):969-975.</a></p>
 </div>
   </div>
 
@@ -464,11 +546,13 @@ <h1 class="hasAnchor">
 
 
       <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+  <p></p>
+<p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p>
+<p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
       </footer>
@@ -477,5 +561,7 @@ <h1 class="hasAnchor">
   
 
 
+  
+
   </body>
 </html>
diff --git a/docs/articles/BuildingMultiplePredictiveModels_files/header-attrs-2.11/header-attrs.js b/docs/articles/BuildingMultiplePredictiveModels_files/header-attrs-2.11/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/BuildingMultiplePredictiveModels_files/header-attrs-2.11/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/BuildingMultiplePredictiveModels_files/header-attrs-2.7/header-attrs.js b/docs/articles/BuildingMultiplePredictiveModels_files/header-attrs-2.7/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/BuildingMultiplePredictiveModels_files/header-attrs-2.7/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/BuildingPredictiveModels.html b/docs/articles/BuildingPredictiveModels.html
index 1cc607322..4e13c99f2 100644
--- a/docs/articles/BuildingPredictiveModels.html
+++ b/docs/articles/BuildingPredictiveModels.html
@@ -19,6 +19,8 @@
 <![endif]-->
 </head>
 <body data-spy="scroll" data-target="#toc">
+    
+
     <div class="container template-article">
       <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
@@ -31,7 +33,7 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
@@ -53,7 +55,7 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
@@ -72,10 +74,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -84,13 +86,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -101,21 +106,30 @@
   </ul>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
       </ul>
 <ul class="nav navbar-nav navbar-right">
 <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
@@ -130,15 +144,16 @@
 
       
 
-      </header><script src="BuildingPredictiveModels_files/header-attrs-2.11/header-attrs.js"></script><div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header toc-ignore">
       <h1 data-toc-skip>Building patient-level predictive models</h1>
-                        <h4 class="author">Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek</h4>
+                        <h4 data-toc-skip class="author">Jenna Reps,
+Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek</h4>
             
-            <h4 class="date">2022-03-09</h4>
+            <h4 data-toc-skip class="date">2024-09-09</h4>
       
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/BuildingPredictiveModels.Rmd"><code>vignettes/BuildingPredictiveModels.Rmd</code></a></small>
+      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/vignettes/BuildingPredictiveModels.Rmd" class="external-link"><code>vignettes/BuildingPredictiveModels.Rmd</code></a></small>
       <div class="hidden name"><code>BuildingPredictiveModels.Rmd</code></div>
 
     </div>
@@ -146,65 +161,224 @@ <h4 class="date">2022-03-09</h4>
     
     
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Building patient-level predictive models}
 -->
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
-<p>Observational healthcare data, such as administrative claims and electronic health records, are increasingly used for clinical characterization of disease progression, quality improvement, and population-level effect estimation for medical product safety surveillance and comparative effectiveness. Advances in machine learning for large dataset analysis have led to increased interest in applying patient-level prediction on this type of data. Patient-level prediction offers the potential for medical practice to move beyond average treatment effects and to consider personalized risks as part of clinical decision-making. However, many published efforts in patient-level-prediction do not follow the model development guidelines, fail to perform extensive external validation, or provide insufficient model details that limits the ability of independent researchers to reproduce the models and perform external validation. This makes it hard to fairly evaluate the predictive performance of the models and reduces the likelihood of the model being used appropriately in clinical practice. To improve standards, several papers have been written detailing guidelines for best practices in developing and reporting prediction models.</p>
-<p>The Transparent Reporting of a multivariable prediction model for <a href="https://www.equator-network.org/reporting-guidelines/tripod-statement/"><code>Individual Prognosis Or Diagnosis (TRIPOD) statement</code></a> provides clear recommendations for reporting prediction model development and validation and addresses some of the concerns related to transparency. However, data structure heterogeneity and inconsistent terminologies still make collaboration and model sharing difficult as different researchers are often required to write new code to extract the data from their databases and may define variables differently.</p>
-<p>In our <a href="https://academic.oup.com/jamia/article/25/8/969/4989437"><code>paper</code></a>, we propose a standardised framework for patient-level prediction that utilizes the OMOP Common Data Model (CDM) and standardized vocabularies, and describe the open-source software that we developed implementing the framework’s pipeline. The framework is the first to support existing best practice guidelines and will enable open dissemination of models that can be extensively validated across the network of OHDSI collaborators.</p>
-<p>Figure 1, illustrates the prediction problem we address. Among a population at risk, we aim to predict which patients at a defined moment in time (t = 0) will experience some outcome during a time-at-risk. Prediction is done using only information about the patients in an observation window prior to that moment in time.</p>
-<div class="figure">
-<img src="Figure1.png" alt=""><p class="caption">The prediction problem</p>
-</div>
-<p>As shown in Figure 2, to define a prediction problem we have to define t=0 by a Target Cohort (T), the outcome we like to predict by an outcome cohort (O), and the time-at-risk (TAR). Furthermore, we have to make design choices for the model we like to develop, and determine the observational datasets to perform internal and external validation. This conceptual framework works for all type of prediction problems, for example those presented in Figure 3.</p>
-<div class="figure">
-<img src="studydesign.png" alt=""><p class="caption">Design choices</p>
-</div>
-<div class="figure">
-<img src="problems.png" alt=""><p class="caption">Examples of prediction problems</p>
-</div>
-<p>This vignette describes how you can use the <code>PatientLevelPrediction</code> package to build patient-level predictive models. The package enables data extraction, model building, and model evaluation using data from databases that are translated into the OMOP CDM. In this vignette we assume you have installed the package correctly using the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/InstallationGuide.pdf"><code>InstallationGuide</code></a>.</p>
-</div>
-<div id="study-specification" class="section level1">
-<h1 class="hasAnchor">
-<a href="#study-specification" class="anchor"></a>Study specification</h1>
-<p>We have to clearly specify our study upfront to be able to implement it. This means we need to define the prediction problem we like to address, in which population we will build the model, which model we will build and how we will evaluate its performance. To guide you through this process we will use a “Disease onset and progression” prediction type as an example.</p>
-<div id="problem-definition-1-stroke-in-afibrilation-patients" class="section level2">
-<h2 class="hasAnchor">
-<a href="#problem-definition-1-stroke-in-afibrilation-patients" class="anchor"></a>Problem definition 1: Stroke in afibrilation patients</h2>
-<p>Atrial fibrillation is a disease characterized by an irregular heart rate that can cause poor blood flow. Patients with atrial fibrillation are at increased risk of ischemic stroke. Anticoagulation is a recommended prophylaxis treatment strategy for patients at high risk of stroke, though the underuse of anticoagulants and persistent severity of ischemic stroke represents a substantial unmet medical need. Various strategies have been developed to predict risk of ischemic stroke in patients with atrial fibrillation. CHADS2 (Gage JAMA 2001) was developed as a risk score based on history of congestive heart failure, hypertension, age&gt;=75, diabetes and stroke. CHADS2 was initially derived using Medicare claims data, where it achieved good discrimination (AUC=0.82). However, subsequent external validation studies revealed the CHADS2 had substantially lower predictive accuracy (Keogh Thromb Haemost 2011). Subsequent stroke risk calculators have been developed and evaluated, including the extension of CHADS2Vasc. The management of atrial fibrillation has evolved substantially over the last decade, for various reasons that include the introduction of novel oral anticoagulants. With these innovations has come a renewed interest in greater precision medicine for stroke prevention.</p>
-<p>We will apply the PatientLevelPrediction package to observational healthcare data to address the following patient-level prediction question:</p>
-<p>Amongst patients who are newly diagnosed with Atrial Fibrillation, which patients will go on to have Ischemic Stroke within 1 year?</p>
-<p>We will define ‘patients who are newly diagnosed with Atrial Fibrillation’ as the first condition record of cardiac arrhythmia, which is followed by another cardiac arrhythmia condition record, at least two drug records for a drug used to treat arrhythmias, or a procedure to treat arrhythmias. We will define ‘Ischemic stroke events’ as ischemic stroke condition records during an inpatient or ER visit; successive records with &gt; 180 day gap are considered independent episodes.</p>
-</div>
-<div id="problem-definition-2-angioedema-in-ace-inhibitor-users" class="section level2">
-<h2 class="hasAnchor">
-<a href="#problem-definition-2-angioedema-in-ace-inhibitor-users" class="anchor"></a>Problem definition 2: Angioedema in ACE inhibitor users</h2>
-<p>Angiotensin converting enzyme inhibitors (ACE inhibitors) are medications used by patients with hypertension that widen the blood vessles and therefore increse the amount of blood pumped by the heart and decreases blood pressure. Ace inhibitors reduce a patients risk of cardiovasular disease but can lead to drug-induced angioedema.</p>
-<p>We will apply the PatientLevelPrediction package to observational healthcare data to address the following patient-level prediction question:</p>
-<p>Amongst patients who are newly dispensed an ACE inhibitor, which patients will go on to have angioedema within 1 year?</p>
-<p>We will define ‘patients who are newly dispensed an ACE inhibitor’ as the first drug record of sny ACE inhibitor, […]which is followed by another cardiac arrhythmia condition record, at least two drug records for a drug used to treat arrhythmias, or a procedure to treat arrhythmias. We will define ‘angioedema’ as an angioedema condition record.</p>
-</div>
-<div id="study-population-definition" class="section level2">
-<h2 class="hasAnchor">
-<a href="#study-population-definition" class="anchor"></a>Study population definition</h2>
-<p>The final study population in which we will develop our model is often a subset of the Target population, because we will e.g. apply criteria that are dependent on T and O or we want to do sensitivity analyses with subpopulations of T. For this we have to answer the following questions:</p>
+<div class="section level2">
+<h2 id="introduction">Introduction<a class="anchor" aria-label="anchor" href="#introduction"></a>
+</h2>
+<p>Observational healthcare data, such as administrative claims and
+electronic health records, are increasingly used for clinical
+characterization of disease progression, quality improvement, and
+population-level effect estimation for medical product safety
+surveillance and comparative effectiveness. Advances in machine learning
+for large dataset analysis have led to increased interest in applying
+patient-level prediction on this type of data. Patient-level prediction
+offers the potential for medical practice to move beyond average
+treatment effects and to consider personalized risks as part of clinical
+decision-making. However, many published efforts in
+patient-level-prediction do not follow the model development guidelines,
+fail to perform extensive external validation, or provide insufficient
+model details that limits the ability of independent researchers to
+reproduce the models and perform external validation. This makes it hard
+to fairly evaluate the predictive performance of the models and reduces
+the likelihood of the model being used appropriately in clinical
+practice. To improve standards, several papers have been written
+detailing guidelines for best practices in developing and reporting
+prediction models.</p>
+<p>The Transparent Reporting of a multivariable prediction model for <a href="https://www.equator-network.org/reporting-guidelines/tripod-statement/" class="external-link"><code>Individual Prognosis Or Diagnosis (TRIPOD) statement</code></a>
+provides clear recommendations for reporting prediction model
+development and validation and addresses some of the concerns related to
+transparency. However, data structure heterogeneity and inconsistent
+terminologies still make collaboration and model sharing difficult as
+different researchers are often required to write new code to extract
+the data from their databases and may define variables differently.</p>
+<p>In our <a href="https://academic.oup.com/jamia/article/25/8/969/4989437" class="external-link"><code>paper</code></a>,
+we propose a standardised framework for patient-level prediction that
+utilizes the OMOP Common Data Model (CDM) and standardized vocabularies,
+and describe the open-source software that we developed implementing the
+framework’s pipeline. The framework is the first to support existing
+best practice guidelines and will enable open dissemination of models
+that can be extensively validated across the network of OHDSI
+collaborators.</p>
+<p>Figure 1, illustrates the prediction problem we address. Among a
+population at risk, we aim to predict which patients at a defined moment
+in time (t = 0) will experience some outcome during a time-at-risk.
+Prediction is done using only information about the patients in an
+observation window prior to that moment in time.</p>
+<div class="float">
+<img src="Figure1.webp" alt="The prediction problem"><div class="figcaption">The prediction problem</div>
+</div>
+<p>As shown in Figure 2, to define a prediction problem we have to
+define t=0 by a Target Cohort (T), the outcome we like to predict by an
+outcome cohort (O), and the time-at-risk (TAR). Furthermore, we have to
+make design choices for the model we like to develop, and determine the
+observational datasets to perform internal and external validation. This
+conceptual framework works for all type of prediction problems, for
+example those presented in Figure 3.</p>
+<div class="float">
+<img src="studydesign.webp" alt="Design choices"><div class="figcaption">Design choices</div>
+</div>
+<div class="float">
+<img src="problems.webp" alt="Examples of prediction problems"><div class="figcaption">Examples of prediction problems</div>
+</div>
+<p>This vignette describes how you can use the
+<code>PatientLevelPrediction</code> package to build patient-level
+predictive models. The package enables data extraction, model building,
+and model evaluation using data from databases that are translated into
+the OMOP CDM. In this vignette we assume you have installed the package
+correctly using the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/InstallationGuide.pdf" class="external-link"><code>InstallationGuide</code></a>.</p>
+</div>
+<div class="section level2">
+<h2 id="study-specification">Study specification<a class="anchor" aria-label="anchor" href="#study-specification"></a>
+</h2>
+<p>We have to clearly specify our study upfront to be able to implement
+it. This means we need to define the prediction problem we like to
+address, in which population we will build the model, which model we
+will build and how we will evaluate its performance. To guide you
+through this process we will use a “Disease onset and progression”
+prediction type as an example.</p>
+<div class="section level3">
+<h3 id="problem-definition-1-stroke-in-afibrilation-patients">Problem definition 1: Stroke in afibrilation patients<a class="anchor" aria-label="anchor" href="#problem-definition-1-stroke-in-afibrilation-patients"></a>
+</h3>
+<p>Atrial fibrillation is a disease characterized by an irregular heart
+rate that can cause poor blood flow. Patients with atrial fibrillation
+are at increased risk of ischemic stroke. Anticoagulation is a
+recommended prophylaxis treatment strategy for patients at high risk of
+stroke, though the underuse of anticoagulants and persistent severity of
+ischemic stroke represents a substantial unmet medical need. Various
+strategies have been developed to predict risk of ischemic stroke in
+patients with atrial fibrillation. CHADS2 (Gage JAMA 2001) was developed
+as a risk score based on history of congestive heart failure,
+hypertension, age&gt;=75, diabetes and stroke. CHADS2 was initially
+derived using Medicare claims data, where it achieved good
+discrimination (AUC=0.82). However, subsequent external validation
+studies revealed the CHADS2 had substantially lower predictive accuracy
+(Keogh Thromb Haemost 2011). Subsequent stroke risk calculators have
+been developed and evaluated, including the extension of CHADS2Vasc. The
+management of atrial fibrillation has evolved substantially over the
+last decade, for various reasons that include the introduction of novel
+oral anticoagulants. With these innovations has come a renewed interest
+in greater precision medicine for stroke prevention.</p>
+<p>We will apply the PatientLevelPrediction package to observational
+healthcare data to address the following patient-level prediction
+question:</p>
+<p>Amongst patients who are newly diagnosed with Atrial Fibrillation,
+which patients will go on to have Ischemic Stroke within 1 year?</p>
+<p>We will define ‘patients who are newly diagnosed with Atrial
+Fibrillation’ as the first condition record of cardiac arrhythmia, which
+is followed by another cardiac arrhythmia condition record, at least two
+drug records for a drug used to treat arrhythmias, or a procedure to
+treat arrhythmias. We will define ‘Ischemic stroke events’ as ischemic
+stroke condition records during an inpatient or ER visit; successive
+records with &gt; 180 day gap are considered independent episodes.</p>
+</div>
+<div class="section level3">
+<h3 id="problem-definition-2-angioedema-in-ace-inhibitor-users">Problem definition 2: Angioedema in ACE inhibitor users<a class="anchor" aria-label="anchor" href="#problem-definition-2-angioedema-in-ace-inhibitor-users"></a>
+</h3>
+<p>Angiotensin converting enzyme inhibitors (ACE inhibitors) are
+medications used by patients with hypertension that widen the blood
+vessles and therefore increse the amount of blood pumped by the heart
+and decreases blood pressure. Ace inhibitors reduce a patients risk of
+cardiovasular disease but can lead to drug-induced angioedema.</p>
+<p>We will apply the PatientLevelPrediction package to observational
+healthcare data to address the following patient-level prediction
+question:</p>
+<p>Amongst patients who are newly dispensed an ACE inhibitor, which
+patients will go on to have angioedema within 1 year?</p>
+<p>We will define ‘patients who are newly dispensed an ACE inhibitor’ as
+the first drug record of sny ACE inhibitor, […]which is followed by
+another cardiac arrhythmia condition record, at least two drug records
+for a drug used to treat arrhythmias, or a procedure to treat
+arrhythmias. We will define ‘angioedema’ as an angioedema condition
+record.</p>
+</div>
+<div class="section level3">
+<h3 id="study-population-definition">Study population definition<a class="anchor" aria-label="anchor" href="#study-population-definition"></a>
+</h3>
+<p>The final study population in which we will develop our model is
+often a subset of the Target population, because we will e.g. apply
+criteria that are dependent on T and O or we want to do sensitivity
+analyses with subpopulations of T. For this we have to answer the
+following questions:</p>
 <ul>
-<li><p><em>What is the minimum amount of observation time we require before the start of the target cohort?</em> This choice could depend on the available patient time in your training data, but also on the time you expect to be available in the data sources you want to apply the model on in the future. The longer the minimum observation time, the more baseline history time is available for each person to use for feature extraction, but the fewer patients will qualify for analysis. Moreover, there could be clinical reasons to choose a short or longer lookback period. For our example, we will use a prior history as lookback period (washout period).</p></li>
-<li><p><em>Can patients enter the target cohort multiple times?</em> In the target cohort definition, a person may qualify for the cohort multiple times during different spans of time, for example if they had different episodes of a disease or separate periods of exposure to a medical product. The cohort definition does not necessarily apply a restriction to only let the patients enter once, but in the context of a particular patient-level prediction problem, a user may want to restrict the cohort to the first qualifying episode. In our example, a person could only enter the target cohort once since our criteria was based on first occurrence of atrial fibrillation.</p></li>
-<li><p><em>Do we allow persons to enter the cohort if they experienced the outcome before?</em> Do we allow persons to enter the target cohort if they experienced the outcome before qualifying for the target cohort? Depending on the particular patient-level prediction problem, there may be a desire to predict ‘incident’ first occurrence of an outcome, in which case patients who have previously experienced the outcome are not ‘at-risk’ for having a first occurrence and therefore should be excluded from the target cohort. In other circumstances, there may be a desire to predict ‘prevalent’ episodes, whereby patients with prior outcomes can be included in the analysis and the prior outcome itself can be a predictor of future outcomes. For our prediction example, the answer to this question is ‘Yes, allow persons with prior outcomes’ because we know from the CHADS2 score that prior strokes are very predictive of future strokes. If this answer would have been ‘No’ we also have to decide how long we would look back for previous occurrences of the outcome.</p></li>
-<li><p><em>How do we define the period in which we will predict our outcome relative to the target cohort start?</em> We actually have to make two decisions to answer that question. First, does the time-at-risk window start at the date of the start of the target cohort or later? Arguments to make it start later could be that you want to avoid outcomes that were entered late in the record that actually occurred before the start of the target cohort or you want to leave a gap where interventions to prevent the outcome could theoretically be implemented. Second, you need to define the time-at-risk by setting the risk window end, as some specification of days offset relative to the target cohort start or end dates. For our problem we will predict in a ‘time-at-risk’ window starting 1 day after the start of the target cohort up to 365 days later (to look for 1-year risk following atrial fibrillation diagnosis).</p></li>
-<li><p><em>Do we require a minimum amount of time-at-risk?</em> We have to decide if we want to include patients that did not experience the outcome but did leave the database earlier than the end of our time-at-risk period. These patients may experience the outcome when we do not observe them. For our prediction problem we decide to answer this question with ‘Yes, require a mimimum time-at-risk’ for that reason. Furthermore, we have to decide if this constraint also applies to persons who experienced the outcome or we will include all persons with the outcome irrespective of their total time at risk. For example, if the outcome is death, then persons with the outcome are likely censored before the full time-at-risk period is complete.</p></li>
+<li><p><em>What is the minimum amount of observation time we require
+before the start of the target cohort?</em> This choice could depend on
+the available patient time in your training data, but also on the time
+you expect to be available in the data sources you want to apply the
+model on in the future. The longer the minimum observation time, the
+more baseline history time is available for each person to use for
+feature extraction, but the fewer patients will qualify for analysis.
+Moreover, there could be clinical reasons to choose a short or longer
+lookback period. For our example, we will use a prior history as
+lookback period (washout period).</p></li>
+<li><p><em>Can patients enter the target cohort multiple times?</em> In
+the target cohort definition, a person may qualify for the cohort
+multiple times during different spans of time, for example if they had
+different episodes of a disease or separate periods of exposure to a
+medical product. The cohort definition does not necessarily apply a
+restriction to only let the patients enter once, but in the context of a
+particular patient-level prediction problem, a user may want to restrict
+the cohort to the first qualifying episode. In our example, a person
+could only enter the target cohort once since our criteria was based on
+first occurrence of atrial fibrillation.</p></li>
+<li><p><em>Do we allow persons to enter the cohort if they experienced
+the outcome before?</em> Do we allow persons to enter the target cohort
+if they experienced the outcome before qualifying for the target cohort?
+Depending on the particular patient-level prediction problem, there may
+be a desire to predict ‘incident’ first occurrence of an outcome, in
+which case patients who have previously experienced the outcome are not
+‘at-risk’ for having a first occurrence and therefore should be excluded
+from the target cohort. In other circumstances, there may be a desire to
+predict ‘prevalent’ episodes, whereby patients with prior outcomes can
+be included in the analysis and the prior outcome itself can be a
+predictor of future outcomes. For our prediction example, the answer to
+this question is ‘Yes, allow persons with prior outcomes’ because we
+know from the CHADS2 score that prior strokes are very predictive of
+future strokes. If this answer would have been ‘No’ we also have to
+decide how long we would look back for previous occurrences of the
+outcome.</p></li>
+<li><p><em>How do we define the period in which we will predict our
+outcome relative to the target cohort start?</em> We actually have to
+make two decisions to answer that question. First, does the time-at-risk
+window start at the date of the start of the target cohort or later?
+Arguments to make it start later could be that you want to avoid
+outcomes that were entered late in the record that actually occurred
+before the start of the target cohort or you want to leave a gap where
+interventions to prevent the outcome could theoretically be implemented.
+Second, you need to define the time-at-risk by setting the risk window
+end, as some specification of days offset relative to the target cohort
+start or end dates. For our problem we will predict in a ‘time-at-risk’
+window starting 1 day after the start of the target cohort up to 365
+days later (to look for 1-year risk following atrial fibrillation
+diagnosis).</p></li>
+<li><p><em>Do we require a minimum amount of time-at-risk?</em> We have
+to decide if we want to include patients that did not experience the
+outcome but did leave the database earlier than the end of our
+time-at-risk period. These patients may experience the outcome when we
+do not observe them. For our prediction problem we decide to answer this
+question with ‘Yes, require a mimimum time-at-risk’ for that reason.
+Furthermore, we have to decide if this constraint also applies to
+persons who experienced the outcome or we will include all persons with
+the outcome irrespective of their total time at risk. For example, if
+the outcome is death, then persons with the outcome are likely censored
+before the full time-at-risk period is complete.</p></li>
 </ul>
 </div>
-<div id="model-development-settings" class="section level2">
-<h2 class="hasAnchor">
-<a href="#model-development-settings" class="anchor"></a>Model development settings</h2>
-<p>To develop the model we have to decide which algorithm(s) we like to train. We see the selection of the best algorithm for a certain prediction problem as an empirical question, i.e. you need to let the data speak for itself and try different approaches to find the best one. There is no algorithm that will work best for all problems (no free lunch). In our package we therefore aim to implement many algorithms. Furthermore, we made the system modular so you can add your own custom algorithms as described in more detail in the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomModels.pdf"><code>AddingCustomModels</code></a> vignette.</p>
-<p>Our package currently contains the following algorithms to choose from:</p>
+<div class="section level3">
+<h3 id="model-development-settings">Model development settings<a class="anchor" aria-label="anchor" href="#model-development-settings"></a>
+</h3>
+<p>To develop the model we have to decide which algorithm(s) we like to
+train. We see the selection of the best algorithm for a certain
+prediction problem as an empirical question, i.e. you need to let the
+data speak for itself and try different approaches to find the best one.
+There is no algorithm that will work best for all problems (no free
+lunch). In our package we therefore aim to implement many algorithms.
+Furthermore, we made the system modular so you can add your own custom
+algorithms as described in more detail in the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomModels.pdf" class="external-link"><code>AddingCustomModels</code></a>
+vignette.</p>
+<p>Our package currently contains the following algorithms to choose
+from:</p>
 <table class="table">
 <colgroup>
 <col width="11%">
@@ -219,77 +393,168 @@ <h2 class="hasAnchor">
 <tbody>
 <tr class="odd">
 <td>Regularized Logistic Regression</td>
-<td>Lasso logistic regression belongs to the family of generalized linear models, where a linear combination of the variables is learned and finally a logistic function maps the linear combination to a value between 0 and 1. The lasso regularization adds a cost based on model complexity to the objective function when training the model. This cost is the sum of the absolute values of the linear combination of the coefficients. The model automatically performs feature selection by minimizing this cost. We use the Cyclic coordinate descent for logistic, Poisson and survival analysis (Cyclops) package to perform large-scale regularized logistic regression: <a href="https://github.com/OHDSI/Cyclops" class="uri">https://github.com/OHDSI/Cyclops</a>
+<td>Lasso logistic regression belongs to the family of generalized
+linear models, where a linear combination of the variables is learned
+and finally a logistic function maps the linear combination to a value
+between 0 and 1. The lasso regularization adds a cost based on model
+complexity to the objective function when training the model. This cost
+is the sum of the absolute values of the linear combination of the
+coefficients. The model automatically performs feature selection by
+minimizing this cost. We use the Cyclic coordinate descent for logistic,
+Poisson and survival analysis (Cyclops) package to perform large-scale
+regularized logistic regression: <a href="https://github.com/OHDSI/Cyclops" class="external-link uri">https://github.com/OHDSI/Cyclops</a>
 </td>
 <td>var (starting variance), seed</td>
 </tr>
 <tr class="even">
 <td>Gradient boosting machines</td>
-<td>Gradient boosting machines is a boosting ensemble technique and in our framework it combines multiple decision trees. Boosting works by iteratively adding decision trees but adds more weight to the data-points that are misclassified by prior decision trees in the cost function when training the next tree. We use Extreme Gradient Boosting, which is an efficient implementation of the gradient boosting framework implemented in the xgboost R package available from CRAN.</td>
-<td>ntree (number of trees), max depth (max levels in tree), min rows (minimum data points in in node), learning rate, balance (balance class labels), seed</td>
+<td>Gradient boosting machines is a boosting ensemble technique and in
+our framework it combines multiple decision trees. Boosting works by
+iteratively adding decision trees but adds more weight to the
+data-points that are misclassified by prior decision trees in the cost
+function when training the next tree. We use Extreme Gradient Boosting,
+which is an efficient implementation of the gradient boosting framework
+implemented in the xgboost R package available from CRAN.</td>
+<td>ntree (number of trees), max depth (max levels in tree), min rows
+(minimum data points in in node), learning rate, balance (balance class
+labels), seed</td>
 </tr>
 <tr class="odd">
 <td>Random forest</td>
-<td>Random forest is a bagging ensemble technique that combines multiple decision trees. The idea behind bagging is to reduce the likelihood of overfitting, by using weak classifiers, but combining multiple diverse weak classifiers into a strong classifier. Random forest accomplishes this by training multiple decision trees but only using a subset of the variables in each tree and the subset of variables differ between trees. Our packages uses the sklearn learn implementation of Random Forest in python.</td>
-<td>mtry (number of features in each tree),ntree (number of trees), maxDepth (max levels in tree), minRows (minimum data points in in node),balance (balance class labels), seed</td>
+<td>Random forest is a bagging ensemble technique that combines multiple
+decision trees. The idea behind bagging is to reduce the likelihood of
+overfitting, by using weak classifiers, but combining multiple diverse
+weak classifiers into a strong classifier. Random forest accomplishes
+this by training multiple decision trees but only using a subset of the
+variables in each tree and the subset of variables differ between trees.
+Our packages uses the sklearn learn implementation of Random Forest in
+python.</td>
+<td>mtry (number of features in each tree),ntree (number of trees),
+maxDepth (max levels in tree), minRows (minimum data points in in
+node),balance (balance class labels), seed</td>
 </tr>
 <tr class="even">
 <td>K-nearest neighbors</td>
-<td>K-nearest neighbors (KNN) is an algorithm that uses some metric to find the K closest labelled data-points, given the specified metric, to a new unlabelled data-point. The prediction of the new data-points is then the most prevalent class of the K-nearest labelled data-points. There is a sharing limitation of KNN, as the model requires labelled data to perform the prediction on new data, and it is often not possible to share this data across data sites.We included the BigKnn classifier developed in OHDSI which is a large scale k-nearest neighbor classifier using the Lucene search engine: <a href="https://github.com/OHDSI/BigKnn" class="uri">https://github.com/OHDSI/BigKnn</a>
+<td>K-nearest neighbors (KNN) is an algorithm that uses some metric to
+find the K closest labelled data-points, given the specified metric, to
+a new unlabelled data-point. The prediction of the new data-points is
+then the most prevalent class of the K-nearest labelled data-points.
+There is a sharing limitation of KNN, as the model requires labelled
+data to perform the prediction on new data, and it is often not possible
+to share this data across data sites.We included the BigKnn classifier
+developed in OHDSI which is a large scale k-nearest neighbor classifier
+using the Lucene search engine: <a href="https://github.com/OHDSI/BigKnn" class="external-link uri">https://github.com/OHDSI/BigKnn</a>
 </td>
 <td>k (number of neighbours),weighted (weight by inverse frequency)</td>
 </tr>
 <tr class="odd">
 <td>Naive Bayes</td>
-<td>The Naive Bayes algorithm applies the Bayes theorem with the ‘naive’ assumption of conditional independence between every pair of features given the value of the class variable. Based on the likelihood the data belongs to a class and the prior distribution of the class, a posterior distribution is obtained.</td>
+<td>The Naive Bayes algorithm applies the Bayes theorem with the ‘naive’
+assumption of conditional independence between every pair of features
+given the value of the class variable. Based on the likelihood the data
+belongs to a class and the prior distribution of the class, a posterior
+distribution is obtained.</td>
 <td>none</td>
 </tr>
 <tr class="even">
 <td>AdaBoost</td>
-<td>AdaBoost is a boosting ensemble technique. Boosting works by iteratively adding classifiers but adds more weight to the data-points that are misclassified by prior classifiers in the cost function when training the next classifier. We use the sklearn ‘AdaboostClassifier’ implementation in Python.</td>
-<td>nEstimators (the maximum number of estimators at which boosting is terminated), learningRate (learning rate shrinks the contribution of each classifier by learning_rate. There is a trade-off between learningRate and nEstimators)</td>
+<td>AdaBoost is a boosting ensemble technique. Boosting works by
+iteratively adding classifiers but adds more weight to the data-points
+that are misclassified by prior classifiers in the cost function when
+training the next classifier. We use the sklearn ‘AdaboostClassifier’
+implementation in Python.</td>
+<td>nEstimators (the maximum number of estimators at which boosting is
+terminated), learningRate (learning rate shrinks the contribution of
+each classifier by learning_rate. There is a trade-off between
+learningRate and nEstimators)</td>
 </tr>
 <tr class="odd">
 <td>Decision Tree</td>
-<td>A decision tree is a classifier that partitions the variable space using individual tests selected using a greedy approach. It aims to find partitions that have the highest information gain to separate the classes. The decision tree can easily overfit by enabling a large number of partitions (tree depth) and often needs some regularization (e.g., pruning or specifying hyper-parameters that limit the complexity of the model). We use the sklearn ‘DecisionTreeClassifier’ implementation in Python.</td>
-<td>maxDepth (the maximum depth of the tree), minSamplesSplit,minSamplesLeaf, minImpuritySplit (threshold for early stopping in tree growth. A node will split if its impurity is above the threshold, otherwise it is a leaf.), seed,classWeight (‘Balance’ or ‘None’)</td>
+<td>A decision tree is a classifier that partitions the variable space
+using individual tests selected using a greedy approach. It aims to find
+partitions that have the highest information gain to separate the
+classes. The decision tree can easily overfit by enabling a large number
+of partitions (tree depth) and often needs some regularization (e.g.,
+pruning or specifying hyper-parameters that limit the complexity of the
+model). We use the sklearn ‘DecisionTreeClassifier’ implementation in
+Python.</td>
+<td>maxDepth (the maximum depth of the tree),
+minSamplesSplit,minSamplesLeaf, minImpuritySplit (threshold for early
+stopping in tree growth. A node will split if its impurity is above the
+threshold, otherwise it is a leaf.), seed,classWeight (‘Balance’ or
+‘None’)</td>
 </tr>
 <tr class="even">
 <td>Multilayer Perception</td>
-<td>Neural networks contain multiple layers that weight their inputs using a non-linear function. The first layer is the input layer, the last layer is the output layer the between are the hidden layers. Neural networks are generally trained using feed forward back-propagation. This is when you go through the network with a data-point and calculate the error between the true label and predicted label, then go backwards through the network and update the linear function weights based on the error. This can also be performed as a batch, where multiple data-points are fee</td>
-<td>size (the number of hidden nodes), alpha (the l2 regularisation), seed</td>
+<td>Neural networks contain multiple layers that weight their inputs
+using a non-linear function. The first layer is the input layer, the
+last layer is the output layer the between are the hidden layers. Neural
+networks are generally trained using feed forward back-propagation. This
+is when you go through the network with a data-point and calculate the
+error between the true label and predicted label, then go backwards
+through the network and update the linear function weights based on the
+error. This can also be performed as a batch, where multiple data-points
+are fee</td>
+<td>size (the number of hidden nodes), alpha (the l2 regularisation),
+seed</td>
 </tr>
 <tr class="odd">
-<td>Deep Learning (now in seperate DeepPatientLevelPrediction R package)</td>
-<td>Deep learning such as deep nets, convolutional neural networks or recurrent neural networks are similar to a neural network but have multiple hidden layers that aim to learn latent representations useful for prediction. In the seperate BuildingDeepLearningModels vignette we describe these models and hyper-parameters in more detail</td>
+<td>Deep Learning (now in seperate DeepPatientLevelPrediction R
+package)</td>
+<td>Deep learning such as deep nets, convolutional neural networks or
+recurrent neural networks are similar to a neural network but have
+multiple hidden layers that aim to learn latent representations useful
+for prediction. In the seperate BuildingDeepLearningModels vignette we
+describe these models and hyper-parameters in more detail</td>
 <td>see OHDSI/DeepPatientLevelPrediction</td>
 </tr>
 </tbody>
 </table>
-<p>Furthermore, we have to decide on the <strong>covariates</strong> that we will use to train our model. This choice can be driven by domain knowledge of available computational resources. In our example, we like to add the Gender, Age, Conditions, Drugs Groups, and Visit Count. We also have to specify in which time windows we will look and we decide to look in year before and any time prior.</p>
-<p>Finally, we have to define how we will train and test our model on our data, i.e. how we perform <strong>internal validation</strong>. For this we have to decide how we divide our dataset in a training and testing dataset and how we randomly assign patients to these two sets. Dependent on the size of the training set we can decide how much data we like to use for training, typically this is a 75%, 25% split. If you have very large datasets you can use more data for training. To randomly assign patients to the training and testing set, there are two commonly used approaches:</p>
+<p>Furthermore, we have to decide on the <strong>covariates</strong>
+that we will use to train our model. This choice can be driven by domain
+knowledge of available computational resources. In our example, we like
+to add the Gender, Age, Conditions, Drugs Groups, and Visit Count. We
+also have to specify in which time windows we will look and we decide to
+look in year before and any time prior.</p>
+<p>Finally, we have to define how we will train and test our model on
+our data, i.e. how we perform <strong>internal validation</strong>. For
+this we have to decide how we divide our dataset in a training and
+testing dataset and how we randomly assign patients to these two sets.
+Dependent on the size of the training set we can decide how much data we
+like to use for training, typically this is a 75%, 25% split. If you
+have very large datasets you can use more data for training. To randomly
+assign patients to the training and testing set, there are two commonly
+used approaches:</p>
 <ol style="list-style-type: decimal">
-<li>split by person. In this case a random seed is used to assign the patient to either sets.</li>
-<li>split by time. In this case a time point is used to split the persons, e.g. 75% of the data is before and 25% is after this date. The advantage of this is that you take into consideration that the health care system has changed over time.</li>
+<li>split by person. In this case a random seed is used to assign the
+patient to either sets.</li>
+<li>split by time. In this case a time point is used to split the
+persons, e.g. 75% of the data is before and 25% is after this date. The
+advantage of this is that you take into consideration that the health
+care system has changed over time.</li>
 </ol>
 <p>We now completely defined our studies and implement them:</p>
 <ul>
-<li><a href="#example1">See example 1: Stroke in afibrilation patients</a></li>
-<li><a href="#example2">See example 2: Agioedema in ACE inhibitor new users</a></li>
+<li><a href="#example1">See example 1: Stroke in afibrilation
+patients</a></li>
+<li><a href="#example2">See example 2: Agioedema in ACE inhibitor new
+users</a></li>
 </ul>
 </div>
 </div>
-<div id="example1" class="section level1">
-<h1 class="hasAnchor">
-<a href="#example1" class="anchor"></a>Example 1: Stroke in afibrilation patients</h1>
-<div id="study-specification-1" class="section level2">
-<h2 class="hasAnchor">
-<a href="#study-specification-1" class="anchor"></a>Study Specification</h2>
-<p>For our first prediction model we decide to start with a Regularized Logistic Regression and will use the default parameters. We will do a 75%-25% split by person.</p>
+<div class="section level2">
+<h2 id="example1">Example 1: Stroke in afibrilation patients<a class="anchor" aria-label="anchor" href="#example1"></a>
+</h2>
+<div class="section level3">
+<h3 id="study-specification-1">Study Specification<a class="anchor" aria-label="anchor" href="#study-specification-1"></a>
+</h3>
+<p>For our first prediction model we decide to start with a Regularized
+Logistic Regression and will use the default parameters. We will do a
+75%-25% split by person.</p>
 <table class="table">
 <colgroup>
-<col width="44%">
-<col width="55%">
+<col width="23%">
+<col width="76%">
 </colgroup>
 <thead><tr class="header">
 <th>Definition</th>
@@ -302,11 +567,17 @@ <h2 class="hasAnchor">
 </tr>
 <tr class="even">
 <td>Target Cohort (T)</td>
-<td>‘Patients who are newly diagnosed with Atrial Fibrillation’ defined as the first condition record of cardiac arrhythmia, which is followed by another cardiac arrhythmia condition record, at least two drug records for a drug used to treat arrhythmias, or a procedure to treat arrhythmias.</td>
+<td>‘Patients who are newly diagnosed with Atrial Fibrillation’ defined
+as the first condition record of cardiac arrhythmia, which is followed
+by another cardiac arrhythmia condition record, at least two drug
+records for a drug used to treat arrhythmias, or a procedure to treat
+arrhythmias.</td>
 </tr>
 <tr class="odd">
 <td>Outcome Cohort (O)</td>
-<td>‘Ischemic stroke events’ defined as ischemic stroke condition records during an inpatient or ER visit; successive records with &gt; 180 day gap are considered independent episodes.</td>
+<td>‘Ischemic stroke events’ defined as ischemic stroke condition
+records during an inpatient or ER visit; successive records with &gt;
+180 day gap are considered independent episodes.</td>
 </tr>
 <tr class="even">
 <td>Time-at-risk (TAR)</td>
@@ -362,7 +633,8 @@ <h2 class="hasAnchor">
 </tr>
 <tr class="odd">
 <td>Covariates</td>
-<td>Gender, Age, Conditions (ever before, &lt;365), Drugs Groups (ever before, &lt;365), and Visit Count</td>
+<td>Gender, Age, Conditions (ever before, &lt;365), Drugs Groups (ever
+before, &lt;365), and Visit Count</td>
 </tr>
 <tr class="even">
 <td>Data split</td>
@@ -370,358 +642,535 @@ <h2 class="hasAnchor">
 </tr>
 </tbody>
 </table>
-<p>According to the best practices we need to make a protocol that completely specifies how we plan to execute our study. This protocol will be assessed by the governance boards of the participating data sources in your network study. For this a template could be used but we prefer to automate this process as much as possible by adding functionality to automatically generate study protocol from a study specification. We will discuss this in more detail later.</p>
-</div>
-<div id="study-implementation" class="section level2">
-<h2 class="hasAnchor">
-<a href="#study-implementation" class="anchor"></a>Study implementation</h2>
-<p>Now we have completely design our study we have to implement the study. We have to generate the target and outcome cohorts and we need to develop the R code to run against our CDM that will execute the full study.</p>
-<div id="cohort-instantiation" class="section level3">
-<h3 class="hasAnchor">
-<a href="#cohort-instantiation" class="anchor"></a>Cohort instantiation</h3>
-<p>For our study we need to know when a person enters the target and outcome cohorts. This is stored in a table on the server that contains the cohort start date and cohort end date for all subjects for a specific cohort definition. This cohort table has a very simple structure as shown below:</p>
+<p>According to the best practices we need to make a protocol that
+completely specifies how we plan to execute our study. This protocol
+will be assessed by the governance boards of the participating data
+sources in your network study. For this a template could be used but we
+prefer to automate this process as much as possible by adding
+functionality to automatically generate study protocol from a study
+specification. We will discuss this in more detail later.</p>
+</div>
+<div class="section level3">
+<h3 id="study-implementation">Study implementation<a class="anchor" aria-label="anchor" href="#study-implementation"></a>
+</h3>
+<p>Now we have completely design our study we have to implement the
+study. We have to generate the target and outcome cohorts and we need to
+develop the R code to run against our CDM that will execute the full
+study.</p>
+<div class="section level4">
+<h4 id="cohort-instantiation">Cohort instantiation<a class="anchor" aria-label="anchor" href="#cohort-instantiation"></a>
+</h4>
+<p>For our study we need to know when a person enters the target and
+outcome cohorts. This is stored in a table on the server that contains
+the cohort start date and cohort end date for all subjects for a
+specific cohort definition. This cohort table has a very simple
+structure as shown below:</p>
 <ul>
 <li>
-<code>cohort_definition_id</code>, a unique identifier for distinguishing between different types of cohorts, e.g. cohorts of interest and outcome cohorts.</li>
+<code>cohort_definition_id</code>, a unique identifier for
+distinguishing between different types of cohorts, e.g. cohorts of
+interest and outcome cohorts.</li>
 <li>
-<code>subject_id</code>, a unique identifier corresponding to the <code>person_id</code> in the CDM.</li>
+<code>subject_id</code>, a unique identifier corresponding to the
+<code>person_id</code> in the CDM.</li>
 <li>
-<code>cohort_start_date</code>, the date the subject enters the cohort.</li>
+<code>cohort_start_date</code>, the date the subject enters the
+cohort.</li>
 <li>
-<code>cohort_end_date</code>, the date the subject leaves the cohort.</li>
+<code>cohort_end_date</code>, the date the subject leaves the
+cohort.</li>
 </ul>
-<p>How do we fill this table according to our cohort definitions? There are two options for this:</p>
+<p>How do we fill this table according to our cohort definitions? There
+are two options for this:</p>
 <ol style="list-style-type: decimal">
-<li><p>use the interactive cohort builder tool in <a href="www.github.com/OHDSI/ATLAS">ATLAS</a> which can be used to create cohorts based on inclusion criteria and will automatically populate this cohort table.</p></li>
-<li><p>write your own custom SQL statements to fill the cohort table.</p></li>
+<li><p>use the interactive cohort builder tool in <a href="www.github.com/OHDSI/ATLAS">ATLAS</a> which can be used to create
+cohorts based on inclusion criteria and will automatically populate this
+cohort table.</p></li>
+<li><p>write your own custom SQL statements to fill the cohort
+table.</p></li>
 </ol>
-<p>Both methods are described below for our example prediction problem.</p>
+<p>Both methods are described below for our example prediction
+problem.</p>
 </div>
-<div id="atlas-cohort-builder" class="section level3">
-<h3 class="hasAnchor">
-<a href="#atlas-cohort-builder" class="anchor"></a>ATLAS cohort builder</h3>
-<div class="figure">
-<img src="example1/ATLAS_T.PNG" alt=""><p class="caption">Target Cohort Atrial Fibrillation</p>
+<div class="section level4">
+<h4 id="atlas-cohort-builder">ATLAS cohort builder<a class="anchor" aria-label="anchor" href="#atlas-cohort-builder"></a>
+</h4>
+<div class="float">
+<img src="example1/ATLAS_T.webp" alt="Target Cohort Atrial Fibrillation"><div class="figcaption">Target Cohort Atrial Fibrillation</div>
 </div>
-<p>ATLAS allows you to define cohorts interactively by specifying cohort entry and cohort exit criteria. Cohort entry criteria involve selecting one or more initial events, which determine the start date for cohort entry, and optionally specifying additional inclusion criteria which filter to the qualifying events. Cohort exit criteria are applied to each cohort entry record to determine the end date when the person’s episode no longer qualifies for the cohort. For the outcome cohort the end date is less relevant. As an example, Figure 4 shows how we created the Atrial Fibrillation cohort and Figure 5 shows how we created the stroke cohort in ATLAS.</p>
-<div class="figure">
-<img src="example1/ATLAS_O.PNG" alt=""><p class="caption">Outcome Cohort Stroke</p>
+<p>ATLAS allows you to define cohorts interactively by specifying cohort
+entry and cohort exit criteria. Cohort entry criteria involve selecting
+one or more initial events, which determine the start date for cohort
+entry, and optionally specifying additional inclusion criteria which
+filter to the qualifying events. Cohort exit criteria are applied to
+each cohort entry record to determine the end date when the person’s
+episode no longer qualifies for the cohort. For the outcome cohort the
+end date is less relevant. As an example, Figure 4 shows how we created
+the Atrial Fibrillation cohort and Figure 5 shows how we created the
+stroke cohort in ATLAS.</p>
+<div class="float">
+<img src="example1/ATLAS_O.webp" alt="Outcome Cohort Stroke"><div class="figcaption">Outcome Cohort Stroke</div>
 </div>
 <p>The T and O cohorts can be found here:</p>
 <ul>
-<li>Atrial Fibrillaton (T): <a href="http://www.ohdsi.org/web/atlas/#/cohortdefinition/1769447" class="uri">http://www.ohdsi.org/web/atlas/#/cohortdefinition/1769447</a>
+<li>Atrial Fibrillaton (T): <a href="http://www.ohdsi.org/web/atlas/#/cohortdefinition/1769447" class="external-link uri">http://www.ohdsi.org/web/atlas/#/cohortdefinition/1769447</a>
 </li>
-<li>Stroke (O) : <a href="http://www.ohdsi.org/web/atlas/#/cohortdefinition/1769448" class="uri">http://www.ohdsi.org/web/atlas/#/cohortdefinition/1769448</a>
+<li>Stroke (O) : <a href="http://www.ohdsi.org/web/atlas/#/cohortdefinition/1769448" class="external-link uri">http://www.ohdsi.org/web/atlas/#/cohortdefinition/1769448</a>
 </li>
 </ul>
-<p>In depth explanation of cohort creation in ATLAS is out of scope of this vignette but can be found on the OHDSI wiki pages <a href="http://www.ohdsi.org/web/wiki/doku.php?id=documentation:software:atlas">(link)</a>.</p>
-<p>Note that when a cohort is created in ATLAS the cohortid is needed to extract the data in R. The cohortid can be found at the top of the ATLAS screen, e.g. 1769447 in Figure 4.</p>
-</div>
-<div id="custom-cohorts" class="section level3">
-<h3 class="hasAnchor">
-<a href="#custom-cohorts" class="anchor"></a>Custom cohorts</h3>
-<p>It is also possible to create cohorts without the use of ATLAS. Using custom cohort code (SQL) you can make more advanced cohorts if needed.</p>
-<p>For our example study, we need to create at table to hold the cohort data and we need to create SQL code to instantiate this table for both the AF and Stroke cohorts. Therefore, we create a file called <em>AfStrokeCohorts.sql</em> with the following contents:</p>
-<div class="sourceCode" id="cb1"><pre class="sourceCode sql"><code class="sourceCode sql"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="co">/***********************************</span></span>
-<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a><span class="co">File AfStrokeCohorts.sql </span></span>
-<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a><span class="co">***********************************/</span></span>
-<span id="cb1-4"><a href="#cb1-4" aria-hidden="true" tabindex="-1"></a><span class="co">/*</span></span>
-<span id="cb1-5"><a href="#cb1-5" aria-hidden="true" tabindex="-1"></a><span class="co">Create a table to store the persons in the T and C cohort</span></span>
-<span id="cb1-6"><a href="#cb1-6" aria-hidden="true" tabindex="-1"></a><span class="co">*/</span></span>
-<span id="cb1-7"><a href="#cb1-7" aria-hidden="true" tabindex="-1"></a></span>
-<span id="cb1-8"><a href="#cb1-8" aria-hidden="true" tabindex="-1"></a><span class="cf">IF</span> OBJECT_ID(<span class="st">'@resultsDatabaseSchema.PLPAFibStrokeCohort'</span>, <span class="st">'U'</span>) <span class="kw">IS</span> <span class="kw">NOT</span> <span class="kw">NULL</span> </span>
-<span id="cb1-9"><a href="#cb1-9" aria-hidden="true" tabindex="-1"></a><span class="kw">DROP</span> <span class="kw">TABLE</span> @resultsDatabaseSchema.PLPAFibStrokeCohort;</span>
-<span id="cb1-10"><a href="#cb1-10" aria-hidden="true" tabindex="-1"></a></span>
-<span id="cb1-11"><a href="#cb1-11" aria-hidden="true" tabindex="-1"></a><span class="kw">CREATE</span> <span class="kw">TABLE</span> @resultsDatabaseSchema.PLPAFibStrokeCohort </span>
-<span id="cb1-12"><a href="#cb1-12" aria-hidden="true" tabindex="-1"></a>( </span>
-<span id="cb1-13"><a href="#cb1-13" aria-hidden="true" tabindex="-1"></a>cohort_definition_id <span class="dt">INT</span>, </span>
-<span id="cb1-14"><a href="#cb1-14" aria-hidden="true" tabindex="-1"></a>subject_id BIGINT,</span>
-<span id="cb1-15"><a href="#cb1-15" aria-hidden="true" tabindex="-1"></a>cohort_start_date <span class="dt">DATE</span>, </span>
-<span id="cb1-16"><a href="#cb1-16" aria-hidden="true" tabindex="-1"></a>cohort_end_date <span class="dt">DATE</span></span>
-<span id="cb1-17"><a href="#cb1-17" aria-hidden="true" tabindex="-1"></a>);</span>
-<span id="cb1-18"><a href="#cb1-18" aria-hidden="true" tabindex="-1"></a></span>
-<span id="cb1-19"><a href="#cb1-19" aria-hidden="true" tabindex="-1"></a></span>
-<span id="cb1-20"><a href="#cb1-20" aria-hidden="true" tabindex="-1"></a><span class="co">/*</span></span>
-<span id="cb1-21"><a href="#cb1-21" aria-hidden="true" tabindex="-1"></a><span class="co">T cohort:  [PatientLevelPrediction vignette]:  T : patients who are newly </span></span>
-<span id="cb1-22"><a href="#cb1-22" aria-hidden="true" tabindex="-1"></a><span class="co">diagnosed with Atrial fibrillation</span></span>
-<span id="cb1-23"><a href="#cb1-23" aria-hidden="true" tabindex="-1"></a><span class="co">- persons with a condition occurrence record of 'Atrial fibrillation' or </span></span>
-<span id="cb1-24"><a href="#cb1-24" aria-hidden="true" tabindex="-1"></a><span class="co">any descendants, indexed at the first diagnosis</span></span>
-<span id="cb1-25"><a href="#cb1-25" aria-hidden="true" tabindex="-1"></a><span class="co">- who have &gt;1095 days of prior observation before their first diagnosis</span></span>
-<span id="cb1-26"><a href="#cb1-26" aria-hidden="true" tabindex="-1"></a><span class="co">- and have no warfarin exposure any time prior to first AFib diagnosis</span></span>
-<span id="cb1-27"><a href="#cb1-27" aria-hidden="true" tabindex="-1"></a><span class="co">*/</span></span>
-<span id="cb1-28"><a href="#cb1-28" aria-hidden="true" tabindex="-1"></a><span class="kw">INSERT</span> <span class="kw">INTO</span> @resultsDatabaseSchema.AFibStrokeCohort (cohort_definition_id, </span>
-<span id="cb1-29"><a href="#cb1-29" aria-hidden="true" tabindex="-1"></a>subject_id, </span>
-<span id="cb1-30"><a href="#cb1-30" aria-hidden="true" tabindex="-1"></a>cohort_start_date, </span>
-<span id="cb1-31"><a href="#cb1-31" aria-hidden="true" tabindex="-1"></a>cohort_end_date)</span>
-<span id="cb1-32"><a href="#cb1-32" aria-hidden="true" tabindex="-1"></a><span class="kw">SELECT</span> <span class="dv">1</span> <span class="kw">AS</span> cohort_definition_id,</span>
-<span id="cb1-33"><a href="#cb1-33" aria-hidden="true" tabindex="-1"></a>AFib.person_id <span class="kw">AS</span> subject_id,</span>
-<span id="cb1-34"><a href="#cb1-34" aria-hidden="true" tabindex="-1"></a>AFib.condition_start_date <span class="kw">AS</span> cohort_start_date,</span>
-<span id="cb1-35"><a href="#cb1-35" aria-hidden="true" tabindex="-1"></a>observation_period.observation_period_end_date <span class="kw">AS</span> cohort_end_date</span>
-<span id="cb1-36"><a href="#cb1-36" aria-hidden="true" tabindex="-1"></a><span class="kw">FROM</span></span>
-<span id="cb1-37"><a href="#cb1-37" aria-hidden="true" tabindex="-1"></a>(</span>
-<span id="cb1-38"><a href="#cb1-38" aria-hidden="true" tabindex="-1"></a>  <span class="kw">SELECT</span> person_id, <span class="fu">min</span>(condition_start_date) <span class="kw">as</span> condition_start_date</span>
-<span id="cb1-39"><a href="#cb1-39" aria-hidden="true" tabindex="-1"></a>  <span class="kw">FROM</span> @cdmDatabaseSchema.condition_occurrence</span>
-<span id="cb1-40"><a href="#cb1-40" aria-hidden="true" tabindex="-1"></a>  <span class="kw">WHERE</span> condition_concept_id <span class="kw">IN</span> (<span class="kw">SELECT</span> descendant_concept_id <span class="kw">FROM</span> </span>
-<span id="cb1-41"><a href="#cb1-41" aria-hidden="true" tabindex="-1"></a>  @cdmDatabaseSchema.concept_ancestor <span class="kw">WHERE</span> ancestor_concept_id <span class="kw">IN</span> </span>
-<span id="cb1-42"><a href="#cb1-42" aria-hidden="true" tabindex="-1"></a>  (<span class="dv">313217</span> <span class="co">/*atrial fibrillation*/</span>))</span>
-<span id="cb1-43"><a href="#cb1-43" aria-hidden="true" tabindex="-1"></a>  <span class="kw">GROUP</span> <span class="kw">BY</span> person_id</span>
-<span id="cb1-44"><a href="#cb1-44" aria-hidden="true" tabindex="-1"></a>) AFib</span>
-<span id="cb1-45"><a href="#cb1-45" aria-hidden="true" tabindex="-1"></a>  <span class="kw">INNER</span> <span class="kw">JOIN</span> @cdmDatabaseSchema.observation_period</span>
-<span id="cb1-46"><a href="#cb1-46" aria-hidden="true" tabindex="-1"></a>  <span class="kw">ON</span> AFib.person_id <span class="op">=</span> observation_period.person_id</span>
-<span id="cb1-47"><a href="#cb1-47" aria-hidden="true" tabindex="-1"></a>  <span class="kw">AND</span> AFib.condition_start_date <span class="op">&gt;=</span> dateadd(dd,<span class="dv">1095</span>, </span>
-<span id="cb1-48"><a href="#cb1-48" aria-hidden="true" tabindex="-1"></a>  observation_period.observation_period_start_date)</span>
-<span id="cb1-49"><a href="#cb1-49" aria-hidden="true" tabindex="-1"></a>  <span class="kw">AND</span> AFib.condition_start_date <span class="op">&lt;=</span> observation_period.observation_period_end_date</span>
-<span id="cb1-50"><a href="#cb1-50" aria-hidden="true" tabindex="-1"></a>  <span class="kw">LEFT</span> <span class="kw">JOIN</span></span>
-<span id="cb1-51"><a href="#cb1-51" aria-hidden="true" tabindex="-1"></a>  (</span>
-<span id="cb1-52"><a href="#cb1-52" aria-hidden="true" tabindex="-1"></a>  <span class="kw">SELECT</span> person_id, <span class="fu">min</span>(drug_exposure_start_date) <span class="kw">as</span> drug_exposure_start_date</span>
-<span id="cb1-53"><a href="#cb1-53" aria-hidden="true" tabindex="-1"></a>  <span class="kw">FROM</span> @cdmDatabaseSchema.drug_exposure</span>
-<span id="cb1-54"><a href="#cb1-54" aria-hidden="true" tabindex="-1"></a>  <span class="kw">WHERE</span> drug_concept_id <span class="kw">IN</span> (<span class="kw">SELECT</span> descendant_concept_id <span class="kw">FROM</span> </span>
-<span id="cb1-55"><a href="#cb1-55" aria-hidden="true" tabindex="-1"></a>  @cdmDatabaseSchema.concept_ancestor <span class="kw">WHERE</span> ancestor_concept_id <span class="kw">IN</span> </span>
-<span id="cb1-56"><a href="#cb1-56" aria-hidden="true" tabindex="-1"></a>  (<span class="dv">1310149</span> <span class="co">/*warfarin*/</span>))</span>
-<span id="cb1-57"><a href="#cb1-57" aria-hidden="true" tabindex="-1"></a>  <span class="kw">GROUP</span> <span class="kw">BY</span> person_id</span>
-<span id="cb1-58"><a href="#cb1-58" aria-hidden="true" tabindex="-1"></a>  ) warfarin</span>
-<span id="cb1-59"><a href="#cb1-59" aria-hidden="true" tabindex="-1"></a>  <span class="kw">ON</span> Afib.person_id <span class="op">=</span> warfarin.person_id</span>
-<span id="cb1-60"><a href="#cb1-60" aria-hidden="true" tabindex="-1"></a>  <span class="kw">AND</span> Afib.condition_start_date <span class="op">&gt;</span> warfarin.drug_exposure_start_date</span>
-<span id="cb1-61"><a href="#cb1-61" aria-hidden="true" tabindex="-1"></a>  <span class="kw">WHERE</span> warfarin.person_id <span class="kw">IS</span> <span class="kw">NULL</span></span>
-<span id="cb1-62"><a href="#cb1-62" aria-hidden="true" tabindex="-1"></a>  ;</span>
-<span id="cb1-63"><a href="#cb1-63" aria-hidden="true" tabindex="-1"></a>  </span>
-<span id="cb1-64"><a href="#cb1-64" aria-hidden="true" tabindex="-1"></a>  <span class="co">/*</span></span>
-<span id="cb1-65"><a href="#cb1-65" aria-hidden="true" tabindex="-1"></a><span class="co">  C cohort:  [PatientLevelPrediction vignette]:  O: Ischemic stroke events</span></span>
-<span id="cb1-66"><a href="#cb1-66" aria-hidden="true" tabindex="-1"></a><span class="co">  - inpatient visits that include a condition occurrence record for </span></span>
-<span id="cb1-67"><a href="#cb1-67" aria-hidden="true" tabindex="-1"></a><span class="co">  'cerebral infarction' and descendants, 'cerebral thrombosis', </span></span>
-<span id="cb1-68"><a href="#cb1-68" aria-hidden="true" tabindex="-1"></a><span class="co">  'cerebral embolism', 'cerebral artery occlusion' </span></span>
-<span id="cb1-69"><a href="#cb1-69" aria-hidden="true" tabindex="-1"></a><span class="co">  */</span></span>
-<span id="cb1-70"><a href="#cb1-70" aria-hidden="true" tabindex="-1"></a>  <span class="kw">INSERT</span> <span class="kw">INTO</span> @resultsDatabaseSchema.AFibStrokeCohort (cohort_definition_id, </span>
-<span id="cb1-71"><a href="#cb1-71" aria-hidden="true" tabindex="-1"></a>  subject_id, </span>
-<span id="cb1-72"><a href="#cb1-72" aria-hidden="true" tabindex="-1"></a>  cohort_start_date, </span>
-<span id="cb1-73"><a href="#cb1-73" aria-hidden="true" tabindex="-1"></a>  cohort_end_date)</span>
-<span id="cb1-74"><a href="#cb1-74" aria-hidden="true" tabindex="-1"></a>  <span class="kw">SELECT</span> <span class="dv">2</span> <span class="kw">AS</span> cohort_definition_id,</span>
-<span id="cb1-75"><a href="#cb1-75" aria-hidden="true" tabindex="-1"></a>  visit_occurrence.person_id <span class="kw">AS</span> subject_id,</span>
-<span id="cb1-76"><a href="#cb1-76" aria-hidden="true" tabindex="-1"></a>  visit_occurrence.visit_start_date <span class="kw">AS</span> cohort_start_date,</span>
-<span id="cb1-77"><a href="#cb1-77" aria-hidden="true" tabindex="-1"></a>  visit_occurrence.visit_end_date <span class="kw">AS</span> cohort_end_date</span>
-<span id="cb1-78"><a href="#cb1-78" aria-hidden="true" tabindex="-1"></a>  <span class="kw">FROM</span>  </span>
-<span id="cb1-79"><a href="#cb1-79" aria-hidden="true" tabindex="-1"></a>  (</span>
-<span id="cb1-80"><a href="#cb1-80" aria-hidden="true" tabindex="-1"></a>  <span class="kw">SELECT</span> person_id, condition_start_date</span>
-<span id="cb1-81"><a href="#cb1-81" aria-hidden="true" tabindex="-1"></a>  <span class="kw">FROM</span> @cdmDatabaseSchema.condition_occurrence</span>
-<span id="cb1-82"><a href="#cb1-82" aria-hidden="true" tabindex="-1"></a>  <span class="kw">WHERE</span> condition_concept_id <span class="kw">IN</span> (<span class="kw">SELECT</span> <span class="kw">DISTINCT</span> descendant_concept_id <span class="kw">FROM</span> </span>
-<span id="cb1-83"><a href="#cb1-83" aria-hidden="true" tabindex="-1"></a>  @cdmDatabaseSchema.concept_ancestor <span class="kw">WHERE</span> ancestor_concept_id <span class="kw">IN</span> </span>
-<span id="cb1-84"><a href="#cb1-84" aria-hidden="true" tabindex="-1"></a>  (<span class="dv">443454</span> <span class="co">/*cerebral infarction*/</span>) <span class="kw">OR</span> descendant_concept_id <span class="kw">IN</span> </span>
-<span id="cb1-85"><a href="#cb1-85" aria-hidden="true" tabindex="-1"></a>  (<span class="dv">441874</span> <span class="co">/*cerebral thrombosis*/</span>, <span class="dv">375557</span> <span class="co">/*cerebral embolism*/</span>, </span>
-<span id="cb1-86"><a href="#cb1-86" aria-hidden="true" tabindex="-1"></a>  <span class="dv">372924</span> <span class="co">/*cerebral artery occlusion*/</span>))</span>
-<span id="cb1-87"><a href="#cb1-87" aria-hidden="true" tabindex="-1"></a>  ) stroke</span>
-<span id="cb1-88"><a href="#cb1-88" aria-hidden="true" tabindex="-1"></a>  <span class="kw">INNER</span> <span class="kw">JOIN</span> @cdmDatabaseSchema.visit_occurrence</span>
-<span id="cb1-89"><a href="#cb1-89" aria-hidden="true" tabindex="-1"></a>  <span class="kw">ON</span> stroke.person_id <span class="op">=</span> visit_occurrence.person_id</span>
-<span id="cb1-90"><a href="#cb1-90" aria-hidden="true" tabindex="-1"></a>  <span class="kw">AND</span> stroke.condition_start_date <span class="op">&gt;=</span> visit_occurrence.visit_start_date</span>
-<span id="cb1-91"><a href="#cb1-91" aria-hidden="true" tabindex="-1"></a>  <span class="kw">AND</span> stroke.condition_start_date <span class="op">&lt;=</span> visit_occurrence.visit_end_date</span>
-<span id="cb1-92"><a href="#cb1-92" aria-hidden="true" tabindex="-1"></a>  <span class="kw">AND</span> visit_occurrence.visit_concept_id <span class="kw">IN</span> (<span class="dv">9201</span>, <span class="dv">262</span> <span class="co">/*'Inpatient Visit'  or </span></span>
-<span id="cb1-93"><a href="#cb1-93" aria-hidden="true" tabindex="-1"></a><span class="co">  'Emergency Room and Inpatient Visit'*/</span>)</span>
-<span id="cb1-94"><a href="#cb1-94" aria-hidden="true" tabindex="-1"></a>  <span class="kw">GROUP</span> <span class="kw">BY</span> visit_occurrence.person_id, visit_occurrence.visit_start_date, </span>
-<span id="cb1-95"><a href="#cb1-95" aria-hidden="true" tabindex="-1"></a>  visit_occurrence.visit_end_date</span>
-<span id="cb1-96"><a href="#cb1-96" aria-hidden="true" tabindex="-1"></a>  ;</span>
-<span id="cb1-97"><a href="#cb1-97" aria-hidden="true" tabindex="-1"></a>  </span></code></pre></div>
-<p>This is parameterized SQL which can be used by the <a href="http://github.com/OHDSI/SqlRender"><code>SqlRender</code></a> package. We use parameterized SQL so we do not have to pre-specify the names of the CDM and result schemas. That way, if we want to run the SQL on a different schema, we only need to change the parameter values; we do not have to change the SQL code. By also making use of translation functionality in <code>SqlRender</code>, we can make sure the SQL code can be run in many different environments.</p>
-<p>To execute this sql against our CDM we first need to tell R how to connect to the server. <code>PatientLevelPrediction</code> uses the <a href="http://github.com/ohdsi/DatabaseConnector"><code>DatabaseConnector</code></a> package, which provides a function called <code>createConnectionDetails</code>. Type <code>?createConnectionDetails</code> for the specific settings required for the various database management systems (DBMS). For example, one might connect to a PostgreSQL database using this code:</p>
+<p>In depth explanation of cohort creation in ATLAS is out of scope of
+this vignette but can be found on the OHDSI wiki pages <a href="http://www.ohdsi.org/web/wiki/doku.php?id=documentation:software:atlas" class="external-link">(link)</a>.</p>
+<p>Note that when a cohort is created in ATLAS the cohortid is needed to
+extract the data in R. The cohortid can be found at the top of the ATLAS
+screen, e.g. 1769447 in Figure 4.</p>
+</div>
+<div class="section level4">
+<h4 id="custom-cohorts">Custom cohorts<a class="anchor" aria-label="anchor" href="#custom-cohorts"></a>
+</h4>
+<p>It is also possible to create cohorts without the use of ATLAS. Using
+custom cohort code (SQL) you can make more advanced cohorts if
+needed.</p>
+<p>For our example study, we need to create at table to hold the cohort
+data and we need to create SQL code to instantiate this table for both
+the AF and Stroke cohorts. Therefore, we create a file called
+<em>AfStrokeCohorts.sql</em> with the following contents:</p>
+<div class="sourceCode" id="cb1"><pre class="sourceCode sql"><code class="sourceCode sql"><span id="cb1-1"><a href="#cb1-1" tabindex="-1"></a><span class="co">/***********************************</span></span>
+<span id="cb1-2"><a href="#cb1-2" tabindex="-1"></a><span class="co">File AfStrokeCohorts.sql </span></span>
+<span id="cb1-3"><a href="#cb1-3" tabindex="-1"></a><span class="co">***********************************/</span></span>
+<span id="cb1-4"><a href="#cb1-4" tabindex="-1"></a><span class="co">/*</span></span>
+<span id="cb1-5"><a href="#cb1-5" tabindex="-1"></a><span class="co">Create a table to store the persons in the T and C cohort</span></span>
+<span id="cb1-6"><a href="#cb1-6" tabindex="-1"></a><span class="co">*/</span></span>
+<span id="cb1-7"><a href="#cb1-7" tabindex="-1"></a></span>
+<span id="cb1-8"><a href="#cb1-8" tabindex="-1"></a><span class="cf">IF</span> OBJECT_ID(<span class="st">'@resultsDatabaseSchema.PLPAFibStrokeCohort'</span>, <span class="st">'U'</span>) <span class="kw">IS</span> <span class="kw">NOT</span> <span class="kw">NULL</span> </span>
+<span id="cb1-9"><a href="#cb1-9" tabindex="-1"></a><span class="kw">DROP</span> <span class="kw">TABLE</span> @resultsDatabaseSchema.PLPAFibStrokeCohort;</span>
+<span id="cb1-10"><a href="#cb1-10" tabindex="-1"></a></span>
+<span id="cb1-11"><a href="#cb1-11" tabindex="-1"></a><span class="kw">CREATE</span> <span class="kw">TABLE</span> @resultsDatabaseSchema.PLPAFibStrokeCohort </span>
+<span id="cb1-12"><a href="#cb1-12" tabindex="-1"></a>( </span>
+<span id="cb1-13"><a href="#cb1-13" tabindex="-1"></a>cohort_definition_id <span class="dt">INT</span>, </span>
+<span id="cb1-14"><a href="#cb1-14" tabindex="-1"></a>subject_id BIGINT,</span>
+<span id="cb1-15"><a href="#cb1-15" tabindex="-1"></a>cohort_start_date <span class="dt">DATE</span>, </span>
+<span id="cb1-16"><a href="#cb1-16" tabindex="-1"></a>cohort_end_date <span class="dt">DATE</span></span>
+<span id="cb1-17"><a href="#cb1-17" tabindex="-1"></a>);</span>
+<span id="cb1-18"><a href="#cb1-18" tabindex="-1"></a></span>
+<span id="cb1-19"><a href="#cb1-19" tabindex="-1"></a></span>
+<span id="cb1-20"><a href="#cb1-20" tabindex="-1"></a><span class="co">/*</span></span>
+<span id="cb1-21"><a href="#cb1-21" tabindex="-1"></a><span class="co">T cohort:  [PatientLevelPrediction vignette]:  T : patients who are newly </span></span>
+<span id="cb1-22"><a href="#cb1-22" tabindex="-1"></a><span class="co">diagnosed with Atrial fibrillation</span></span>
+<span id="cb1-23"><a href="#cb1-23" tabindex="-1"></a><span class="co">- persons with a condition occurrence record of 'Atrial fibrillation' or </span></span>
+<span id="cb1-24"><a href="#cb1-24" tabindex="-1"></a><span class="co">any descendants, indexed at the first diagnosis</span></span>
+<span id="cb1-25"><a href="#cb1-25" tabindex="-1"></a><span class="co">- who have &gt;1095 days of prior observation before their first diagnosis</span></span>
+<span id="cb1-26"><a href="#cb1-26" tabindex="-1"></a><span class="co">- and have no warfarin exposure any time prior to first AFib diagnosis</span></span>
+<span id="cb1-27"><a href="#cb1-27" tabindex="-1"></a><span class="co">*/</span></span>
+<span id="cb1-28"><a href="#cb1-28" tabindex="-1"></a><span class="kw">INSERT</span> <span class="kw">INTO</span> @resultsDatabaseSchema.AFibStrokeCohort (cohort_definition_id, </span>
+<span id="cb1-29"><a href="#cb1-29" tabindex="-1"></a>subject_id, </span>
+<span id="cb1-30"><a href="#cb1-30" tabindex="-1"></a>cohort_start_date, </span>
+<span id="cb1-31"><a href="#cb1-31" tabindex="-1"></a>cohort_end_date)</span>
+<span id="cb1-32"><a href="#cb1-32" tabindex="-1"></a><span class="kw">SELECT</span> <span class="dv">1</span> <span class="kw">AS</span> cohort_definition_id,</span>
+<span id="cb1-33"><a href="#cb1-33" tabindex="-1"></a>AFib.person_id <span class="kw">AS</span> subject_id,</span>
+<span id="cb1-34"><a href="#cb1-34" tabindex="-1"></a>AFib.condition_start_date <span class="kw">AS</span> cohort_start_date,</span>
+<span id="cb1-35"><a href="#cb1-35" tabindex="-1"></a>observation_period.observation_period_end_date <span class="kw">AS</span> cohort_end_date</span>
+<span id="cb1-36"><a href="#cb1-36" tabindex="-1"></a><span class="kw">FROM</span></span>
+<span id="cb1-37"><a href="#cb1-37" tabindex="-1"></a>(</span>
+<span id="cb1-38"><a href="#cb1-38" tabindex="-1"></a>  <span class="kw">SELECT</span> person_id, <span class="fu">min</span>(condition_start_date) <span class="kw">as</span> condition_start_date</span>
+<span id="cb1-39"><a href="#cb1-39" tabindex="-1"></a>  <span class="kw">FROM</span> @cdmDatabaseSchema.condition_occurrence</span>
+<span id="cb1-40"><a href="#cb1-40" tabindex="-1"></a>  <span class="kw">WHERE</span> condition_concept_id <span class="kw">IN</span> (<span class="kw">SELECT</span> descendant_concept_id <span class="kw">FROM</span> </span>
+<span id="cb1-41"><a href="#cb1-41" tabindex="-1"></a>  @cdmDatabaseSchema.concept_ancestor <span class="kw">WHERE</span> ancestor_concept_id <span class="kw">IN</span> </span>
+<span id="cb1-42"><a href="#cb1-42" tabindex="-1"></a>  (<span class="dv">313217</span> <span class="co">/*atrial fibrillation*/</span>))</span>
+<span id="cb1-43"><a href="#cb1-43" tabindex="-1"></a>  <span class="kw">GROUP</span> <span class="kw">BY</span> person_id</span>
+<span id="cb1-44"><a href="#cb1-44" tabindex="-1"></a>) AFib</span>
+<span id="cb1-45"><a href="#cb1-45" tabindex="-1"></a>  <span class="kw">INNER</span> <span class="kw">JOIN</span> @cdmDatabaseSchema.observation_period</span>
+<span id="cb1-46"><a href="#cb1-46" tabindex="-1"></a>  <span class="kw">ON</span> AFib.person_id <span class="op">=</span> observation_period.person_id</span>
+<span id="cb1-47"><a href="#cb1-47" tabindex="-1"></a>  <span class="kw">AND</span> AFib.condition_start_date <span class="op">&gt;=</span> dateadd(dd,<span class="dv">1095</span>, </span>
+<span id="cb1-48"><a href="#cb1-48" tabindex="-1"></a>  observation_period.observation_period_start_date)</span>
+<span id="cb1-49"><a href="#cb1-49" tabindex="-1"></a>  <span class="kw">AND</span> AFib.condition_start_date <span class="op">&lt;=</span> observation_period.observation_period_end_date</span>
+<span id="cb1-50"><a href="#cb1-50" tabindex="-1"></a>  <span class="kw">LEFT</span> <span class="kw">JOIN</span></span>
+<span id="cb1-51"><a href="#cb1-51" tabindex="-1"></a>  (</span>
+<span id="cb1-52"><a href="#cb1-52" tabindex="-1"></a>  <span class="kw">SELECT</span> person_id, <span class="fu">min</span>(drug_exposure_start_date) <span class="kw">as</span> drug_exposure_start_date</span>
+<span id="cb1-53"><a href="#cb1-53" tabindex="-1"></a>  <span class="kw">FROM</span> @cdmDatabaseSchema.drug_exposure</span>
+<span id="cb1-54"><a href="#cb1-54" tabindex="-1"></a>  <span class="kw">WHERE</span> drug_concept_id <span class="kw">IN</span> (<span class="kw">SELECT</span> descendant_concept_id <span class="kw">FROM</span> </span>
+<span id="cb1-55"><a href="#cb1-55" tabindex="-1"></a>  @cdmDatabaseSchema.concept_ancestor <span class="kw">WHERE</span> ancestor_concept_id <span class="kw">IN</span> </span>
+<span id="cb1-56"><a href="#cb1-56" tabindex="-1"></a>  (<span class="dv">1310149</span> <span class="co">/*warfarin*/</span>))</span>
+<span id="cb1-57"><a href="#cb1-57" tabindex="-1"></a>  <span class="kw">GROUP</span> <span class="kw">BY</span> person_id</span>
+<span id="cb1-58"><a href="#cb1-58" tabindex="-1"></a>  ) warfarin</span>
+<span id="cb1-59"><a href="#cb1-59" tabindex="-1"></a>  <span class="kw">ON</span> Afib.person_id <span class="op">=</span> warfarin.person_id</span>
+<span id="cb1-60"><a href="#cb1-60" tabindex="-1"></a>  <span class="kw">AND</span> Afib.condition_start_date <span class="op">&gt;</span> warfarin.drug_exposure_start_date</span>
+<span id="cb1-61"><a href="#cb1-61" tabindex="-1"></a>  <span class="kw">WHERE</span> warfarin.person_id <span class="kw">IS</span> <span class="kw">NULL</span></span>
+<span id="cb1-62"><a href="#cb1-62" tabindex="-1"></a>  ;</span>
+<span id="cb1-63"><a href="#cb1-63" tabindex="-1"></a>  </span>
+<span id="cb1-64"><a href="#cb1-64" tabindex="-1"></a>  <span class="co">/*</span></span>
+<span id="cb1-65"><a href="#cb1-65" tabindex="-1"></a><span class="co">  C cohort:  [PatientLevelPrediction vignette]:  O: Ischemic stroke events</span></span>
+<span id="cb1-66"><a href="#cb1-66" tabindex="-1"></a><span class="co">  - inpatient visits that include a condition occurrence record for </span></span>
+<span id="cb1-67"><a href="#cb1-67" tabindex="-1"></a><span class="co">  'cerebral infarction' and descendants, 'cerebral thrombosis', </span></span>
+<span id="cb1-68"><a href="#cb1-68" tabindex="-1"></a><span class="co">  'cerebral embolism', 'cerebral artery occlusion' </span></span>
+<span id="cb1-69"><a href="#cb1-69" tabindex="-1"></a><span class="co">  */</span></span>
+<span id="cb1-70"><a href="#cb1-70" tabindex="-1"></a>  <span class="kw">INSERT</span> <span class="kw">INTO</span> @resultsDatabaseSchema.AFibStrokeCohort (cohort_definition_id, </span>
+<span id="cb1-71"><a href="#cb1-71" tabindex="-1"></a>  subject_id, </span>
+<span id="cb1-72"><a href="#cb1-72" tabindex="-1"></a>  cohort_start_date, </span>
+<span id="cb1-73"><a href="#cb1-73" tabindex="-1"></a>  cohort_end_date)</span>
+<span id="cb1-74"><a href="#cb1-74" tabindex="-1"></a>  <span class="kw">SELECT</span> <span class="dv">2</span> <span class="kw">AS</span> cohort_definition_id,</span>
+<span id="cb1-75"><a href="#cb1-75" tabindex="-1"></a>  visit_occurrence.person_id <span class="kw">AS</span> subject_id,</span>
+<span id="cb1-76"><a href="#cb1-76" tabindex="-1"></a>  visit_occurrence.visit_start_date <span class="kw">AS</span> cohort_start_date,</span>
+<span id="cb1-77"><a href="#cb1-77" tabindex="-1"></a>  visit_occurrence.visit_end_date <span class="kw">AS</span> cohort_end_date</span>
+<span id="cb1-78"><a href="#cb1-78" tabindex="-1"></a>  <span class="kw">FROM</span>  </span>
+<span id="cb1-79"><a href="#cb1-79" tabindex="-1"></a>  (</span>
+<span id="cb1-80"><a href="#cb1-80" tabindex="-1"></a>  <span class="kw">SELECT</span> person_id, condition_start_date</span>
+<span id="cb1-81"><a href="#cb1-81" tabindex="-1"></a>  <span class="kw">FROM</span> @cdmDatabaseSchema.condition_occurrence</span>
+<span id="cb1-82"><a href="#cb1-82" tabindex="-1"></a>  <span class="kw">WHERE</span> condition_concept_id <span class="kw">IN</span> (<span class="kw">SELECT</span> <span class="kw">DISTINCT</span> descendant_concept_id <span class="kw">FROM</span> </span>
+<span id="cb1-83"><a href="#cb1-83" tabindex="-1"></a>  @cdmDatabaseSchema.concept_ancestor <span class="kw">WHERE</span> ancestor_concept_id <span class="kw">IN</span> </span>
+<span id="cb1-84"><a href="#cb1-84" tabindex="-1"></a>  (<span class="dv">443454</span> <span class="co">/*cerebral infarction*/</span>) <span class="kw">OR</span> descendant_concept_id <span class="kw">IN</span> </span>
+<span id="cb1-85"><a href="#cb1-85" tabindex="-1"></a>  (<span class="dv">441874</span> <span class="co">/*cerebral thrombosis*/</span>, <span class="dv">375557</span> <span class="co">/*cerebral embolism*/</span>, </span>
+<span id="cb1-86"><a href="#cb1-86" tabindex="-1"></a>  <span class="dv">372924</span> <span class="co">/*cerebral artery occlusion*/</span>))</span>
+<span id="cb1-87"><a href="#cb1-87" tabindex="-1"></a>  ) stroke</span>
+<span id="cb1-88"><a href="#cb1-88" tabindex="-1"></a>  <span class="kw">INNER</span> <span class="kw">JOIN</span> @cdmDatabaseSchema.visit_occurrence</span>
+<span id="cb1-89"><a href="#cb1-89" tabindex="-1"></a>  <span class="kw">ON</span> stroke.person_id <span class="op">=</span> visit_occurrence.person_id</span>
+<span id="cb1-90"><a href="#cb1-90" tabindex="-1"></a>  <span class="kw">AND</span> stroke.condition_start_date <span class="op">&gt;=</span> visit_occurrence.visit_start_date</span>
+<span id="cb1-91"><a href="#cb1-91" tabindex="-1"></a>  <span class="kw">AND</span> stroke.condition_start_date <span class="op">&lt;=</span> visit_occurrence.visit_end_date</span>
+<span id="cb1-92"><a href="#cb1-92" tabindex="-1"></a>  <span class="kw">AND</span> visit_occurrence.visit_concept_id <span class="kw">IN</span> (<span class="dv">9201</span>, <span class="dv">262</span> <span class="co">/*'Inpatient Visit'  or </span></span>
+<span id="cb1-93"><a href="#cb1-93" tabindex="-1"></a><span class="co">  'Emergency Room and Inpatient Visit'*/</span>)</span>
+<span id="cb1-94"><a href="#cb1-94" tabindex="-1"></a>  <span class="kw">GROUP</span> <span class="kw">BY</span> visit_occurrence.person_id, visit_occurrence.visit_start_date, </span>
+<span id="cb1-95"><a href="#cb1-95" tabindex="-1"></a>  visit_occurrence.visit_end_date</span>
+<span id="cb1-96"><a href="#cb1-96" tabindex="-1"></a>  ;</span>
+<span id="cb1-97"><a href="#cb1-97" tabindex="-1"></a>  </span></code></pre></div>
+<p>This is parameterized SQL which can be used by the <a href="http://github.com/OHDSI/SqlRender" class="external-link"><code>SqlRender</code></a>
+package. We use parameterized SQL so we do not have to pre-specify the
+names of the CDM and result schemas. That way, if we want to run the SQL
+on a different schema, we only need to change the parameter values; we
+do not have to change the SQL code. By also making use of translation
+functionality in <code>SqlRender</code>, we can make sure the SQL code
+can be run in many different environments.</p>
+<p>To execute this sql against our CDM we first need to tell R how to
+connect to the server. <code>PatientLevelPrediction</code> uses the <a href="http://github.com/ohdsi/DatabaseConnector" class="external-link"><code>DatabaseConnector</code></a>
+package, which provides a function called
+<code>createConnectionDetails</code>. Type
+<code>?createConnectionDetails</code> for the specific settings required
+for the various database management systems (DBMS). For example, one
+might connect to a PostgreSQL database using this code:</p>
 <div class="sourceCode" id="cb2"><pre class="downlit sourceCode r">
-<code class="sourceCode R">  <span class="va">connectionDetails</span> <span class="op">&lt;-</span> <span class="fu">createConnectionDetails</span><span class="op">(</span>dbms <span class="op">=</span> <span class="st">"postgresql"</span>, 
-  server <span class="op">=</span> <span class="st">"localhost/ohdsi"</span>, 
-  user <span class="op">=</span> <span class="st">"joe"</span>, 
-  password <span class="op">=</span> <span class="st">"supersecret"</span><span class="op">)</span>
-  
-  <span class="va">cdmDatabaseSchema</span> <span class="op">&lt;-</span> <span class="st">"my_cdm_data"</span>
-  <span class="va">cohortsDatabaseSchema</span> <span class="op">&lt;-</span> <span class="st">"my_results"</span>
-  <span class="va">cdmVersion</span> <span class="op">&lt;-</span> <span class="st">"5"</span></code></pre></div>
-<p>The last three lines define the <code>cdmDatabaseSchema</code> and <code>cohortsDatabaseSchema</code> variables, as well as the CDM version. We will use these later to tell R where the data in CDM format live, where we want to create the cohorts of interest, and what version CDM is used. Note that for Microsoft SQL Server, databaseschemas need to specify both the database and the schema, so for example <code>cdmDatabaseSchema &lt;- "my_cdm_data.dbo"</code>.</p>
+<code class="sourceCode R"><span>  <span class="va">connectionDetails</span> <span class="op">&lt;-</span> <span class="fu">createConnectionDetails</span><span class="op">(</span>dbms <span class="op">=</span> <span class="st">"postgresql"</span>, </span>
+<span>  server <span class="op">=</span> <span class="st">"localhost/ohdsi"</span>, </span>
+<span>  user <span class="op">=</span> <span class="st">"joe"</span>, </span>
+<span>  password <span class="op">=</span> <span class="st">"supersecret"</span><span class="op">)</span></span>
+<span>  </span>
+<span>  <span class="va">cdmDatabaseSchema</span> <span class="op">&lt;-</span> <span class="st">"my_cdm_data"</span></span>
+<span>  <span class="va">cohortsDatabaseSchema</span> <span class="op">&lt;-</span> <span class="st">"my_results"</span></span>
+<span>  <span class="va">cdmVersion</span> <span class="op">&lt;-</span> <span class="st">"5"</span></span></code></pre></div>
+<p>The last three lines define the <code>cdmDatabaseSchema</code> and
+<code>cohortsDatabaseSchema</code> variables, as well as the CDM
+version. We will use these later to tell R where the data in CDM format
+live, where we want to create the cohorts of interest, and what version
+CDM is used. Note that for Microsoft SQL Server, databaseschemas need to
+specify both the database and the schema, so for example
+<code>cdmDatabaseSchema &lt;- "my_cdm_data.dbo"</code>.</p>
 <div class="sourceCode" id="cb3"><pre class="downlit sourceCode r">
-<code class="sourceCode R">  <span class="kw"><a href="https://rdrr.io/r/base/library.html">library</a></span><span class="op">(</span><span class="va"><a href="https://ohdsi.github.io/SqlRender/">SqlRender</a></span><span class="op">)</span>
-  <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/readSql.html">readSql</a></span><span class="op">(</span><span class="st">"AfStrokeCohorts.sql"</span><span class="op">)</span>
-  <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/renderSql.html">renderSql</a></span><span class="op">(</span><span class="va">sql</span>,
-  cdmDatabaseSchema <span class="op">=</span> <span class="va">cdmDatabaseSchema</span>,
-  cohortsDatabaseSchema <span class="op">=</span> <span class="va">cohortsDatabaseSchema</span>,
-  post_time <span class="op">=</span> <span class="fl">30</span>,
-  pre_time <span class="op">=</span> <span class="fl">365</span><span class="op">)</span><span class="op">$</span><span class="va">sql</span>
-  <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/translateSql.html">translateSql</a></span><span class="op">(</span><span class="va">sql</span>, targetDialect <span class="op">=</span> <span class="va">connectionDetails</span><span class="op">$</span><span class="va">dbms</span><span class="op">)</span><span class="op">$</span><span class="va">sql</span>
-  
-  <span class="va">connection</span> <span class="op">&lt;-</span> <span class="fu">connect</span><span class="op">(</span><span class="va">connectionDetails</span><span class="op">)</span>
-  <span class="fu">executeSql</span><span class="op">(</span><span class="va">connection</span>, <span class="va">sql</span><span class="op">)</span></code></pre></div>
-<p>In this code, we first read the SQL from the file into memory. In the next line, we replace four parameter names with the actual values. We then translate the SQL into the dialect appropriate for the DBMS we already specified in the <code>connectionDetails</code>. Next, we connect to the server, and submit the rendered and translated SQL.</p>
-<p>If all went well, we now have a table with the events of interest. We can see how many events per type:</p>
+<code class="sourceCode R"><span>  <span class="kw"><a href="https://rdrr.io/r/base/library.html" class="external-link">library</a></span><span class="op">(</span><span class="va"><a href="https://ohdsi.github.io/SqlRender/" class="external-link">SqlRender</a></span><span class="op">)</span></span>
+<span>  <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/readSql.html" class="external-link">readSql</a></span><span class="op">(</span><span class="st">"AfStrokeCohorts.sql"</span><span class="op">)</span></span>
+<span>  <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/renderSql.html" class="external-link">renderSql</a></span><span class="op">(</span><span class="va">sql</span>,</span>
+<span>  cdmDatabaseSchema <span class="op">=</span> <span class="va">cdmDatabaseSchema</span>,</span>
+<span>  cohortsDatabaseSchema <span class="op">=</span> <span class="va">cohortsDatabaseSchema</span>,</span>
+<span>  post_time <span class="op">=</span> <span class="fl">30</span>,</span>
+<span>  pre_time <span class="op">=</span> <span class="fl">365</span><span class="op">)</span><span class="op">$</span><span class="va">sql</span></span>
+<span>  <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/translateSql.html" class="external-link">translateSql</a></span><span class="op">(</span><span class="va">sql</span>, targetDialect <span class="op">=</span> <span class="va">connectionDetails</span><span class="op">$</span><span class="va">dbms</span><span class="op">)</span><span class="op">$</span><span class="va">sql</span></span>
+<span>  </span>
+<span>  <span class="va">connection</span> <span class="op">&lt;-</span> <span class="fu">connect</span><span class="op">(</span><span class="va">connectionDetails</span><span class="op">)</span></span>
+<span>  <span class="fu">executeSql</span><span class="op">(</span><span class="va">connection</span>, <span class="va">sql</span><span class="op">)</span></span></code></pre></div>
+<p>In this code, we first read the SQL from the file into memory. In the
+next line, we replace four parameter names with the actual values. We
+then translate the SQL into the dialect appropriate for the DBMS we
+already specified in the <code>connectionDetails</code>. Next, we
+connect to the server, and submit the rendered and translated SQL.</p>
+<p>If all went well, we now have a table with the events of interest. We
+can see how many events per type:</p>
 <div class="sourceCode" id="cb4"><pre class="downlit sourceCode r">
-<code class="sourceCode R">  <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste</a></span><span class="op">(</span><span class="st">"SELECT cohort_definition_id, COUNT(*) AS count"</span>,
-  <span class="st">"FROM @cohortsDatabaseSchema.AFibStrokeCohort"</span>,
-  <span class="st">"GROUP BY cohort_definition_id"</span><span class="op">)</span>
-  <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/renderSql.html">renderSql</a></span><span class="op">(</span><span class="va">sql</span>, cohortsDatabaseSchema <span class="op">=</span> <span class="va">cohortsDatabaseSchema</span><span class="op">)</span><span class="op">$</span><span class="va">sql</span>
-  <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/translateSql.html">translateSql</a></span><span class="op">(</span><span class="va">sql</span>, targetDialect <span class="op">=</span> <span class="va">connectionDetails</span><span class="op">$</span><span class="va">dbms</span><span class="op">)</span><span class="op">$</span><span class="va">sql</span>
-  
-  <span class="fu">querySql</span><span class="op">(</span><span class="va">connection</span>, <span class="va">sql</span><span class="op">)</span></code></pre></div>
-<pre><code>##   cohort_definition_id  count
-## 1                    1 527616
-## 2                    2 221555</code></pre>
-</div>
-<div id="study-script-creation" class="section level3">
-<h3 class="hasAnchor">
-<a href="#study-script-creation" class="anchor"></a>Study script creation</h3>
-<p>In this section we assume that our cohorts have been created either by using ATLAS or a custom SQL script. We will first explain how to create an R script yourself that will execute our study as we have defined earlier.</p>
-</div>
-<div id="data-extraction" class="section level3">
-<h3 class="hasAnchor">
-<a href="#data-extraction" class="anchor"></a>Data extraction</h3>
-<p>Now we can tell <code>PatientLevelPrediction</code> to extract all necessary data for our analysis. This is done using the <a href="https://github.com/OHDSI/FeatureExtration"><code>FeatureExtractionPackage</code></a>. In short the FeatureExtractionPackage allows you to specify which features (covariates) need to be extracted, e.g. all conditions and drug exposures. It also supports the creation of custom covariates. For more detailed information on the FeatureExtraction package see its <a href="https://github.com/OHDSI/FeatureExtration">vignettes</a>. For our example study we decided to use these settings:</p>
+<code class="sourceCode R"><span>  <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html" class="external-link">paste</a></span><span class="op">(</span><span class="st">"SELECT cohort_definition_id, COUNT(*) AS count"</span>,</span>
+<span>  <span class="st">"FROM @cohortsDatabaseSchema.AFibStrokeCohort"</span>,</span>
+<span>  <span class="st">"GROUP BY cohort_definition_id"</span><span class="op">)</span></span>
+<span>  <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/renderSql.html" class="external-link">renderSql</a></span><span class="op">(</span><span class="va">sql</span>, cohortsDatabaseSchema <span class="op">=</span> <span class="va">cohortsDatabaseSchema</span><span class="op">)</span><span class="op">$</span><span class="va">sql</span></span>
+<span>  <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/translateSql.html" class="external-link">translateSql</a></span><span class="op">(</span><span class="va">sql</span>, targetDialect <span class="op">=</span> <span class="va">connectionDetails</span><span class="op">$</span><span class="va">dbms</span><span class="op">)</span><span class="op">$</span><span class="va">sql</span></span>
+<span>  </span>
+<span>  <span class="fu">querySql</span><span class="op">(</span><span class="va">connection</span>, <span class="va">sql</span><span class="op">)</span></span></code></pre></div>
+<pre><code><span><span class="co">##   cohort_definition_id  count</span></span>
+<span><span class="co">## 1                    1 527616</span></span>
+<span><span class="co">## 2                    2 221555</span></span></code></pre>
+</div>
+<div class="section level4">
+<h4 id="study-script-creation">Study script creation<a class="anchor" aria-label="anchor" href="#study-script-creation"></a>
+</h4>
+<p>In this section we assume that our cohorts have been created either
+by using ATLAS or a custom SQL script. We will first explain how to
+create an R script yourself that will execute our study as we have
+defined earlier.</p>
+</div>
+<div class="section level4">
+<h4 id="data-extraction">Data extraction<a class="anchor" aria-label="anchor" href="#data-extraction"></a>
+</h4>
+<p>Now we can tell <code>PatientLevelPrediction</code> to extract all
+necessary data for our analysis. This is done using the <a href="https://github.com/OHDSI/FeatureExtraction" class="external-link"><code>FeatureExtractionPackage</code></a>.
+In short the FeatureExtractionPackage allows you to specify which
+features (covariates) need to be extracted, e.g. all conditions and drug
+exposures. It also supports the creation of custom covariates. For more
+detailed information on the FeatureExtraction package see its <a href="https://github.com/OHDSI/FeatureExtraction" class="external-link">vignettes</a>. For our
+example study we decided to use these settings:</p>
 <div class="sourceCode" id="cb6"><pre class="downlit sourceCode r">
-<code class="sourceCode R">  <span class="va">covariateSettings</span> <span class="op">&lt;-</span> <span class="fu">createCovariateSettings</span><span class="op">(</span>useDemographicsGender <span class="op">=</span> <span class="cn">TRUE</span>,
-  useDemographicsAge <span class="op">=</span> <span class="cn">TRUE</span>,
-  useConditionGroupEraLongTerm <span class="op">=</span> <span class="cn">TRUE</span>,
-  useConditionGroupEraAnyTimePrior <span class="op">=</span> <span class="cn">TRUE</span>,
-  useDrugGroupEraLongTerm <span class="op">=</span> <span class="cn">TRUE</span>,
-  useDrugGroupEraAnyTimePrior <span class="op">=</span> <span class="cn">TRUE</span>,
-  useVisitConceptCountLongTerm <span class="op">=</span> <span class="cn">TRUE</span>,
-  longTermStartDays <span class="op">=</span> <span class="op">-</span><span class="fl">365</span>,
-  endDays <span class="op">=</span> <span class="op">-</span><span class="fl">1</span><span class="op">)</span></code></pre></div>
-<p>The final step for extracting the data is to run the <code>getPlpData</code> function and input the connection details, the database schema where the cohorts are stored, the cohort definition ids for the cohort and outcome, and the washoutPeriod which is the minimum number of days prior to cohort index date that the person must have been observed to be included into the data, and finally input the previously constructed covariate settings.</p>
+<code class="sourceCode R"><span>  <span class="va">covariateSettings</span> <span class="op">&lt;-</span> <span class="fu">createCovariateSettings</span><span class="op">(</span>useDemographicsGender <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>  useDemographicsAge <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>  useConditionGroupEraLongTerm <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>  useConditionGroupEraAnyTimePrior <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>  useDrugGroupEraLongTerm <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>  useDrugGroupEraAnyTimePrior <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>  useVisitConceptCountLongTerm <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>  longTermStartDays <span class="op">=</span> <span class="op">-</span><span class="fl">365</span>,</span>
+<span>  endDays <span class="op">=</span> <span class="op">-</span><span class="fl">1</span><span class="op">)</span></span></code></pre></div>
+<p>The final step for extracting the data is to run the
+<code>getPlpData</code> function and input the connection details, the
+database schema where the cohorts are stored, the cohort definition ids
+for the cohort and outcome, and the washoutPeriod which is the minimum
+number of days prior to cohort index date that the person must have been
+observed to be included into the data, and finally input the previously
+constructed covariate settings.</p>
 <div class="sourceCode" id="cb7"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">databaseDetails</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDatabaseDetails.html">createDatabaseDetails</a></span><span class="op">(</span>
-  connectionDetails <span class="op">=</span> <span class="va">connectionDetails</span>,
-  cdmDatabaseSchema <span class="op">=</span> <span class="va">cdmDatabaseSchema</span>,
-  cdmDatabaseName <span class="op">=</span> <span class="st">''</span>,
-  cohortDatabaseSchema <span class="op">=</span> <span class="va">resultsDatabaseSchema</span>,
-  cohortTable <span class="op">=</span> <span class="st">'AFibStrokeCohort'</span>,
-  cohortId <span class="op">=</span> <span class="fl">1</span>,
-  outcomeDatabaseSchema <span class="op">=</span> <span class="va">resultsDatabaseSchema</span>,
-  outcomeTable <span class="op">=</span> <span class="st">'AFibStrokeCohort'</span>,
-  outcomeIds <span class="op">=</span> <span class="fl">2</span>,
-  cdmVersion <span class="op">=</span> <span class="fl">5</span>
-  <span class="op">)</span>
-
-<span class="co"># here you can define whether you want to sample the target cohort and add any</span>
-<span class="co"># restrictions based on minimum prior observation, index date restrictions</span>
-<span class="co"># or restricting to first index date (if people can be in target cohort multiple times)</span>
-<span class="va">restrictPlpDataSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createRestrictPlpDataSettings.html">createRestrictPlpDataSettings</a></span><span class="op">(</span>sampleSize <span class="op">=</span> <span class="fl">10000</span><span class="op">)</span>
-
-  <span class="va">plpData</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/getPlpData.html">getPlpData</a></span><span class="op">(</span>
-    databaseDetails <span class="op">=</span> <span class="va">databaseDetails</span>, 
-    covariateSettings <span class="op">=</span> <span class="va">covariateSettings</span>,
-    restrictPlpDataSettings <span class="op">=</span> <span class="va">restrictPlpDataSettings</span>
-  <span class="op">)</span></code></pre></div>
-<p>Note that if the cohorts are created in ATLAS its corresponding cohort database schema needs to be selected. There are many additional parameters for the <code>createRestrictPlpDataSettings</code> function which are all documented in the <code>PatientLevelPrediction</code> manual. The resulting <code>plpData</code> object uses the package <code>Andromeda</code> (which uses <a href="https://www.sqlite.org/index.html">SQLite</a>) to store information in a way that ensures R does not run out of memory, even when the data are large.</p>
-<p>Creating the <code>plpData</code> object can take considerable computing time, and it is probably a good idea to save it for future sessions. Because <code>plpData</code> uses <code>Andromeda</code>, we cannot use R’s regular save function. Instead, we’ll have to use the <code><a href="../reference/savePlpData.html">savePlpData()</a></code> function:</p>
+<code class="sourceCode R"><span><span class="va">databaseDetails</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDatabaseDetails.html">createDatabaseDetails</a></span><span class="op">(</span></span>
+<span>  connectionDetails <span class="op">=</span> <span class="va">connectionDetails</span>,</span>
+<span>  cdmDatabaseSchema <span class="op">=</span> <span class="va">cdmDatabaseSchema</span>,</span>
+<span>  cdmDatabaseName <span class="op">=</span> <span class="st">''</span>,</span>
+<span>  cohortDatabaseSchema <span class="op">=</span> <span class="va">resultsDatabaseSchema</span>,</span>
+<span>  cohortTable <span class="op">=</span> <span class="st">'AFibStrokeCohort'</span>,</span>
+<span>  cohortId <span class="op">=</span> <span class="fl">1</span>,</span>
+<span>  outcomeDatabaseSchema <span class="op">=</span> <span class="va">resultsDatabaseSchema</span>,</span>
+<span>  outcomeTable <span class="op">=</span> <span class="st">'AFibStrokeCohort'</span>,</span>
+<span>  outcomeIds <span class="op">=</span> <span class="fl">2</span>,</span>
+<span>  cdmVersion <span class="op">=</span> <span class="fl">5</span></span>
+<span>  <span class="op">)</span></span>
+<span></span>
+<span><span class="co"># here you can define whether you want to sample the target cohort and add any</span></span>
+<span><span class="co"># restrictions based on minimum prior observation, index date restrictions</span></span>
+<span><span class="co"># or restricting to first index date (if people can be in target cohort multiple times)</span></span>
+<span><span class="va">restrictPlpDataSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createRestrictPlpDataSettings.html">createRestrictPlpDataSettings</a></span><span class="op">(</span>sampleSize <span class="op">=</span> <span class="fl">10000</span><span class="op">)</span></span>
+<span></span>
+<span>  <span class="va">plpData</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/getPlpData.html">getPlpData</a></span><span class="op">(</span></span>
+<span>    databaseDetails <span class="op">=</span> <span class="va">databaseDetails</span>, </span>
+<span>    covariateSettings <span class="op">=</span> <span class="va">covariateSettings</span>,</span>
+<span>    restrictPlpDataSettings <span class="op">=</span> <span class="va">restrictPlpDataSettings</span></span>
+<span>  <span class="op">)</span></span></code></pre></div>
+<p>Note that if the cohorts are created in ATLAS its corresponding
+cohort database schema needs to be selected. There are many additional
+parameters for the <code>createRestrictPlpDataSettings</code> function
+which are all documented in the <code>PatientLevelPrediction</code>
+manual. The resulting <code>plpData</code> object uses the package
+<code>Andromeda</code> (which uses <a href="https://www.sqlite.org/index.html" class="external-link">SQLite</a>) to store
+information in a way that ensures R does not run out of memory, even
+when the data are large.</p>
+<p>Creating the <code>plpData</code> object can take considerable
+computing time, and it is probably a good idea to save it for future
+sessions. Because <code>plpData</code> uses <code>Andromeda</code>, we
+cannot use R’s regular save function. Instead, we’ll have to use the
+<code><a href="../reference/savePlpData.html">savePlpData()</a></code> function:</p>
 <div class="sourceCode" id="cb8"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="../reference/savePlpData.html">savePlpData</a></span><span class="op">(</span><span class="va">plpData</span>, <span class="st">"stroke_in_af_data"</span><span class="op">)</span></code></pre></div>
-<p>We can use the <code><a href="../reference/loadPlpData.html">loadPlpData()</a></code> function to load the data in a future session.</p>
-</div>
-<div id="additional-inclusion-criteria" class="section level3">
-<h3 class="hasAnchor">
-<a href="#additional-inclusion-criteria" class="anchor"></a>Additional inclusion criteria</h3>
-<p>To completely define the prediction problem the final study population is obtained by applying additional constraints on the two earlier defined cohorts, e.g., a minumim time at risk can be enforced (<code>requireTimeAtRisk, minTimeAtRisk</code>) and we can specify if this also applies to patients with the outcome (<code>includeAllOutcomes</code>). Here we also specify the start and end of the risk window relative to target cohort start. For example, if we like the risk window to start 30 days after the at-risk cohort start and end a year later we can set <code>riskWindowStart = 30</code> and <code>riskWindowEnd = 365</code>. In some cases the risk window needs to start at the cohort end date. This can be achieved by setting <code>addExposureToStart = TRUE</code> which adds the cohort (exposure) time to the start date.</p>
-<p>In Appendix 1, we demonstrate the effect of these settings on the subset of the persons in the target cohort that end up in the final study population.</p>
-<p>In the example below all the settings we defined for our study are imposed:</p>
+<code class="sourceCode R"><span><span class="fu"><a href="../reference/savePlpData.html">savePlpData</a></span><span class="op">(</span><span class="va">plpData</span>, <span class="st">"stroke_in_af_data"</span><span class="op">)</span></span></code></pre></div>
+<p>We can use the <code><a href="../reference/loadPlpData.html">loadPlpData()</a></code> function to load the data
+in a future session.</p>
+</div>
+<div class="section level4">
+<h4 id="additional-inclusion-criteria">Additional inclusion criteria<a class="anchor" aria-label="anchor" href="#additional-inclusion-criteria"></a>
+</h4>
+<p>To completely define the prediction problem the final study
+population is obtained by applying additional constraints on the two
+earlier defined cohorts, e.g., a minumim time at risk can be enforced
+(<code>requireTimeAtRisk, minTimeAtRisk</code>) and we can specify if
+this also applies to patients with the outcome
+(<code>includeAllOutcomes</code>). Here we also specify the start and
+end of the risk window relative to target cohort start. For example, if
+we like the risk window to start 30 days after the at-risk cohort start
+and end a year later we can set <code>riskWindowStart = 30</code> and
+<code>riskWindowEnd = 365</code>. In some cases the risk window needs to
+start at the cohort end date. This can be achieved by setting
+<code>addExposureToStart = TRUE</code> which adds the cohort (exposure)
+time to the start date.</p>
+<p>In Appendix 1, we demonstrate the effect of these settings on the
+subset of the persons in the target cohort that end up in the final
+study population.</p>
+<p>In the example below all the settings we defined for our study are
+imposed:</p>
 <div class="sourceCode" id="cb9"><pre class="downlit sourceCode r">
-<code class="sourceCode R">  <span class="va">populationSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createStudyPopulationSettings.html">createStudyPopulationSettings</a></span><span class="op">(</span>
-  washoutPeriod <span class="op">=</span> <span class="fl">1095</span>,
-  firstExposureOnly <span class="op">=</span> <span class="cn">FALSE</span>,
-  removeSubjectsWithPriorOutcome <span class="op">=</span> <span class="cn">FALSE</span>,
-  priorOutcomeLookback <span class="op">=</span> <span class="fl">1</span>,
-  riskWindowStart <span class="op">=</span> <span class="fl">1</span>,
-  riskWindowEnd <span class="op">=</span> <span class="fl">365</span>,
-  startAnchor <span class="op">=</span>  <span class="st">'cohort start'</span>,
-  endAnchor <span class="op">=</span>  <span class="st">'cohort start'</span>,
-  minTimeAtRisk <span class="op">=</span> <span class="fl">364</span>,
-  requireTimeAtRisk <span class="op">=</span> <span class="cn">TRUE</span>,
-  includeAllOutcomes <span class="op">=</span> <span class="cn">TRUE</span>
-  <span class="op">)</span></code></pre></div>
-</div>
-<div id="spliting-the-data-into-trainingvalidationtesting-datasets" class="section level3">
-<h3 class="hasAnchor">
-<a href="#spliting-the-data-into-trainingvalidationtesting-datasets" class="anchor"></a>Spliting the data into training/validation/testing datasets</h3>
-<p>When developing a prediction model using supervised learning (when you have features paired with labels for a set of patients), the first step is to design the development/internal validation process. This requires specifying how to select the model hyper-parameters, how to learn the model parameters and how to fairly evaluate the model. In general, the validation set is used to pick hyper-parameters, the training set is used to learn the model parameters and the test set is used to perform fair internal validation. However, cross-validation can be implemented to pick the hyper-parameters on the training data (so a validation data set is not required). Cross validation can also be used to estimate internal validation (so a testing data set is not required).</p>
-<p>In small data the best approach for internal validation has been shown to be boostrapping. However, in big data (many patients and many features) bootstrapping is generally not feasible. In big data our research has shown that it is just important to have some form of fair evaluation (use a test set or cross validation). For full details see <a href="add%20link">our BMJ open paper</a>.</p>
-<p>In the PatientLevelPrediction package, the splitSettings define how the plpData are partitioned into training/validation/testing data. Cross validation is always done, but using a test set is optional (when the data are small, it may be optimal to not use a test set). For the splitSettings we can use the type (stratified/time/subject) and testFraction parameters to split the data in a 75%-25% split and run the patient-level prediction pipeline:</p>
+<code class="sourceCode R"><span>  <span class="va">populationSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createStudyPopulationSettings.html">createStudyPopulationSettings</a></span><span class="op">(</span></span>
+<span>  washoutPeriod <span class="op">=</span> <span class="fl">1095</span>,</span>
+<span>  firstExposureOnly <span class="op">=</span> <span class="cn">FALSE</span>,</span>
+<span>  removeSubjectsWithPriorOutcome <span class="op">=</span> <span class="cn">FALSE</span>,</span>
+<span>  priorOutcomeLookback <span class="op">=</span> <span class="fl">1</span>,</span>
+<span>  riskWindowStart <span class="op">=</span> <span class="fl">1</span>,</span>
+<span>  riskWindowEnd <span class="op">=</span> <span class="fl">365</span>,</span>
+<span>  startAnchor <span class="op">=</span>  <span class="st">'cohort start'</span>,</span>
+<span>  endAnchor <span class="op">=</span>  <span class="st">'cohort start'</span>,</span>
+<span>  minTimeAtRisk <span class="op">=</span> <span class="fl">364</span>,</span>
+<span>  requireTimeAtRisk <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>  includeAllOutcomes <span class="op">=</span> <span class="cn">TRUE</span></span>
+<span>  <span class="op">)</span></span></code></pre></div>
+</div>
+<div class="section level4">
+<h4 id="spliting-the-data-into-trainingvalidationtesting-datasets">Spliting the data into training/validation/testing datasets<a class="anchor" aria-label="anchor" href="#spliting-the-data-into-trainingvalidationtesting-datasets"></a>
+</h4>
+<p>When developing a prediction model using supervised learning (when
+you have features paired with labels for a set of patients), the first
+step is to design the development/internal validation process. This
+requires specifying how to select the model hyper-parameters, how to
+learn the model parameters and how to fairly evaluate the model. In
+general, the validation set is used to pick hyper-parameters, the
+training set is used to learn the model parameters and the test set is
+used to perform fair internal validation. However, cross-validation can
+be implemented to pick the hyper-parameters on the training data (so a
+validation data set is not required). Cross validation can also be used
+to estimate internal validation (so a testing data set is not
+required).</p>
+<p>In small data the best approach for internal validation has been
+shown to be boostrapping. However, in big data (many patients and many
+features) bootstrapping is generally not feasible. In big data our
+research has shown that it is just important to have some form of fair
+evaluation (use a test set or cross validation). For full details see <a href="add%20link">our BMJ open paper</a>.</p>
+<p>In the PatientLevelPrediction package, the splitSettings define how
+the plpData are partitioned into training/validation/testing data. Cross
+validation is always done, but using a test set is optional (when the
+data are small, it may be optimal to not use a test set). For the
+splitSettings we can use the type (stratified/time/subject) and
+testFraction parameters to split the data in a 75%-25% split and run the
+patient-level prediction pipeline:</p>
 <div class="sourceCode" id="cb10"><pre class="downlit sourceCode r">
-<code class="sourceCode R">  <span class="va">splitSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDefaultSplitSetting.html">createDefaultSplitSetting</a></span><span class="op">(</span>
-    trainFraction <span class="op">=</span> <span class="fl">0.75</span>,
-    testFraction <span class="op">=</span> <span class="fl">0.25</span>,
-    type <span class="op">=</span> <span class="st">'stratified'</span>,
-    nfold <span class="op">=</span> <span class="fl">2</span>, 
-    splitSeed <span class="op">=</span> <span class="fl">1234</span>
-    <span class="op">)</span></code></pre></div>
-<p>Note: it is possible to add a custom method to specify how the plpData are partitioned into training/validation/testing data, see <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSplitting.pdf">vignette for custom splitting</a>.</p>
-</div>
-<div id="preprocessing-the-training-data" class="section level3">
-<h3 class="hasAnchor">
-<a href="#preprocessing-the-training-data" class="anchor"></a>Preprocessing the training data</h3>
-<p>There a numerous data processing settings that a user must specify when developing a prediction model. These are: * Whether to under-sample or over-sample the training data (this may be useful when there is class imballance (e.g., the outcome is very rare or very common)) * Whether to perform feature engineering or feature selection (e.g., create latent variables that are not observed in the data or reduce the dimensionality of the data) * Whether to remove redundant features and normalize the data (this is required for some models)</p>
-<p>The default sample settings does nothing, it simply returns the trainData as input, see below:</p>
+<code class="sourceCode R"><span>  <span class="va">splitSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDefaultSplitSetting.html">createDefaultSplitSetting</a></span><span class="op">(</span></span>
+<span>    trainFraction <span class="op">=</span> <span class="fl">0.75</span>,</span>
+<span>    testFraction <span class="op">=</span> <span class="fl">0.25</span>,</span>
+<span>    type <span class="op">=</span> <span class="st">'stratified'</span>,</span>
+<span>    nfold <span class="op">=</span> <span class="fl">2</span>, </span>
+<span>    splitSeed <span class="op">=</span> <span class="fl">1234</span></span>
+<span>    <span class="op">)</span></span></code></pre></div>
+<p>Note: it is possible to add a custom method to specify how the
+plpData are partitioned into training/validation/testing data, see <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSplitting.pdf" class="external-link">vignette
+for custom splitting</a>.</p>
+</div>
+<div class="section level4">
+<h4 id="preprocessing-the-training-data">Preprocessing the training data<a class="anchor" aria-label="anchor" href="#preprocessing-the-training-data"></a>
+</h4>
+<p>There a numerous data processing settings that a user must specify
+when developing a prediction model. These are: * Whether to under-sample
+or over-sample the training data (this may be useful when there is class
+imballance (e.g., the outcome is very rare or very common)) * Whether to
+perform feature engineering or feature selection (e.g., create latent
+variables that are not observed in the data or reduce the dimensionality
+of the data) * Whether to remove redundant features and normalize the
+data (this is required for some models)</p>
+<p>The default sample settings does nothing, it simply returns the
+trainData as input, see below:</p>
 <div class="sourceCode" id="cb11"><pre class="downlit sourceCode r">
-<code class="sourceCode R">  <span class="va">sampleSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createSampleSettings.html">createSampleSettings</a></span><span class="op">(</span><span class="op">)</span></code></pre></div>
-<p>However, the current package contains methods of under-sampling the non-outcome patients. To perform undersampling, the <code>type</code> input should be ‘underSample’ and <code>numberOutcomestoNonOutcomes</code> must be specified (an integer specifying the number of non-outcomes per outcome). It is possible to add any custom function for over/under sampling, see <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSamples.pdf">vignette for custom sampling</a>.</p>
-<p>It is possible to specify a combination of feature engineering functions that take as input the trainData and output a new trainData with different features. The default feature engineering setting does nothing:</p>
+<code class="sourceCode R"><span>  <span class="va">sampleSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createSampleSettings.html">createSampleSettings</a></span><span class="op">(</span><span class="op">)</span></span></code></pre></div>
+<p>However, the current package contains methods of under-sampling the
+non-outcome patients. To perform undersampling, the <code>type</code>
+input should be ‘underSample’ and
+<code>numberOutcomestoNonOutcomes</code> must be specified (an integer
+specifying the number of non-outcomes per outcome). It is possible to
+add any custom function for over/under sampling, see <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSamples.pdf" class="external-link">vignette
+for custom sampling</a>.</p>
+<p>It is possible to specify a combination of feature engineering
+functions that take as input the trainData and output a new trainData
+with different features. The default feature engineering setting does
+nothing:</p>
 <div class="sourceCode" id="cb12"><pre class="downlit sourceCode r">
-<code class="sourceCode R">  <span class="va">featureEngineeringSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createFeatureEngineeringSettings.html">createFeatureEngineeringSettings</a></span><span class="op">(</span><span class="op">)</span></code></pre></div>
-<p>However, it is possible to add custom feature engineering functions into the pipeline, see <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomFeatureEngineering.pdf">vignette for custom feature engineering</a>.</p>
-<p>Finally, the preprocessing setting is required. For this setting the user can define <code>minFraction</code>, this removes any features that is observed in the training data for less than 0.01 fraction of the patients. So, if <code>minFraction = 0.01</code> then any feature that is seen in less than 1 percent of the target population is removed. The input <code>normalize</code> specifies whether the features are scaled between 0 and 1, this is required for certain models (e.g., LASSO logistic regression). The input <code>removeRedundancy</code> specifies whether features that are observed in all of the target population are removed.</p>
+<code class="sourceCode R"><span>  <span class="va">featureEngineeringSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createFeatureEngineeringSettings.html">createFeatureEngineeringSettings</a></span><span class="op">(</span><span class="op">)</span></span></code></pre></div>
+<p>However, it is possible to add custom feature engineering functions
+into the pipeline, see <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomFeatureEngineering.pdf" class="external-link">vignette
+for custom feature engineering</a>.</p>
+<p>Finally, the preprocessing setting is required. For this setting the
+user can define <code>minFraction</code>, this removes any features that
+is observed in the training data for less than 0.01 fraction of the
+patients. So, if <code>minFraction = 0.01</code> then any feature that
+is seen in less than 1 percent of the target population is removed. The
+input <code>normalize</code> specifies whether the features are scaled
+between 0 and 1, this is required for certain models (e.g., LASSO
+logistic regression). The input <code>removeRedundancy</code> specifies
+whether features that are observed in all of the target population are
+removed.</p>
 <div class="sourceCode" id="cb13"><pre class="downlit sourceCode r">
-<code class="sourceCode R">  <span class="va">preprocessSettingsSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createPreprocessSettings.html">createPreprocessSettings</a></span><span class="op">(</span>
-    minFraction <span class="op">=</span> <span class="fl">0.01</span>, 
-    normalize <span class="op">=</span> <span class="cn">T</span>, 
-    removeRedundancy <span class="op">=</span> <span class="cn">T</span>
-      <span class="op">)</span></code></pre></div>
-</div>
-<div id="model-development" class="section level3">
-<h3 class="hasAnchor">
-<a href="#model-development" class="anchor"></a>Model Development</h3>
-<p>In the set function of an algorithm the user can specify a list of eligible values for each hyper-parameter. All possible combinations of the hyper-parameters are included in a so-called grid search using cross-validation on the training set. If a user does not specify any value then the default value is used instead.</p>
-<p>For example, if we use the following settings for the gradientBoostingMachine: ntrees=c(100,200), maxDepth=4 the grid search will apply the gradient boosting machine algorithm with ntrees=100 and maxDepth=4 plus the default settings for other hyper-parameters and ntrees=200 and maxDepth=4 plus the default settings for other hyper-parameters. The hyper-parameters that lead to the bestcross-validation performance will then be chosen for the final model. For our problem we choose to build a logistic regression model with the default hyper-parameters</p>
+<code class="sourceCode R"><span>  <span class="va">preprocessSettingsSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createPreprocessSettings.html">createPreprocessSettings</a></span><span class="op">(</span></span>
+<span>    minFraction <span class="op">=</span> <span class="fl">0.01</span>, </span>
+<span>    normalize <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>    removeRedundancy <span class="op">=</span> <span class="cn">T</span></span>
+<span>      <span class="op">)</span></span></code></pre></div>
+</div>
+<div class="section level4">
+<h4 id="model-development">Model Development<a class="anchor" aria-label="anchor" href="#model-development"></a>
+</h4>
+<p>In the set function of an algorithm the user can specify a list of
+eligible values for each hyper-parameter. All possible combinations of
+the hyper-parameters are included in a so-called grid search using
+cross-validation on the training set. If a user does not specify any
+value then the default value is used instead.</p>
+<p>For example, if we use the following settings for the
+gradientBoostingMachine: ntrees=c(100,200), maxDepth=4 the grid search
+will apply the gradient boosting machine algorithm with ntrees=100 and
+maxDepth=4 plus the default settings for other hyper-parameters and
+ntrees=200 and maxDepth=4 plus the default settings for other
+hyper-parameters. The hyper-parameters that lead to the
+bestcross-validation performance will then be chosen for the final
+model. For our problem we choose to build a logistic regression model
+with the default hyper-parameters</p>
 <div class="sourceCode" id="cb14"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">lrModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span></code></pre></div>
-<p>The <code>runPlP</code> function requires the <code>plpData</code>, the <code>outcomeId</code> specifying the outcome being predicted and the settings: <code>populationSettings</code>, <code>splitSettings</code>, <code>sampleSettings</code>, <code>featureEngineeringSettings</code>, <code>preprocessSettings</code> and <code>modelSettings</code> to train and evaluate the model.</p>
+<code class="sourceCode R"><span><span class="va">lrModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span></span></code></pre></div>
+<p>The <code>runPlP</code> function requires the <code>plpData</code>,
+the <code>outcomeId</code> specifying the outcome being predicted and
+the settings: <code>populationSettings</code>,
+<code>splitSettings</code>, <code>sampleSettings</code>,
+<code>featureEngineeringSettings</code>, <code>preprocessSettings</code>
+and <code>modelSettings</code> to train and evaluate the model.</p>
 <div class="sourceCode" id="cb15"><pre class="downlit sourceCode r">
-<code class="sourceCode R">  <span class="va">lrResults</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/runPlp.html">runPlp</a></span><span class="op">(</span>
-    plpData <span class="op">=</span> <span class="va">plpData</span>,
-    outcomeId <span class="op">=</span> <span class="fl">2</span>, 
-    analysisId <span class="op">=</span> <span class="st">'singleDemo'</span>,
-    analysisName <span class="op">=</span> <span class="st">'Demonstration of runPlp for training single PLP models'</span>,
-    populationSettings <span class="op">=</span> <span class="va">populationSettings</span>, 
-    splitSettings <span class="op">=</span> <span class="va">splitSettings</span>,
-    sampleSettings <span class="op">=</span> <span class="va">sampleSettings</span>, 
-    featureEngineeringSettings <span class="op">=</span> <span class="va">featureEngineeringSettings</span>, 
-    preprocessSettings <span class="op">=</span> <span class="va">preprocessSettings</span>,
-    modelSettings <span class="op">=</span> <span class="va">lrModel</span>,
-    logSettings <span class="op">=</span> <span class="fu"><a href="../reference/createLogSettings.html">createLogSettings</a></span><span class="op">(</span><span class="op">)</span>, 
-    executeSettings <span class="op">=</span> <span class="fu"><a href="../reference/createExecuteSettings.html">createExecuteSettings</a></span><span class="op">(</span>
-      runSplitData <span class="op">=</span> <span class="cn">T</span>, 
-      runSampleData <span class="op">=</span> <span class="cn">T</span>, 
-      runfeatureEngineering <span class="op">=</span> <span class="cn">T</span>, 
-      runPreprocessData <span class="op">=</span> <span class="cn">T</span>, 
-      runModelDevelopment <span class="op">=</span> <span class="cn">T</span>, 
-      runCovariateSummary <span class="op">=</span> <span class="cn">T</span>
-    <span class="op">)</span>, 
-    saveDirectory <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">'singlePlp'</span><span class="op">)</span>
-    <span class="op">)</span></code></pre></div>
-<p>Under the hood the package will now use the <a href="www.github.com/OHDSI/Cyclops"><code>Cyclops</code></a> package to fit a large-scale regularized regression using 75% of the data and will evaluate the model on the remaining 25%. A results data structure is returned containing information about the model, its performance etc.</p>
+<code class="sourceCode R"><span>  <span class="va">lrResults</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/runPlp.html">runPlp</a></span><span class="op">(</span></span>
+<span>    plpData <span class="op">=</span> <span class="va">plpData</span>,</span>
+<span>    outcomeId <span class="op">=</span> <span class="fl">2</span>, </span>
+<span>    analysisId <span class="op">=</span> <span class="st">'singleDemo'</span>,</span>
+<span>    analysisName <span class="op">=</span> <span class="st">'Demonstration of runPlp for training single PLP models'</span>,</span>
+<span>    populationSettings <span class="op">=</span> <span class="va">populationSettings</span>, </span>
+<span>    splitSettings <span class="op">=</span> <span class="va">splitSettings</span>,</span>
+<span>    sampleSettings <span class="op">=</span> <span class="va">sampleSettings</span>, </span>
+<span>    featureEngineeringSettings <span class="op">=</span> <span class="va">featureEngineeringSettings</span>, </span>
+<span>    preprocessSettings <span class="op">=</span> <span class="va">preprocessSettings</span>,</span>
+<span>    modelSettings <span class="op">=</span> <span class="va">lrModel</span>,</span>
+<span>    logSettings <span class="op">=</span> <span class="fu"><a href="../reference/createLogSettings.html">createLogSettings</a></span><span class="op">(</span><span class="op">)</span>, </span>
+<span>    executeSettings <span class="op">=</span> <span class="fu"><a href="../reference/createExecuteSettings.html">createExecuteSettings</a></span><span class="op">(</span></span>
+<span>      runSplitData <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>      runSampleData <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>      runfeatureEngineering <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>      runPreprocessData <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>      runModelDevelopment <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>      runCovariateSummary <span class="op">=</span> <span class="cn">T</span></span>
+<span>    <span class="op">)</span>, </span>
+<span>    saveDirectory <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html" class="external-link">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">'singlePlp'</span><span class="op">)</span></span>
+<span>    <span class="op">)</span></span></code></pre></div>
+<p>Under the hood the package will now use the <a href="www.github.com/OHDSI/Cyclops"><code>Cyclops</code></a> package to
+fit a large-scale regularized regression using 75% of the data and will
+evaluate the model on the remaining 25%. A results data structure is
+returned containing information about the model, its performance
+etc.</p>
 <p>You can save the model using:</p>
 <div class="sourceCode" id="cb16"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="../reference/savePlpModel.html">savePlpModel</a></span><span class="op">(</span><span class="va">lrResults</span><span class="op">$</span><span class="va">model</span>, dirPath <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"model"</span><span class="op">)</span><span class="op">)</span></code></pre></div>
+<code class="sourceCode R"><span><span class="fu"><a href="../reference/savePlpModel.html">savePlpModel</a></span><span class="op">(</span><span class="va">lrResults</span><span class="op">$</span><span class="va">model</span>, dirPath <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html" class="external-link">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"model"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <p>You can load the model using:</p>
 <div class="sourceCode" id="cb17"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">plpModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/loadPlpModel.html">loadPlpModel</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"model"</span><span class="op">)</span><span class="op">)</span></code></pre></div>
+<code class="sourceCode R"><span><span class="va">plpModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/loadPlpModel.html">loadPlpModel</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html" class="external-link">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"model"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <p>You can also save the full results structure using:</p>
 <div class="sourceCode" id="cb18"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="../reference/savePlpResult.html">savePlpResult</a></span><span class="op">(</span><span class="va">lrResults</span>, location <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"lr"</span><span class="op">)</span><span class="op">)</span></code></pre></div>
+<code class="sourceCode R"><span><span class="fu"><a href="../reference/savePlpResult.html">savePlpResult</a></span><span class="op">(</span><span class="va">lrResults</span>, location <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html" class="external-link">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"lr"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <p>To load the full results structure use:</p>
 <div class="sourceCode" id="cb19"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">lrResults</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/loadPlpResult.html">loadPlpResult</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"lr"</span><span class="op">)</span><span class="op">)</span></code></pre></div>
+<code class="sourceCode R"><span><span class="va">lrResults</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/loadPlpResult.html">loadPlpResult</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html" class="external-link">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"lr"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <div style="page-break-after: always;"></div>
 </div>
 </div>
 </div>
-<div id="example2" class="section level1">
-<h1 class="hasAnchor">
-<a href="#example2" class="anchor"></a>Example 2: Angioedema in ACE inhibitor users</h1>
-<div id="study-specification-2" class="section level2">
-<h2 class="hasAnchor">
-<a href="#study-specification-2" class="anchor"></a>Study Specification</h2>
+<div class="section level2">
+<h2 id="example2">Example 2: Angioedema in ACE inhibitor users<a class="anchor" aria-label="anchor" href="#example2"></a>
+</h2>
+<div class="section level3">
+<h3 id="study-specification-2">Study Specification<a class="anchor" aria-label="anchor" href="#study-specification-2"></a>
+</h3>
 <table class="table">
 <colgroup>
-<col width="44%">
-<col width="55%">
+<col width="30%">
+<col width="69%">
 </colgroup>
 <thead><tr class="header">
 <th>Definition</th>
@@ -734,11 +1183,13 @@ <h2 class="hasAnchor">
 </tr>
 <tr class="even">
 <td>Target Cohort (T)</td>
-<td>‘Patients who are newly dispensed an ACE inhibitor’ defined as the first drug record of any ACE inhibitor</td>
+<td>‘Patients who are newly dispensed an ACE inhibitor’ defined as the
+first drug record of any ACE inhibitor</td>
 </tr>
 <tr class="odd">
 <td>Outcome Cohort (O)</td>
-<td>‘Angioedema’ defined as an angioedema condition record during an inpatient or ER visit</td>
+<td>‘Angioedema’ defined as an angioedema condition record during an
+inpatient or ER visit</td>
 </tr>
 <tr class="even">
 <td>Time-at-risk (TAR)</td>
@@ -790,11 +1241,13 @@ <h2 class="hasAnchor">
 </tr>
 <tr class="even">
 <td>Hyper-parameters</td>
-<td>ntree:5000, max depth:4 or 7 or 10 and learning rate: 0.001 or 0.01 or 0.1 or 0.9</td>
+<td>ntree:5000, max depth:4 or 7 or 10 and learning rate: 0.001 or 0.01
+or 0.1 or 0.9</td>
 </tr>
 <tr class="odd">
 <td>Covariates</td>
-<td>Gender, Age, Conditions (ever before, &lt;365), Drugs Groups (ever before, &lt;365), and Visit Count</td>
+<td>Gender, Age, Conditions (ever before, &lt;365), Drugs Groups (ever
+before, &lt;365), and Visit Count</td>
 </tr>
 <tr class="even">
 <td>Data split</td>
@@ -802,504 +1255,812 @@ <h2 class="hasAnchor">
 </tr>
 </tbody>
 </table>
-<p>According to the best practices we need to make a protocol that completely specifies how we plan to execute our study. This protocol will be assessed by the governance boards of the participating data sources in your network study. For this a template could be used but we prefer to automate this process as much as possible by adding functionality to automatically generate study protocol from a study specification. We will discuss this in more detail later.</p>
-</div>
-<div id="study-implementation-1" class="section level2">
-<h2 class="hasAnchor">
-<a href="#study-implementation-1" class="anchor"></a>Study implementation</h2>
-<p>Now we have completely design our study we have to implement the study. We have to generate the target and outcome cohorts and we need to develop the R code to run against our CDM that will execute the full study.</p>
-<div id="cohort-instantiation-1" class="section level3">
-<h3 class="hasAnchor">
-<a href="#cohort-instantiation-1" class="anchor"></a>Cohort instantiation</h3>
-<p>For our study we need to know when a person enters the target and outcome cohorts. This is stored in a table on the server that contains the cohort start date and cohort end date for all subjects for a specific cohort definition. This cohort table has a very simple structure as shown below:</p>
+<p>According to the best practices we need to make a protocol that
+completely specifies how we plan to execute our study. This protocol
+will be assessed by the governance boards of the participating data
+sources in your network study. For this a template could be used but we
+prefer to automate this process as much as possible by adding
+functionality to automatically generate study protocol from a study
+specification. We will discuss this in more detail later.</p>
+</div>
+<div class="section level3">
+<h3 id="study-implementation-1">Study implementation<a class="anchor" aria-label="anchor" href="#study-implementation-1"></a>
+</h3>
+<p>Now we have completely design our study we have to implement the
+study. We have to generate the target and outcome cohorts and we need to
+develop the R code to run against our CDM that will execute the full
+study.</p>
+<div class="section level4">
+<h4 id="cohort-instantiation-1">Cohort instantiation<a class="anchor" aria-label="anchor" href="#cohort-instantiation-1"></a>
+</h4>
+<p>For our study we need to know when a person enters the target and
+outcome cohorts. This is stored in a table on the server that contains
+the cohort start date and cohort end date for all subjects for a
+specific cohort definition. This cohort table has a very simple
+structure as shown below:</p>
 <ul>
 <li>
-<code>cohort_definition_id</code>, a unique identifier for distinguishing between different types of cohorts, e.g. cohorts of interest and outcome cohorts.</li>
+<code>cohort_definition_id</code>, a unique identifier for
+distinguishing between different types of cohorts, e.g. cohorts of
+interest and outcome cohorts.</li>
 <li>
-<code>subject_id</code>, a unique identifier corresponding to the <code>person_id</code> in the CDM.</li>
+<code>subject_id</code>, a unique identifier corresponding to the
+<code>person_id</code> in the CDM.</li>
 <li>
-<code>cohort_start_date</code>, the date the subject enters the cohort.</li>
+<code>cohort_start_date</code>, the date the subject enters the
+cohort.</li>
 <li>
-<code>cohort_end_date</code>, the date the subject leaves the cohort.</li>
+<code>cohort_end_date</code>, the date the subject leaves the
+cohort.</li>
 </ul>
-<p>How do we fill this table according to our cohort definitions? There are two options for this:</p>
+<p>How do we fill this table according to our cohort definitions? There
+are two options for this:</p>
 <ol style="list-style-type: decimal">
-<li><p>use the interactive cohort builder tool in <a href="www.github.com/OHDSI/ATLAS">ATLAS</a> which can be used to create cohorts based on inclusion criteria and will automatically populate this cohort table.</p></li>
-<li><p>write your own custom SQL statements to fill the cohort table.</p></li>
+<li><p>use the interactive cohort builder tool in <a href="www.github.com/OHDSI/ATLAS">ATLAS</a> which can be used to create
+cohorts based on inclusion criteria and will automatically populate this
+cohort table.</p></li>
+<li><p>write your own custom SQL statements to fill the cohort
+table.</p></li>
 </ol>
-<p>Both methods are described below for our example prediction problem.</p>
+<p>Both methods are described below for our example prediction
+problem.</p>
 </div>
-<div id="atlas-cohort-builder-1" class="section level3">
-<h3 class="hasAnchor">
-<a href="#atlas-cohort-builder-1" class="anchor"></a>ATLAS cohort builder</h3>
-<div class="figure">
-<img src="example2/aceinhibitors.png" alt=""><p class="caption">Target Cohort ACE inhibitors</p>
+<div class="section level4">
+<h4 id="atlas-cohort-builder-1">ATLAS cohort builder<a class="anchor" aria-label="anchor" href="#atlas-cohort-builder-1"></a>
+</h4>
+<div class="float">
+<img src="example2/aceinhibitors.webp" alt="Target Cohort ACE inhibitors"><div class="figcaption">Target Cohort ACE inhibitors</div>
 </div>
-<p>ATLAS allows you to define cohorts interactively by specifying cohort entry and cohort exit criteria. Cohort entry criteria involve selecting one or more initial events, which determine the start date for cohort entry, and optionally specifying additional inclusion criteria which filter to the qualifying events. Cohort exit criteria are applied to each cohort entry record to determine the end date when the person’s episode no longer qualifies for the cohort. For the outcome cohort the end date is less relevant. As an example, Figure 6 shows how we created the ACE inhibitors cohort and Figure 7 shows how we created the angioedema cohort in ATLAS.</p>
-<div class="figure">
-<img src="example2/angioedema.png" alt=""><p class="caption">Outcome Cohort Angioedema</p>
+<p>ATLAS allows you to define cohorts interactively by specifying cohort
+entry and cohort exit criteria. Cohort entry criteria involve selecting
+one or more initial events, which determine the start date for cohort
+entry, and optionally specifying additional inclusion criteria which
+filter to the qualifying events. Cohort exit criteria are applied to
+each cohort entry record to determine the end date when the person’s
+episode no longer qualifies for the cohort. For the outcome cohort the
+end date is less relevant. As an example, Figure 6 shows how we created
+the ACE inhibitors cohort and Figure 7 shows how we created the
+angioedema cohort in ATLAS.</p>
+<div class="float">
+<img src="example2/angioedema.webp" alt="Outcome Cohort Angioedema"><div class="figcaption">Outcome Cohort Angioedema</div>
 </div>
 <p>The T and O cohorts can be found here:</p>
 <ul>
-<li>Ace inhibitors (T): <a href="http://www.ohdsi.org/web/atlas/#/cohortdefinition/1770617" class="uri">http://www.ohdsi.org/web/atlas/#/cohortdefinition/1770617</a>
+<li>Ace inhibitors (T): <a href="http://www.ohdsi.org/web/atlas/#/cohortdefinition/1770617" class="external-link uri">http://www.ohdsi.org/web/atlas/#/cohortdefinition/1770617</a>
 </li>
-<li>Angioedema (O) : <a href="http://www.ohdsi.org/web/atlas/#/cohortdefinition/1770616" class="uri">http://www.ohdsi.org/web/atlas/#/cohortdefinition/1770616</a>
+<li>Angioedema (O) : <a href="http://www.ohdsi.org/web/atlas/#/cohortdefinition/1770616" class="external-link uri">http://www.ohdsi.org/web/atlas/#/cohortdefinition/1770616</a>
 </li>
 </ul>
-<p>In depth explanation of cohort creation in ATLAS is out of scope of this vignette but can be found on the OHDSI wiki pages <a href="http://www.ohdsi.org/web/wiki/doku.php?id=documentation:software:atlas">(link)</a>.</p>
-<p>Note that when a cohort is created in ATLAS the cohortid is needed to extract the data in R. The cohortid can be found at the top of the ATLAS screen, e.g. 1770617 in Figure 6.</p>
-</div>
-<div id="custom-cohorts-1" class="section level3">
-<h3 class="hasAnchor">
-<a href="#custom-cohorts-1" class="anchor"></a>Custom cohorts</h3>
-<p>It is also possible to create cohorts without the use of ATLAS. Using custom cohort code (SQL) you can make more advanced cohorts if needed.</p>
-<p>For our example study, we need to create at table to hold the cohort data and we need to create SQL code to instantiate this table for both the AF and Stroke cohorts. Therefore, we create a file called <em>AceAngioCohorts.sql</em> with the following contents:</p>
-<div class="sourceCode" id="cb20"><pre class="sourceCode sql"><code class="sourceCode sql"><span id="cb20-1"><a href="#cb20-1" aria-hidden="true" tabindex="-1"></a>  <span class="co">/***********************************</span></span>
-<span id="cb20-2"><a href="#cb20-2" aria-hidden="true" tabindex="-1"></a><span class="co">    File AceAngioCohorts.sql </span></span>
-<span id="cb20-3"><a href="#cb20-3" aria-hidden="true" tabindex="-1"></a><span class="co">  ***********************************/</span></span>
-<span id="cb20-4"><a href="#cb20-4" aria-hidden="true" tabindex="-1"></a>    <span class="co">/*</span></span>
-<span id="cb20-5"><a href="#cb20-5" aria-hidden="true" tabindex="-1"></a><span class="co">    Create a table to store the persons in the T and C cohort</span></span>
-<span id="cb20-6"><a href="#cb20-6" aria-hidden="true" tabindex="-1"></a><span class="co">  */</span></span>
-<span id="cb20-7"><a href="#cb20-7" aria-hidden="true" tabindex="-1"></a>    </span>
-<span id="cb20-8"><a href="#cb20-8" aria-hidden="true" tabindex="-1"></a>    <span class="cf">IF</span> OBJECT_ID(<span class="st">'@resultsDatabaseSchema.PLPAceAngioCohort'</span>, <span class="st">'U'</span>) <span class="kw">IS</span> <span class="kw">NOT</span> <span class="kw">NULL</span> </span>
-<span id="cb20-9"><a href="#cb20-9" aria-hidden="true" tabindex="-1"></a>  <span class="kw">DROP</span> <span class="kw">TABLE</span> @resultsDatabaseSchema.PLPAceAngioCohort;</span>
-<span id="cb20-10"><a href="#cb20-10" aria-hidden="true" tabindex="-1"></a>  </span>
-<span id="cb20-11"><a href="#cb20-11" aria-hidden="true" tabindex="-1"></a>  <span class="kw">CREATE</span> <span class="kw">TABLE</span> @resultsDatabaseSchema.PLPAceAngioCohort </span>
-<span id="cb20-12"><a href="#cb20-12" aria-hidden="true" tabindex="-1"></a>  ( </span>
-<span id="cb20-13"><a href="#cb20-13" aria-hidden="true" tabindex="-1"></a>    cohort_definition_id <span class="dt">INT</span>, </span>
-<span id="cb20-14"><a href="#cb20-14" aria-hidden="true" tabindex="-1"></a>    subject_id BIGINT,</span>
-<span id="cb20-15"><a href="#cb20-15" aria-hidden="true" tabindex="-1"></a>    cohort_start_date <span class="dt">DATE</span>, </span>
-<span id="cb20-16"><a href="#cb20-16" aria-hidden="true" tabindex="-1"></a>    cohort_end_date <span class="dt">DATE</span></span>
-<span id="cb20-17"><a href="#cb20-17" aria-hidden="true" tabindex="-1"></a>  );</span>
-<span id="cb20-18"><a href="#cb20-18" aria-hidden="true" tabindex="-1"></a>  </span>
-<span id="cb20-19"><a href="#cb20-19" aria-hidden="true" tabindex="-1"></a>  </span>
-<span id="cb20-20"><a href="#cb20-20" aria-hidden="true" tabindex="-1"></a>  <span class="co">/*</span></span>
-<span id="cb20-21"><a href="#cb20-21" aria-hidden="true" tabindex="-1"></a><span class="co">    T cohort:  [PatientLevelPrediction vignette]:  T : patients who are newly </span></span>
-<span id="cb20-22"><a href="#cb20-22" aria-hidden="true" tabindex="-1"></a><span class="co">  dispensed an ACE inhibitor</span></span>
-<span id="cb20-23"><a href="#cb20-23" aria-hidden="true" tabindex="-1"></a><span class="co">  - persons with a drug exposure record of any 'ACE inhibitor' or </span></span>
-<span id="cb20-24"><a href="#cb20-24" aria-hidden="true" tabindex="-1"></a><span class="co">  any descendants, indexed at the first diagnosis</span></span>
-<span id="cb20-25"><a href="#cb20-25" aria-hidden="true" tabindex="-1"></a><span class="co">  - who have &gt;364 days of prior observation before their first dispensing</span></span>
-<span id="cb20-26"><a href="#cb20-26" aria-hidden="true" tabindex="-1"></a><span class="co">  */</span></span>
-<span id="cb20-27"><a href="#cb20-27" aria-hidden="true" tabindex="-1"></a>    <span class="kw">INSERT</span> <span class="kw">INTO</span> @resultsDatabaseSchema.AceAngioCohort (cohort_definition_id, </span>
-<span id="cb20-28"><a href="#cb20-28" aria-hidden="true" tabindex="-1"></a>                                                       subject_id, </span>
-<span id="cb20-29"><a href="#cb20-29" aria-hidden="true" tabindex="-1"></a>                                                       cohort_start_date, </span>
-<span id="cb20-30"><a href="#cb20-30" aria-hidden="true" tabindex="-1"></a>                                                       cohort_end_date)</span>
-<span id="cb20-31"><a href="#cb20-31" aria-hidden="true" tabindex="-1"></a>  <span class="kw">SELECT</span> <span class="dv">1</span> <span class="kw">AS</span> cohort_definition_id,</span>
-<span id="cb20-32"><a href="#cb20-32" aria-hidden="true" tabindex="-1"></a>  Ace.person_id <span class="kw">AS</span> subject_id,</span>
-<span id="cb20-33"><a href="#cb20-33" aria-hidden="true" tabindex="-1"></a>  Ace.drug_start_date <span class="kw">AS</span> cohort_start_date,</span>
-<span id="cb20-34"><a href="#cb20-34" aria-hidden="true" tabindex="-1"></a>  observation_period.observation_period_end_date <span class="kw">AS</span> cohort_end_date</span>
-<span id="cb20-35"><a href="#cb20-35" aria-hidden="true" tabindex="-1"></a>  <span class="kw">FROM</span></span>
-<span id="cb20-36"><a href="#cb20-36" aria-hidden="true" tabindex="-1"></a>  (</span>
-<span id="cb20-37"><a href="#cb20-37" aria-hidden="true" tabindex="-1"></a>    <span class="kw">SELECT</span> person_id, <span class="fu">min</span>(drug_exposure_date) <span class="kw">as</span> drug_start_date</span>
-<span id="cb20-38"><a href="#cb20-38" aria-hidden="true" tabindex="-1"></a>    <span class="kw">FROM</span> @cdmDatabaseSchema.drug_exposure</span>
-<span id="cb20-39"><a href="#cb20-39" aria-hidden="true" tabindex="-1"></a>    <span class="kw">WHERE</span> drug_concept_id <span class="kw">IN</span> (<span class="kw">SELECT</span> descendant_concept_id <span class="kw">FROM</span> </span>
-<span id="cb20-40"><a href="#cb20-40" aria-hidden="true" tabindex="-1"></a>                              @cdmDatabaseSchema.concept_ancestor <span class="kw">WHERE</span> ancestor_concept_id <span class="kw">IN</span> </span>
-<span id="cb20-41"><a href="#cb20-41" aria-hidden="true" tabindex="-1"></a>                              (<span class="dv">1342439</span>,<span class="dv">1334456</span>, <span class="dv">1331235</span>, <span class="dv">1373225</span>, <span class="dv">1310756</span>, <span class="dv">1308216</span>, <span class="dv">1363749</span>, <span class="dv">1341927</span>, <span class="dv">1340128</span>, <span class="dv">1335471</span> <span class="co">/*ace inhibitors*/</span>))</span>
-<span id="cb20-42"><a href="#cb20-42" aria-hidden="true" tabindex="-1"></a>    <span class="kw">GROUP</span> <span class="kw">BY</span> person_id</span>
-<span id="cb20-43"><a href="#cb20-43" aria-hidden="true" tabindex="-1"></a>  ) Ace</span>
-<span id="cb20-44"><a href="#cb20-44" aria-hidden="true" tabindex="-1"></a>  <span class="kw">INNER</span> <span class="kw">JOIN</span> @cdmDatabaseSchema.observation_period</span>
-<span id="cb20-45"><a href="#cb20-45" aria-hidden="true" tabindex="-1"></a>  <span class="kw">ON</span> Ace.person_id <span class="op">=</span> observation_period.person_id</span>
-<span id="cb20-46"><a href="#cb20-46" aria-hidden="true" tabindex="-1"></a>  <span class="kw">AND</span> Ace.drug_start_date <span class="op">&gt;=</span> dateadd(dd,<span class="dv">364</span>, </span>
-<span id="cb20-47"><a href="#cb20-47" aria-hidden="true" tabindex="-1"></a>                                     observation_period.observation_period_start_date)</span>
-<span id="cb20-48"><a href="#cb20-48" aria-hidden="true" tabindex="-1"></a>  <span class="kw">AND</span> Ace.drug_start_date <span class="op">&lt;=</span> observation_period.observation_period_end_date</span>
-<span id="cb20-49"><a href="#cb20-49" aria-hidden="true" tabindex="-1"></a>  ;</span>
-<span id="cb20-50"><a href="#cb20-50" aria-hidden="true" tabindex="-1"></a>  </span>
-<span id="cb20-51"><a href="#cb20-51" aria-hidden="true" tabindex="-1"></a>  <span class="co">/*</span></span>
-<span id="cb20-52"><a href="#cb20-52" aria-hidden="true" tabindex="-1"></a><span class="co">    C cohort:  [PatientLevelPrediction vignette]:  O: Angioedema</span></span>
-<span id="cb20-53"><a href="#cb20-53" aria-hidden="true" tabindex="-1"></a><span class="co">  */</span></span>
-<span id="cb20-54"><a href="#cb20-54" aria-hidden="true" tabindex="-1"></a>    <span class="kw">INSERT</span> <span class="kw">INTO</span> @resultsDatabaseSchema.AceAngioCohort (cohort_definition_id, </span>
-<span id="cb20-55"><a href="#cb20-55" aria-hidden="true" tabindex="-1"></a>                                                       subject_id, </span>
-<span id="cb20-56"><a href="#cb20-56" aria-hidden="true" tabindex="-1"></a>                                                       cohort_start_date, </span>
-<span id="cb20-57"><a href="#cb20-57" aria-hidden="true" tabindex="-1"></a>                                                       cohort_end_date)</span>
-<span id="cb20-58"><a href="#cb20-58" aria-hidden="true" tabindex="-1"></a>  <span class="kw">SELECT</span> <span class="dv">2</span> <span class="kw">AS</span> cohort_definition_id,</span>
-<span id="cb20-59"><a href="#cb20-59" aria-hidden="true" tabindex="-1"></a>  angioedema.person_id <span class="kw">AS</span> subject_id,</span>
-<span id="cb20-60"><a href="#cb20-60" aria-hidden="true" tabindex="-1"></a>  angioedema.condition_start_date <span class="kw">AS</span> cohort_start_date,</span>
-<span id="cb20-61"><a href="#cb20-61" aria-hidden="true" tabindex="-1"></a>  angioedema.condition_start_date <span class="kw">AS</span> cohort_end_date</span>
-<span id="cb20-62"><a href="#cb20-62" aria-hidden="true" tabindex="-1"></a>  <span class="kw">FROM</span>  </span>
-<span id="cb20-63"><a href="#cb20-63" aria-hidden="true" tabindex="-1"></a>  (</span>
-<span id="cb20-64"><a href="#cb20-64" aria-hidden="true" tabindex="-1"></a>    <span class="kw">SELECT</span> person_id, condition_start_date</span>
-<span id="cb20-65"><a href="#cb20-65" aria-hidden="true" tabindex="-1"></a>    <span class="kw">FROM</span> @cdmDatabaseSchema.condition_occurrence</span>
-<span id="cb20-66"><a href="#cb20-66" aria-hidden="true" tabindex="-1"></a>    <span class="kw">WHERE</span> condition_concept_id <span class="kw">IN</span> (<span class="kw">SELECT</span> <span class="kw">DISTINCT</span> descendant_concept_id <span class="kw">FROM</span> </span>
-<span id="cb20-67"><a href="#cb20-67" aria-hidden="true" tabindex="-1"></a>                                   @cdmDatabaseSchema.concept_ancestor <span class="kw">WHERE</span> ancestor_concept_id <span class="kw">IN</span> </span>
-<span id="cb20-68"><a href="#cb20-68" aria-hidden="true" tabindex="-1"></a>                                   (<span class="dv">432791</span> <span class="co">/*angioedema*/</span>) <span class="kw">OR</span> descendant_concept_id <span class="kw">IN</span> </span>
-<span id="cb20-69"><a href="#cb20-69" aria-hidden="true" tabindex="-1"></a>                                   (<span class="dv">432791</span> <span class="co">/*angioedema*/</span>)</span>
-<span id="cb20-70"><a href="#cb20-70" aria-hidden="true" tabindex="-1"></a>    ) angioedema</span>
-<span id="cb20-71"><a href="#cb20-71" aria-hidden="true" tabindex="-1"></a>    </span>
-<span id="cb20-72"><a href="#cb20-72" aria-hidden="true" tabindex="-1"></a>    ;</span>
-<span id="cb20-73"><a href="#cb20-73" aria-hidden="true" tabindex="-1"></a>    </span></code></pre></div>
-<p>This is parameterized SQL which can be used by the <a href="http://github.com/OHDSI/SqlRender"><code>SqlRender</code></a> package. We use parameterized SQL so we do not have to pre-specify the names of the CDM and result schemas. That way, if we want to run the SQL on a different schema, we only need to change the parameter values; we do not have to change the SQL code. By also making use of translation functionality in <code>SqlRender</code>, we can make sure the SQL code can be run in many different environments.</p>
-<p>To execute this sql against our CDM we first need to tell R how to connect to the server. <code>PatientLevelPrediction</code> uses the <a href="http://github.com/ohdsi/DatabaseConnector"><code>DatabaseConnector</code></a> package, which provides a function called <code>createConnectionDetails</code>. Type <code>?createConnectionDetails</code> for the specific settings required for the various database management systems (DBMS). For example, one might connect to a PostgreSQL database using this code:</p>
+<p>In depth explanation of cohort creation in ATLAS is out of scope of
+this vignette but can be found on the OHDSI wiki pages <a href="http://www.ohdsi.org/web/wiki/doku.php?id=documentation:software:atlas" class="external-link">(link)</a>.</p>
+<p>Note that when a cohort is created in ATLAS the cohortid is needed to
+extract the data in R. The cohortid can be found at the top of the ATLAS
+screen, e.g. 1770617 in Figure 6.</p>
+</div>
+<div class="section level4">
+<h4 id="custom-cohorts-1">Custom cohorts<a class="anchor" aria-label="anchor" href="#custom-cohorts-1"></a>
+</h4>
+<p>It is also possible to create cohorts without the use of ATLAS. Using
+custom cohort code (SQL) you can make more advanced cohorts if
+needed.</p>
+<p>For our example study, we need to create at table to hold the cohort
+data and we need to create SQL code to instantiate this table for both
+the AF and Stroke cohorts. Therefore, we create a file called
+<em>AceAngioCohorts.sql</em> with the following contents:</p>
+<div class="sourceCode" id="cb20"><pre class="sourceCode sql"><code class="sourceCode sql"><span id="cb20-1"><a href="#cb20-1" tabindex="-1"></a>  <span class="co">/***********************************</span></span>
+<span id="cb20-2"><a href="#cb20-2" tabindex="-1"></a><span class="co">    File AceAngioCohorts.sql </span></span>
+<span id="cb20-3"><a href="#cb20-3" tabindex="-1"></a><span class="co">  ***********************************/</span></span>
+<span id="cb20-4"><a href="#cb20-4" tabindex="-1"></a>    <span class="co">/*</span></span>
+<span id="cb20-5"><a href="#cb20-5" tabindex="-1"></a><span class="co">    Create a table to store the persons in the T and C cohort</span></span>
+<span id="cb20-6"><a href="#cb20-6" tabindex="-1"></a><span class="co">  */</span></span>
+<span id="cb20-7"><a href="#cb20-7" tabindex="-1"></a>    </span>
+<span id="cb20-8"><a href="#cb20-8" tabindex="-1"></a>    <span class="cf">IF</span> OBJECT_ID(<span class="st">'@resultsDatabaseSchema.PLPAceAngioCohort'</span>, <span class="st">'U'</span>) <span class="kw">IS</span> <span class="kw">NOT</span> <span class="kw">NULL</span> </span>
+<span id="cb20-9"><a href="#cb20-9" tabindex="-1"></a>  <span class="kw">DROP</span> <span class="kw">TABLE</span> @resultsDatabaseSchema.PLPAceAngioCohort;</span>
+<span id="cb20-10"><a href="#cb20-10" tabindex="-1"></a>  </span>
+<span id="cb20-11"><a href="#cb20-11" tabindex="-1"></a>  <span class="kw">CREATE</span> <span class="kw">TABLE</span> @resultsDatabaseSchema.PLPAceAngioCohort </span>
+<span id="cb20-12"><a href="#cb20-12" tabindex="-1"></a>  ( </span>
+<span id="cb20-13"><a href="#cb20-13" tabindex="-1"></a>    cohort_definition_id <span class="dt">INT</span>, </span>
+<span id="cb20-14"><a href="#cb20-14" tabindex="-1"></a>    subject_id BIGINT,</span>
+<span id="cb20-15"><a href="#cb20-15" tabindex="-1"></a>    cohort_start_date <span class="dt">DATE</span>, </span>
+<span id="cb20-16"><a href="#cb20-16" tabindex="-1"></a>    cohort_end_date <span class="dt">DATE</span></span>
+<span id="cb20-17"><a href="#cb20-17" tabindex="-1"></a>  );</span>
+<span id="cb20-18"><a href="#cb20-18" tabindex="-1"></a>  </span>
+<span id="cb20-19"><a href="#cb20-19" tabindex="-1"></a>  </span>
+<span id="cb20-20"><a href="#cb20-20" tabindex="-1"></a>  <span class="co">/*</span></span>
+<span id="cb20-21"><a href="#cb20-21" tabindex="-1"></a><span class="co">    T cohort:  [PatientLevelPrediction vignette]:  T : patients who are newly </span></span>
+<span id="cb20-22"><a href="#cb20-22" tabindex="-1"></a><span class="co">  dispensed an ACE inhibitor</span></span>
+<span id="cb20-23"><a href="#cb20-23" tabindex="-1"></a><span class="co">  - persons with a drug exposure record of any 'ACE inhibitor' or </span></span>
+<span id="cb20-24"><a href="#cb20-24" tabindex="-1"></a><span class="co">  any descendants, indexed at the first diagnosis</span></span>
+<span id="cb20-25"><a href="#cb20-25" tabindex="-1"></a><span class="co">  - who have &gt;364 days of prior observation before their first dispensing</span></span>
+<span id="cb20-26"><a href="#cb20-26" tabindex="-1"></a><span class="co">  */</span></span>
+<span id="cb20-27"><a href="#cb20-27" tabindex="-1"></a>    <span class="kw">INSERT</span> <span class="kw">INTO</span> @resultsDatabaseSchema.AceAngioCohort (cohort_definition_id, </span>
+<span id="cb20-28"><a href="#cb20-28" tabindex="-1"></a>                                                       subject_id, </span>
+<span id="cb20-29"><a href="#cb20-29" tabindex="-1"></a>                                                       cohort_start_date, </span>
+<span id="cb20-30"><a href="#cb20-30" tabindex="-1"></a>                                                       cohort_end_date)</span>
+<span id="cb20-31"><a href="#cb20-31" tabindex="-1"></a>  <span class="kw">SELECT</span> <span class="dv">1</span> <span class="kw">AS</span> cohort_definition_id,</span>
+<span id="cb20-32"><a href="#cb20-32" tabindex="-1"></a>  Ace.person_id <span class="kw">AS</span> subject_id,</span>
+<span id="cb20-33"><a href="#cb20-33" tabindex="-1"></a>  Ace.drug_start_date <span class="kw">AS</span> cohort_start_date,</span>
+<span id="cb20-34"><a href="#cb20-34" tabindex="-1"></a>  observation_period.observation_period_end_date <span class="kw">AS</span> cohort_end_date</span>
+<span id="cb20-35"><a href="#cb20-35" tabindex="-1"></a>  <span class="kw">FROM</span></span>
+<span id="cb20-36"><a href="#cb20-36" tabindex="-1"></a>  (</span>
+<span id="cb20-37"><a href="#cb20-37" tabindex="-1"></a>    <span class="kw">SELECT</span> person_id, <span class="fu">min</span>(drug_exposure_date) <span class="kw">as</span> drug_start_date</span>
+<span id="cb20-38"><a href="#cb20-38" tabindex="-1"></a>    <span class="kw">FROM</span> @cdmDatabaseSchema.drug_exposure</span>
+<span id="cb20-39"><a href="#cb20-39" tabindex="-1"></a>    <span class="kw">WHERE</span> drug_concept_id <span class="kw">IN</span> (<span class="kw">SELECT</span> descendant_concept_id <span class="kw">FROM</span> </span>
+<span id="cb20-40"><a href="#cb20-40" tabindex="-1"></a>                              @cdmDatabaseSchema.concept_ancestor <span class="kw">WHERE</span> ancestor_concept_id <span class="kw">IN</span> </span>
+<span id="cb20-41"><a href="#cb20-41" tabindex="-1"></a>                              (<span class="dv">1342439</span>,<span class="dv">1334456</span>, <span class="dv">1331235</span>, <span class="dv">1373225</span>, <span class="dv">1310756</span>, <span class="dv">1308216</span>, <span class="dv">1363749</span>, <span class="dv">1341927</span>, <span class="dv">1340128</span>, <span class="dv">1335471</span> <span class="co">/*ace inhibitors*/</span>))</span>
+<span id="cb20-42"><a href="#cb20-42" tabindex="-1"></a>    <span class="kw">GROUP</span> <span class="kw">BY</span> person_id</span>
+<span id="cb20-43"><a href="#cb20-43" tabindex="-1"></a>  ) Ace</span>
+<span id="cb20-44"><a href="#cb20-44" tabindex="-1"></a>  <span class="kw">INNER</span> <span class="kw">JOIN</span> @cdmDatabaseSchema.observation_period</span>
+<span id="cb20-45"><a href="#cb20-45" tabindex="-1"></a>  <span class="kw">ON</span> Ace.person_id <span class="op">=</span> observation_period.person_id</span>
+<span id="cb20-46"><a href="#cb20-46" tabindex="-1"></a>  <span class="kw">AND</span> Ace.drug_start_date <span class="op">&gt;=</span> dateadd(dd,<span class="dv">364</span>, </span>
+<span id="cb20-47"><a href="#cb20-47" tabindex="-1"></a>                                     observation_period.observation_period_start_date)</span>
+<span id="cb20-48"><a href="#cb20-48" tabindex="-1"></a>  <span class="kw">AND</span> Ace.drug_start_date <span class="op">&lt;=</span> observation_period.observation_period_end_date</span>
+<span id="cb20-49"><a href="#cb20-49" tabindex="-1"></a>  ;</span>
+<span id="cb20-50"><a href="#cb20-50" tabindex="-1"></a>  </span>
+<span id="cb20-51"><a href="#cb20-51" tabindex="-1"></a>  <span class="co">/*</span></span>
+<span id="cb20-52"><a href="#cb20-52" tabindex="-1"></a><span class="co">    C cohort:  [PatientLevelPrediction vignette]:  O: Angioedema</span></span>
+<span id="cb20-53"><a href="#cb20-53" tabindex="-1"></a><span class="co">  */</span></span>
+<span id="cb20-54"><a href="#cb20-54" tabindex="-1"></a>    <span class="kw">INSERT</span> <span class="kw">INTO</span> @resultsDatabaseSchema.AceAngioCohort (cohort_definition_id, </span>
+<span id="cb20-55"><a href="#cb20-55" tabindex="-1"></a>                                                       subject_id, </span>
+<span id="cb20-56"><a href="#cb20-56" tabindex="-1"></a>                                                       cohort_start_date, </span>
+<span id="cb20-57"><a href="#cb20-57" tabindex="-1"></a>                                                       cohort_end_date)</span>
+<span id="cb20-58"><a href="#cb20-58" tabindex="-1"></a>  <span class="kw">SELECT</span> <span class="dv">2</span> <span class="kw">AS</span> cohort_definition_id,</span>
+<span id="cb20-59"><a href="#cb20-59" tabindex="-1"></a>  angioedema.person_id <span class="kw">AS</span> subject_id,</span>
+<span id="cb20-60"><a href="#cb20-60" tabindex="-1"></a>  angioedema.condition_start_date <span class="kw">AS</span> cohort_start_date,</span>
+<span id="cb20-61"><a href="#cb20-61" tabindex="-1"></a>  angioedema.condition_start_date <span class="kw">AS</span> cohort_end_date</span>
+<span id="cb20-62"><a href="#cb20-62" tabindex="-1"></a>  <span class="kw">FROM</span>  </span>
+<span id="cb20-63"><a href="#cb20-63" tabindex="-1"></a>  (</span>
+<span id="cb20-64"><a href="#cb20-64" tabindex="-1"></a>    <span class="kw">SELECT</span> person_id, condition_start_date</span>
+<span id="cb20-65"><a href="#cb20-65" tabindex="-1"></a>    <span class="kw">FROM</span> @cdmDatabaseSchema.condition_occurrence</span>
+<span id="cb20-66"><a href="#cb20-66" tabindex="-1"></a>    <span class="kw">WHERE</span> condition_concept_id <span class="kw">IN</span> (<span class="kw">SELECT</span> <span class="kw">DISTINCT</span> descendant_concept_id <span class="kw">FROM</span> </span>
+<span id="cb20-67"><a href="#cb20-67" tabindex="-1"></a>                                   @cdmDatabaseSchema.concept_ancestor <span class="kw">WHERE</span> ancestor_concept_id <span class="kw">IN</span> </span>
+<span id="cb20-68"><a href="#cb20-68" tabindex="-1"></a>                                   (<span class="dv">432791</span> <span class="co">/*angioedema*/</span>) <span class="kw">OR</span> descendant_concept_id <span class="kw">IN</span> </span>
+<span id="cb20-69"><a href="#cb20-69" tabindex="-1"></a>                                   (<span class="dv">432791</span> <span class="co">/*angioedema*/</span>)</span>
+<span id="cb20-70"><a href="#cb20-70" tabindex="-1"></a>    ) angioedema</span>
+<span id="cb20-71"><a href="#cb20-71" tabindex="-1"></a>    </span>
+<span id="cb20-72"><a href="#cb20-72" tabindex="-1"></a>    ;</span>
+<span id="cb20-73"><a href="#cb20-73" tabindex="-1"></a>    </span></code></pre></div>
+<p>This is parameterized SQL which can be used by the <a href="http://github.com/OHDSI/SqlRender" class="external-link"><code>SqlRender</code></a>
+package. We use parameterized SQL so we do not have to pre-specify the
+names of the CDM and result schemas. That way, if we want to run the SQL
+on a different schema, we only need to change the parameter values; we
+do not have to change the SQL code. By also making use of translation
+functionality in <code>SqlRender</code>, we can make sure the SQL code
+can be run in many different environments.</p>
+<p>To execute this sql against our CDM we first need to tell R how to
+connect to the server. <code>PatientLevelPrediction</code> uses the <a href="http://github.com/ohdsi/DatabaseConnector" class="external-link"><code>DatabaseConnector</code></a>
+package, which provides a function called
+<code>createConnectionDetails</code>. Type
+<code>?createConnectionDetails</code> for the specific settings required
+for the various database management systems (DBMS). For example, one
+might connect to a PostgreSQL database using this code:</p>
 <div class="sourceCode" id="cb21"><pre class="downlit sourceCode r">
-<code class="sourceCode R">    <span class="va">connectionDetails</span> <span class="op">&lt;-</span> <span class="fu">createConnectionDetails</span><span class="op">(</span>dbms <span class="op">=</span> <span class="st">"postgresql"</span>, 
-                                                 server <span class="op">=</span> <span class="st">"localhost/ohdsi"</span>, 
-                                                 user <span class="op">=</span> <span class="st">"joe"</span>, 
-                                                 password <span class="op">=</span> <span class="st">"supersecret"</span><span class="op">)</span>
-    
-    <span class="va">cdmDatabaseSchema</span> <span class="op">&lt;-</span> <span class="st">"my_cdm_data"</span>
-    <span class="va">cohortsDatabaseSchema</span> <span class="op">&lt;-</span> <span class="st">"my_results"</span>
-    <span class="va">cdmVersion</span> <span class="op">&lt;-</span> <span class="st">"5"</span></code></pre></div>
-<p>The last three lines define the <code>cdmDatabaseSchema</code> and <code>cohortsDatabaseSchema</code> variables, as well as the CDM version. We will use these later to tell R where the data in CDM format live, where we want to create the cohorts of interest, and what version CDM is used. Note that for Microsoft SQL Server, databaseschemas need to specify both the database and the schema, so for example <code>cdmDatabaseSchema &lt;- "my_cdm_data.dbo"</code>.</p>
+<code class="sourceCode R"><span>    <span class="va">connectionDetails</span> <span class="op">&lt;-</span> <span class="fu">createConnectionDetails</span><span class="op">(</span>dbms <span class="op">=</span> <span class="st">"postgresql"</span>, </span>
+<span>                                                 server <span class="op">=</span> <span class="st">"localhost/ohdsi"</span>, </span>
+<span>                                                 user <span class="op">=</span> <span class="st">"joe"</span>, </span>
+<span>                                                 password <span class="op">=</span> <span class="st">"supersecret"</span><span class="op">)</span></span>
+<span>    </span>
+<span>    <span class="va">cdmDatabaseSchema</span> <span class="op">&lt;-</span> <span class="st">"my_cdm_data"</span></span>
+<span>    <span class="va">cohortsDatabaseSchema</span> <span class="op">&lt;-</span> <span class="st">"my_results"</span></span>
+<span>    <span class="va">cdmVersion</span> <span class="op">&lt;-</span> <span class="st">"5"</span></span></code></pre></div>
+<p>The last three lines define the <code>cdmDatabaseSchema</code> and
+<code>cohortsDatabaseSchema</code> variables, as well as the CDM
+version. We will use these later to tell R where the data in CDM format
+live, where we want to create the cohorts of interest, and what version
+CDM is used. Note that for Microsoft SQL Server, databaseschemas need to
+specify both the database and the schema, so for example
+<code>cdmDatabaseSchema &lt;- "my_cdm_data.dbo"</code>.</p>
 <div class="sourceCode" id="cb22"><pre class="downlit sourceCode r">
-<code class="sourceCode R">    <span class="kw"><a href="https://rdrr.io/r/base/library.html">library</a></span><span class="op">(</span><span class="va"><a href="https://ohdsi.github.io/SqlRender/">SqlRender</a></span><span class="op">)</span>
-    <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/readSql.html">readSql</a></span><span class="op">(</span><span class="st">"AceAngioCohorts.sql"</span><span class="op">)</span>
-    <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/render.html">render</a></span><span class="op">(</span><span class="va">sql</span>,
-                  cdmDatabaseSchema <span class="op">=</span> <span class="va">cdmDatabaseSchema</span>,
-                  cohortsDatabaseSchema <span class="op">=</span> <span class="va">cohortsDatabaseSchema</span><span class="op">)</span>
-    <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/translate.html">translate</a></span><span class="op">(</span><span class="va">sql</span>, targetDialect <span class="op">=</span> <span class="va">connectionDetails</span><span class="op">$</span><span class="va">dbms</span><span class="op">)</span>
-    
-    <span class="va">connection</span> <span class="op">&lt;-</span> <span class="fu">connect</span><span class="op">(</span><span class="va">connectionDetails</span><span class="op">)</span>
-    <span class="fu">executeSql</span><span class="op">(</span><span class="va">connection</span>, <span class="va">sql</span><span class="op">)</span></code></pre></div>
-<p>In this code, we first read the SQL from the file into memory. In the next line, we replace four parameter names with the actual values. We then translate the SQL into the dialect appropriate for the DBMS we already specified in the <code>connectionDetails</code>. Next, we connect to the server, and submit the rendered and translated SQL.</p>
-<p>If all went well, we now have a table with the events of interest. We can see how many events per type:</p>
+<code class="sourceCode R"><span>    <span class="kw"><a href="https://rdrr.io/r/base/library.html" class="external-link">library</a></span><span class="op">(</span><span class="va"><a href="https://ohdsi.github.io/SqlRender/" class="external-link">SqlRender</a></span><span class="op">)</span></span>
+<span>    <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/readSql.html" class="external-link">readSql</a></span><span class="op">(</span><span class="st">"AceAngioCohorts.sql"</span><span class="op">)</span></span>
+<span>    <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/render.html" class="external-link">render</a></span><span class="op">(</span><span class="va">sql</span>,</span>
+<span>                  cdmDatabaseSchema <span class="op">=</span> <span class="va">cdmDatabaseSchema</span>,</span>
+<span>                  cohortsDatabaseSchema <span class="op">=</span> <span class="va">cohortsDatabaseSchema</span><span class="op">)</span></span>
+<span>    <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/translate.html" class="external-link">translate</a></span><span class="op">(</span><span class="va">sql</span>, targetDialect <span class="op">=</span> <span class="va">connectionDetails</span><span class="op">$</span><span class="va">dbms</span><span class="op">)</span></span>
+<span>    </span>
+<span>    <span class="va">connection</span> <span class="op">&lt;-</span> <span class="fu">connect</span><span class="op">(</span><span class="va">connectionDetails</span><span class="op">)</span></span>
+<span>    <span class="fu">executeSql</span><span class="op">(</span><span class="va">connection</span>, <span class="va">sql</span><span class="op">)</span></span></code></pre></div>
+<p>In this code, we first read the SQL from the file into memory. In the
+next line, we replace four parameter names with the actual values. We
+then translate the SQL into the dialect appropriate for the DBMS we
+already specified in the <code>connectionDetails</code>. Next, we
+connect to the server, and submit the rendered and translated SQL.</p>
+<p>If all went well, we now have a table with the events of interest. We
+can see how many events per type:</p>
 <div class="sourceCode" id="cb23"><pre class="downlit sourceCode r">
-<code class="sourceCode R">    <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste</a></span><span class="op">(</span><span class="st">"SELECT cohort_definition_id, COUNT(*) AS count"</span>,
-                 <span class="st">"FROM @cohortsDatabaseSchema.AceAngioCohort"</span>,
-                 <span class="st">"GROUP BY cohort_definition_id"</span><span class="op">)</span>
-    <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/render.html">render</a></span><span class="op">(</span><span class="va">sql</span>, cohortsDatabaseSchema <span class="op">=</span> <span class="va">cohortsDatabaseSchema</span><span class="op">)</span>
-    <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/translate.html">translate</a></span><span class="op">(</span><span class="va">sql</span>, targetDialect <span class="op">=</span> <span class="va">connectionDetails</span><span class="op">$</span><span class="va">dbms</span><span class="op">)</span>
-    
-    <span class="fu">querySql</span><span class="op">(</span><span class="va">connection</span>, <span class="va">sql</span><span class="op">)</span></code></pre></div>
-<pre><code>##   cohort_definition_id count
-## 1                    1     0
-## 2                    2     0</code></pre>
-</div>
-<div id="study-script-creation-1" class="section level3">
-<h3 class="hasAnchor">
-<a href="#study-script-creation-1" class="anchor"></a>Study script creation</h3>
-<p>In this section we assume that our cohorts have been created either by using ATLAS or a custom SQL script. We will first explain how to create an R script yourself that will execute our study as we have defined earlier.</p>
-</div>
-<div id="data-extraction-1" class="section level3">
-<h3 class="hasAnchor">
-<a href="#data-extraction-1" class="anchor"></a>Data extraction</h3>
-<p>Now we can tell <code>PatientLevelPrediction</code> to extract all necessary data for our analysis. This is done using the <a href="https://github.com/OHDSI/FeatureExtration"><code>FeatureExtractionPackage</code></a>. In short the FeatureExtractionPackage allows you to specify which features (covariates) need to be extracted, e.g. all conditions and drug exposures. It also supports the creation of custom covariates. For more detailed information on the FeatureExtraction package see its <a href="https://github.com/OHDSI/FeatureExtration">vignettes</a>. For our example study we decided to use these settings:</p>
+<code class="sourceCode R"><span>    <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html" class="external-link">paste</a></span><span class="op">(</span><span class="st">"SELECT cohort_definition_id, COUNT(*) AS count"</span>,</span>
+<span>                 <span class="st">"FROM @cohortsDatabaseSchema.AceAngioCohort"</span>,</span>
+<span>                 <span class="st">"GROUP BY cohort_definition_id"</span><span class="op">)</span></span>
+<span>    <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/render.html" class="external-link">render</a></span><span class="op">(</span><span class="va">sql</span>, cohortsDatabaseSchema <span class="op">=</span> <span class="va">cohortsDatabaseSchema</span><span class="op">)</span></span>
+<span>    <span class="va">sql</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/SqlRender/reference/translate.html" class="external-link">translate</a></span><span class="op">(</span><span class="va">sql</span>, targetDialect <span class="op">=</span> <span class="va">connectionDetails</span><span class="op">$</span><span class="va">dbms</span><span class="op">)</span></span>
+<span>    </span>
+<span>    <span class="fu">querySql</span><span class="op">(</span><span class="va">connection</span>, <span class="va">sql</span><span class="op">)</span></span></code></pre></div>
+<pre><code><span><span class="co">##   cohort_definition_id count</span></span>
+<span><span class="co">## 1                    1     0</span></span>
+<span><span class="co">## 2                    2     0</span></span></code></pre>
+</div>
+<div class="section level4">
+<h4 id="study-script-creation-1">Study script creation<a class="anchor" aria-label="anchor" href="#study-script-creation-1"></a>
+</h4>
+<p>In this section we assume that our cohorts have been created either
+by using ATLAS or a custom SQL script. We will first explain how to
+create an R script yourself that will execute our study as we have
+defined earlier.</p>
+</div>
+<div class="section level4">
+<h4 id="data-extraction-1">Data extraction<a class="anchor" aria-label="anchor" href="#data-extraction-1"></a>
+</h4>
+<p>Now we can tell <code>PatientLevelPrediction</code> to extract all
+necessary data for our analysis. This is done using the <a href="https://github.com/OHDSI/FeatureExtraction" class="external-link"><code>FeatureExtractionPackage</code></a>.
+In short the FeatureExtractionPackage allows you to specify which
+features (covariates) need to be extracted, e.g. all conditions and drug
+exposures. It also supports the creation of custom covariates. For more
+detailed information on the FeatureExtraction package see its <a href="https://github.com/OHDSI/FeatureExtraction" class="external-link">vignettes</a>. For our
+example study we decided to use these settings:</p>
 <div class="sourceCode" id="cb25"><pre class="downlit sourceCode r">
-<code class="sourceCode R">    <span class="va">covariateSettings</span> <span class="op">&lt;-</span> <span class="fu">createCovariateSettings</span><span class="op">(</span>useDemographicsGender <span class="op">=</span> <span class="cn">TRUE</span>,
-                                                 useDemographicsAge <span class="op">=</span> <span class="cn">TRUE</span>,
-                                                 useConditionGroupEraLongTerm <span class="op">=</span> <span class="cn">TRUE</span>,
-                                                 useConditionGroupEraAnyTimePrior <span class="op">=</span> <span class="cn">TRUE</span>,
-                                                 useDrugGroupEraLongTerm <span class="op">=</span> <span class="cn">TRUE</span>,
-                                                 useDrugGroupEraAnyTimePrior <span class="op">=</span> <span class="cn">TRUE</span>,
-                                                 useVisitConceptCountLongTerm <span class="op">=</span> <span class="cn">TRUE</span>,
-                                                 longTermStartDays <span class="op">=</span> <span class="op">-</span><span class="fl">365</span>,
-                                                 endDays <span class="op">=</span> <span class="op">-</span><span class="fl">1</span><span class="op">)</span></code></pre></div>
-<p>The final step for extracting the data is to run the <code>getPlpData</code> function and input the connection details, the database schema where the cohorts are stored, the cohort definition ids for the cohort and outcome, and the washoutPeriod which is the minimum number of days prior to cohort index date that the person must have been observed to be included into the data, and finally input the previously constructed covariate settings.</p>
+<code class="sourceCode R"><span>    <span class="va">covariateSettings</span> <span class="op">&lt;-</span> <span class="fu">createCovariateSettings</span><span class="op">(</span>useDemographicsGender <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>                                                 useDemographicsAge <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>                                                 useConditionGroupEraLongTerm <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>                                                 useConditionGroupEraAnyTimePrior <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>                                                 useDrugGroupEraLongTerm <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>                                                 useDrugGroupEraAnyTimePrior <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>                                                 useVisitConceptCountLongTerm <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>                                                 longTermStartDays <span class="op">=</span> <span class="op">-</span><span class="fl">365</span>,</span>
+<span>                                                 endDays <span class="op">=</span> <span class="op">-</span><span class="fl">1</span><span class="op">)</span></span></code></pre></div>
+<p>The final step for extracting the data is to run the
+<code>getPlpData</code> function and input the connection details, the
+database schema where the cohorts are stored, the cohort definition ids
+for the cohort and outcome, and the washoutPeriod which is the minimum
+number of days prior to cohort index date that the person must have been
+observed to be included into the data, and finally input the previously
+constructed covariate settings.</p>
 <div class="sourceCode" id="cb26"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">databaseDetails</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDatabaseDetails.html">createDatabaseDetails</a></span><span class="op">(</span>
-  connectionDetails <span class="op">=</span> <span class="va">connectionDetails</span>,
-  cdmDatabaseSchema <span class="op">=</span> <span class="va">cdmDatabaseSchema</span>,
-  cohortDatabaseSchema <span class="op">=</span> <span class="va">resultsDatabaseSchema</span>,
-  cohortTable <span class="op">=</span> <span class="st">'AceAngioCohort'</span>,
-  cohortId <span class="op">=</span> <span class="fl">1</span>,
-  outcomeDatabaseSchema <span class="op">=</span> <span class="va">resultsDatabaseSchema</span>,
-  outcomeTable <span class="op">=</span> <span class="st">'AceAngioCohort'</span>,
-  outcomeIds <span class="op">=</span> <span class="fl">2</span>
-  <span class="op">)</span>
-
-<span class="va">restrictPlpDataSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createRestrictPlpDataSettings.html">createRestrictPlpDataSettings</a></span><span class="op">(</span>
-  sampleSize <span class="op">=</span> <span class="fl">10000</span>
-  <span class="op">)</span>
-
-<span class="va">plpData</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/getPlpData.html">getPlpData</a></span><span class="op">(</span>
-  databaseDetails <span class="op">=</span> <span class="va">databaseDetails</span>, 
-  covariateSettings <span class="op">=</span> <span class="va">covariateSettings</span>, 
-  restrictPlpDataSettings <span class="op">=</span> <span class="va">restrictPlpDataSettings</span>
-  <span class="op">)</span></code></pre></div>
-<p>Note that if the cohorts are created in ATLAS its corresponding cohort database schema needs to be selected. There are many additional parameters for the <code>getPlpData</code> function which are all documented in the <code>PatientLevelPrediction</code> manual. The resulting <code>plpData</code> object uses the package <code>ff</code> to store information in a way that ensures R does not run out of memory, even when the data are large.</p>
-<p>Creating the <code>plpData</code> object can take considerable computing time, and it is probably a good idea to save it for future sessions. Because <code>plpData</code> uses <code>ff</code>, we cannot use R’s regular save function. Instead, we’ll have to use the <code><a href="../reference/savePlpData.html">savePlpData()</a></code> function:</p>
+<code class="sourceCode R"><span><span class="va">databaseDetails</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDatabaseDetails.html">createDatabaseDetails</a></span><span class="op">(</span></span>
+<span>  connectionDetails <span class="op">=</span> <span class="va">connectionDetails</span>,</span>
+<span>  cdmDatabaseSchema <span class="op">=</span> <span class="va">cdmDatabaseSchema</span>,</span>
+<span>  cohortDatabaseSchema <span class="op">=</span> <span class="va">resultsDatabaseSchema</span>,</span>
+<span>  cohortTable <span class="op">=</span> <span class="st">'AceAngioCohort'</span>,</span>
+<span>  cohortId <span class="op">=</span> <span class="fl">1</span>,</span>
+<span>  outcomeDatabaseSchema <span class="op">=</span> <span class="va">resultsDatabaseSchema</span>,</span>
+<span>  outcomeTable <span class="op">=</span> <span class="st">'AceAngioCohort'</span>,</span>
+<span>  outcomeIds <span class="op">=</span> <span class="fl">2</span></span>
+<span>  <span class="op">)</span></span>
+<span></span>
+<span><span class="va">restrictPlpDataSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createRestrictPlpDataSettings.html">createRestrictPlpDataSettings</a></span><span class="op">(</span></span>
+<span>  sampleSize <span class="op">=</span> <span class="fl">10000</span></span>
+<span>  <span class="op">)</span></span>
+<span></span>
+<span><span class="va">plpData</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/getPlpData.html">getPlpData</a></span><span class="op">(</span></span>
+<span>  databaseDetails <span class="op">=</span> <span class="va">databaseDetails</span>, </span>
+<span>  covariateSettings <span class="op">=</span> <span class="va">covariateSettings</span>, </span>
+<span>  restrictPlpDataSettings <span class="op">=</span> <span class="va">restrictPlpDataSettings</span></span>
+<span>  <span class="op">)</span></span></code></pre></div>
+<p>Note that if the cohorts are created in ATLAS its corresponding
+cohort database schema needs to be selected. There are many additional
+parameters for the <code>getPlpData</code> function which are all
+documented in the <code>PatientLevelPrediction</code> manual. The
+resulting <code>plpData</code> object uses the package <code>ff</code>
+to store information in a way that ensures R does not run out of memory,
+even when the data are large.</p>
+<p>Creating the <code>plpData</code> object can take considerable
+computing time, and it is probably a good idea to save it for future
+sessions. Because <code>plpData</code> uses <code>ff</code>, we cannot
+use R’s regular save function. Instead, we’ll have to use the
+<code><a href="../reference/savePlpData.html">savePlpData()</a></code> function:</p>
 <div class="sourceCode" id="cb27"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="../reference/savePlpData.html">savePlpData</a></span><span class="op">(</span><span class="va">plpData</span>, <span class="st">"angio_in_ace_data"</span><span class="op">)</span></code></pre></div>
-<p>We can use the <code><a href="../reference/loadPlpData.html">loadPlpData()</a></code> function to load the data in a future session.</p>
-</div>
-<div id="additional-inclusion-criteria-1" class="section level3">
-<h3 class="hasAnchor">
-<a href="#additional-inclusion-criteria-1" class="anchor"></a>Additional inclusion criteria</h3>
-<p>To completely define the prediction problem the final study population is obtained by applying additional constraints on the two earlier defined cohorts, e.g., a minumim time at risk can be enforced (<code>requireTimeAtRisk, minTimeAtRisk</code>) and we can specify if this also applies to patients with the outcome (<code>includeAllOutcomes</code>). Here we also specify the start and end of the risk window relative to target cohort start. For example, if we like the risk window to start 30 days after the at-risk cohort start and end a year later we can set <code>riskWindowStart = 30</code> and <code>riskWindowEnd = 365</code>. In some cases the risk window needs to start at the cohort end date. This can be achieved by setting <code>addExposureToStart = TRUE</code> which adds the cohort (exposure) time to the start date.</p>
-<p>In Appendix 1, we demonstrate the effect of these settings on the subset of the persons in the target cohort that end up in the final study population.</p>
-<p>In the example below all the settings we defined for our study are imposed:</p>
+<code class="sourceCode R"><span><span class="fu"><a href="../reference/savePlpData.html">savePlpData</a></span><span class="op">(</span><span class="va">plpData</span>, <span class="st">"angio_in_ace_data"</span><span class="op">)</span></span></code></pre></div>
+<p>We can use the <code><a href="../reference/loadPlpData.html">loadPlpData()</a></code> function to load the data
+in a future session.</p>
+</div>
+<div class="section level4">
+<h4 id="additional-inclusion-criteria-1">Additional inclusion criteria<a class="anchor" aria-label="anchor" href="#additional-inclusion-criteria-1"></a>
+</h4>
+<p>To completely define the prediction problem the final study
+population is obtained by applying additional constraints on the two
+earlier defined cohorts, e.g., a minumim time at risk can be enforced
+(<code>requireTimeAtRisk, minTimeAtRisk</code>) and we can specify if
+this also applies to patients with the outcome
+(<code>includeAllOutcomes</code>). Here we also specify the start and
+end of the risk window relative to target cohort start. For example, if
+we like the risk window to start 30 days after the at-risk cohort start
+and end a year later we can set <code>riskWindowStart = 30</code> and
+<code>riskWindowEnd = 365</code>. In some cases the risk window needs to
+start at the cohort end date. This can be achieved by setting
+<code>addExposureToStart = TRUE</code> which adds the cohort (exposure)
+time to the start date.</p>
+<p>In Appendix 1, we demonstrate the effect of these settings on the
+subset of the persons in the target cohort that end up in the final
+study population.</p>
+<p>In the example below all the settings we defined for our study are
+imposed:</p>
 <div class="sourceCode" id="cb28"><pre class="downlit sourceCode r">
-<code class="sourceCode R">    <span class="va">populationSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createStudyPopulationSettings.html">createStudyPopulationSettings</a></span><span class="op">(</span>
-      washoutPeriod <span class="op">=</span> <span class="fl">364</span>,
-      firstExposureOnly <span class="op">=</span> <span class="cn">FALSE</span>,
-      removeSubjectsWithPriorOutcome <span class="op">=</span> <span class="cn">TRUE</span>,
-      priorOutcomeLookback <span class="op">=</span> <span class="fl">9999</span>,
-      riskWindowStart <span class="op">=</span> <span class="fl">1</span>,
-      riskWindowEnd <span class="op">=</span> <span class="fl">365</span>, 
-      minTimeAtRisk <span class="op">=</span> <span class="fl">364</span>,
-      startAnchor <span class="op">=</span> <span class="st">'cohort start'</span>,
-      endAnchor <span class="op">=</span> <span class="st">'cohort start'</span>,
-      requireTimeAtRisk <span class="op">=</span> <span class="cn">TRUE</span>,
-      includeAllOutcomes <span class="op">=</span> <span class="cn">TRUE</span>
-    <span class="op">)</span></code></pre></div>
-</div>
-<div id="spliting-the-data-into-trainingvalidationtesting-datasets-1" class="section level3">
-<h3 class="hasAnchor">
-<a href="#spliting-the-data-into-trainingvalidationtesting-datasets-1" class="anchor"></a>Spliting the data into training/validation/testing datasets</h3>
-<p>When developing a prediction model using supervised learning (when you have features paired with labels for a set of patients), the first step is to design the development/internal validation process. This requires specifying how to select the model hyper-parameters, how to learn the model parameters and how to fairly evaluate the model. In general, the validation set is used to pick hyper-parameters, the training set is used to learn the model parameters and the test set is used to perform fair internal validation. However, cross-validation can be implemented to pick the hyper-parameters on the training data (so a validation data set is not required). Cross validation can also be used to estimate internal validation (so a testing data set is not required).</p>
-<p>In small data the best approach for internal validation has been shown to be boostrapping. However, in big data (many patients and many features) bootstrapping is generally not feasible. In big data our research has shown that it is just important to have some form of fair evaluation (use a test set or cross validation). For full details see <a href="add%20link">our BMJ open paper</a>.</p>
-<p>In the PatientLevelPrediction package, the splitSettings define how the plpData are partitioned into training/validation/testing data. Cross validation is always done, but using a test set is optional (when the data are small, it may be optimal to not use a test set). For the splitSettings we can use the type (stratified/time/subject) and testFraction parameters to split the data in a 75%-25% split and run the patient-level prediction pipeline:</p>
+<code class="sourceCode R"><span>    <span class="va">populationSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createStudyPopulationSettings.html">createStudyPopulationSettings</a></span><span class="op">(</span></span>
+<span>      washoutPeriod <span class="op">=</span> <span class="fl">364</span>,</span>
+<span>      firstExposureOnly <span class="op">=</span> <span class="cn">FALSE</span>,</span>
+<span>      removeSubjectsWithPriorOutcome <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>      priorOutcomeLookback <span class="op">=</span> <span class="fl">9999</span>,</span>
+<span>      riskWindowStart <span class="op">=</span> <span class="fl">1</span>,</span>
+<span>      riskWindowEnd <span class="op">=</span> <span class="fl">365</span>, </span>
+<span>      minTimeAtRisk <span class="op">=</span> <span class="fl">364</span>,</span>
+<span>      startAnchor <span class="op">=</span> <span class="st">'cohort start'</span>,</span>
+<span>      endAnchor <span class="op">=</span> <span class="st">'cohort start'</span>,</span>
+<span>      requireTimeAtRisk <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>      includeAllOutcomes <span class="op">=</span> <span class="cn">TRUE</span></span>
+<span>    <span class="op">)</span></span></code></pre></div>
+</div>
+<div class="section level4">
+<h4 id="spliting-the-data-into-trainingvalidationtesting-datasets-1">Spliting the data into training/validation/testing datasets<a class="anchor" aria-label="anchor" href="#spliting-the-data-into-trainingvalidationtesting-datasets-1"></a>
+</h4>
+<p>When developing a prediction model using supervised learning (when
+you have features paired with labels for a set of patients), the first
+step is to design the development/internal validation process. This
+requires specifying how to select the model hyper-parameters, how to
+learn the model parameters and how to fairly evaluate the model. In
+general, the validation set is used to pick hyper-parameters, the
+training set is used to learn the model parameters and the test set is
+used to perform fair internal validation. However, cross-validation can
+be implemented to pick the hyper-parameters on the training data (so a
+validation data set is not required). Cross validation can also be used
+to estimate internal validation (so a testing data set is not
+required).</p>
+<p>In small data the best approach for internal validation has been
+shown to be boostrapping. However, in big data (many patients and many
+features) bootstrapping is generally not feasible. In big data our
+research has shown that it is just important to have some form of fair
+evaluation (use a test set or cross validation). For full details see <a href="add%20link">our BMJ open paper</a>.</p>
+<p>In the PatientLevelPrediction package, the splitSettings define how
+the plpData are partitioned into training/validation/testing data. Cross
+validation is always done, but using a test set is optional (when the
+data are small, it may be optimal to not use a test set). For the
+splitSettings we can use the type (stratified/time/subject) and
+testFraction parameters to split the data in a 75%-25% split and run the
+patient-level prediction pipeline:</p>
 <div class="sourceCode" id="cb29"><pre class="downlit sourceCode r">
-<code class="sourceCode R">  <span class="va">splitSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDefaultSplitSetting.html">createDefaultSplitSetting</a></span><span class="op">(</span>
-    trainFraction <span class="op">=</span> <span class="fl">0.75</span>,
-    testFraction <span class="op">=</span> <span class="fl">0.25</span>,
-    type <span class="op">=</span> <span class="st">'stratified'</span>,
-    nfold <span class="op">=</span> <span class="fl">2</span>, 
-    splitSeed <span class="op">=</span> <span class="fl">1234</span>
-    <span class="op">)</span></code></pre></div>
-<p>Note: it is possible to add a custom method to specify how the plpData are partitioned into training/validation/testing data, see <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSplitting.pdf">vignette for custom splitting</a>.</p>
-</div>
-<div id="preprocessing-the-training-data-1" class="section level3">
-<h3 class="hasAnchor">
-<a href="#preprocessing-the-training-data-1" class="anchor"></a>Preprocessing the training data</h3>
-<p>There a numerous data processing settings that a user must specify when developing a prediction model. These are: * Whether to under-sample or over-sample the training data (this may be useful when there is class imballance (e.g., the outcome is very rare or very common)) * Whether to perform feature engineering or feature selection (e.g., create latent variables that are not observed in the data or reduce the dimensionality of the data) * Whether to remove redundant features and normalize the data (this is required for some models)</p>
-<p>The default sample settings does nothing, it simply returns the trainData as input, see below:</p>
+<code class="sourceCode R"><span>  <span class="va">splitSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDefaultSplitSetting.html">createDefaultSplitSetting</a></span><span class="op">(</span></span>
+<span>    trainFraction <span class="op">=</span> <span class="fl">0.75</span>,</span>
+<span>    testFraction <span class="op">=</span> <span class="fl">0.25</span>,</span>
+<span>    type <span class="op">=</span> <span class="st">'stratified'</span>,</span>
+<span>    nfold <span class="op">=</span> <span class="fl">2</span>, </span>
+<span>    splitSeed <span class="op">=</span> <span class="fl">1234</span></span>
+<span>    <span class="op">)</span></span></code></pre></div>
+<p>Note: it is possible to add a custom method to specify how the
+plpData are partitioned into training/validation/testing data, see <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSplitting.pdf" class="external-link">vignette
+for custom splitting</a>.</p>
+</div>
+<div class="section level4">
+<h4 id="preprocessing-the-training-data-1">Preprocessing the training data<a class="anchor" aria-label="anchor" href="#preprocessing-the-training-data-1"></a>
+</h4>
+<p>There a numerous data processing settings that a user must specify
+when developing a prediction model. These are: * Whether to under-sample
+or over-sample the training data (this may be useful when there is class
+imballance (e.g., the outcome is very rare or very common)) * Whether to
+perform feature engineering or feature selection (e.g., create latent
+variables that are not observed in the data or reduce the dimensionality
+of the data) * Whether to remove redundant features and normalize the
+data (this is required for some models)</p>
+<p>The default sample settings does nothing, it simply returns the
+trainData as input, see below:</p>
 <div class="sourceCode" id="cb30"><pre class="downlit sourceCode r">
-<code class="sourceCode R">  <span class="va">sampleSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createSampleSettings.html">createSampleSettings</a></span><span class="op">(</span><span class="op">)</span></code></pre></div>
-<p>However, the current package contains methods of under-sampling the non-outcome patients. To perform undersampling, the <code>type</code> input should be ‘underSample’ and <code>numberOutcomestoNonOutcomes</code> must be specified (an integer specifying the number of non-outcomes per outcome). It is possible to add any custom function for over/under sampling, see <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSamples.pdf">vignette for custom sampling</a>.</p>
-<p>It is possible to specify a combination of feature engineering functions that take as input the trainData and output a new trainData with different features. The default feature engineering setting does nothing:</p>
+<code class="sourceCode R"><span>  <span class="va">sampleSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createSampleSettings.html">createSampleSettings</a></span><span class="op">(</span><span class="op">)</span></span></code></pre></div>
+<p>However, the current package contains methods of under-sampling the
+non-outcome patients. To perform undersampling, the <code>type</code>
+input should be ‘underSample’ and
+<code>numberOutcomestoNonOutcomes</code> must be specified (an integer
+specifying the number of non-outcomes per outcome). It is possible to
+add any custom function for over/under sampling, see <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSamples.pdf" class="external-link">vignette
+for custom sampling</a>.</p>
+<p>It is possible to specify a combination of feature engineering
+functions that take as input the trainData and output a new trainData
+with different features. The default feature engineering setting does
+nothing:</p>
 <div class="sourceCode" id="cb31"><pre class="downlit sourceCode r">
-<code class="sourceCode R">  <span class="va">featureEngineeringSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createFeatureEngineeringSettings.html">createFeatureEngineeringSettings</a></span><span class="op">(</span><span class="op">)</span></code></pre></div>
-<p>However, it is possible to add custom feature engineering functions into the pipeline, see <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomfeatureEngineering.pdf">vignette for custom feature engineering</a>.</p>
-<p>Finally, the preprocessing setting is required. For this setting the user can define <code>minFraction</code>, this removes any features that is observed in the training data for less than 0.01 fraction of the patients. So, if <code>minFraction = 0.01</code> then any feature that is seen in less than 1 percent of the target population is removed. The input <code>normalize</code> specifies whether the features are scaled between 0 and 1, this is required for certain models (e.g., LASSO logistic regression). The input <code>removeRedundancy</code> specifies whether features that are observed in all of the target population are removed.</p>
+<code class="sourceCode R"><span>  <span class="va">featureEngineeringSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createFeatureEngineeringSettings.html">createFeatureEngineeringSettings</a></span><span class="op">(</span><span class="op">)</span></span></code></pre></div>
+<p>However, it is possible to add custom feature engineering functions
+into the pipeline, see <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomfeatureEngineering.pdf" class="external-link">vignette
+for custom feature engineering</a>.</p>
+<p>Finally, the preprocessing setting is required. For this setting the
+user can define <code>minFraction</code>, this removes any features that
+is observed in the training data for less than 0.01 fraction of the
+patients. So, if <code>minFraction = 0.01</code> then any feature that
+is seen in less than 1 percent of the target population is removed. The
+input <code>normalize</code> specifies whether the features are scaled
+between 0 and 1, this is required for certain models (e.g., LASSO
+logistic regression). The input <code>removeRedundancy</code> specifies
+whether features that are observed in all of the target population are
+removed.</p>
 <div class="sourceCode" id="cb32"><pre class="downlit sourceCode r">
-<code class="sourceCode R">  <span class="va">preprocessSettingsSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createPreprocessSettings.html">createPreprocessSettings</a></span><span class="op">(</span>
-    minFraction <span class="op">=</span> <span class="fl">0.01</span>, 
-    normalize <span class="op">=</span> <span class="cn">T</span>, 
-    removeRedundancy <span class="op">=</span> <span class="cn">T</span>
-      <span class="op">)</span></code></pre></div>
-</div>
-<div id="model-development-1" class="section level3">
-<h3 class="hasAnchor">
-<a href="#model-development-1" class="anchor"></a>Model Development</h3>
-<p>In the set function of an algorithm the user can specify a list of eligible values for each hyper-parameter. All possible combinations of the hyper-parameters are included in a so-called grid search using cross-validation on the training set. If a user does not specify any value then the default value is used instead.</p>
-<p>For example, if we use the following settings for the gradientBoostingMachine: ntrees=c(100,200), maxDepth=4 the grid search will apply the gradient boosting machine algorithm with ntrees=100 and maxDepth=4 plus the default settings for other hyper-parameters and ntrees=200 and maxDepth=4 plus the default settings for other hyper-parameters. The hyper-parameters that lead to the bestcross-validation performance will then be chosen for the final model. For our problem we choose to build a logistic regression model with the default hyper-parameters</p>
+<code class="sourceCode R"><span>  <span class="va">preprocessSettingsSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createPreprocessSettings.html">createPreprocessSettings</a></span><span class="op">(</span></span>
+<span>    minFraction <span class="op">=</span> <span class="fl">0.01</span>, </span>
+<span>    normalize <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>    removeRedundancy <span class="op">=</span> <span class="cn">T</span></span>
+<span>      <span class="op">)</span></span></code></pre></div>
+</div>
+<div class="section level4">
+<h4 id="model-development-1">Model Development<a class="anchor" aria-label="anchor" href="#model-development-1"></a>
+</h4>
+<p>In the set function of an algorithm the user can specify a list of
+eligible values for each hyper-parameter. All possible combinations of
+the hyper-parameters are included in a so-called grid search using
+cross-validation on the training set. If a user does not specify any
+value then the default value is used instead.</p>
+<p>For example, if we use the following settings for the
+gradientBoostingMachine: ntrees=c(100,200), maxDepth=4 the grid search
+will apply the gradient boosting machine algorithm with ntrees=100 and
+maxDepth=4 plus the default settings for other hyper-parameters and
+ntrees=200 and maxDepth=4 plus the default settings for other
+hyper-parameters. The hyper-parameters that lead to the
+bestcross-validation performance will then be chosen for the final
+model. For our problem we choose to build a logistic regression model
+with the default hyper-parameters</p>
 <div class="sourceCode" id="cb33"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">gbmModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/setGradientBoostingMachine.html">setGradientBoostingMachine</a></span><span class="op">(</span>ntrees <span class="op">=</span> <span class="fl">5000</span>, maxDepth <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="fl">4</span>, <span class="fl">7</span>, <span class="fl">10</span><span class="op">)</span>, learnRate <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="fl">0.001</span>, 
-    <span class="fl">0.01</span>, <span class="fl">0.1</span>, <span class="fl">0.9</span><span class="op">)</span><span class="op">)</span></code></pre></div>
-<p>The <code>runPlP</code> function requires the <code>plpData</code>, the <code>outcomeId</code> specifying the outcome being predicted and the settings: <code>populationSettings</code>, <code>splitSettings</code>, <code>sampleSettings</code>, <code>featureEngineeringSettings</code>, <code>preprocessSettings</code> and <code>modelSettings</code> to train and evaluate the model.</p>
+<code class="sourceCode R"><span><span class="va">gbmModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/setGradientBoostingMachine.html">setGradientBoostingMachine</a></span><span class="op">(</span>ntrees <span class="op">=</span> <span class="fl">5000</span>, maxDepth <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">4</span>, <span class="fl">7</span>, <span class="fl">10</span><span class="op">)</span>, learnRate <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">0.001</span>,</span>
+<span>    <span class="fl">0.01</span>, <span class="fl">0.1</span>, <span class="fl">0.9</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
+<p>The <code>runPlP</code> function requires the <code>plpData</code>,
+the <code>outcomeId</code> specifying the outcome being predicted and
+the settings: <code>populationSettings</code>,
+<code>splitSettings</code>, <code>sampleSettings</code>,
+<code>featureEngineeringSettings</code>, <code>preprocessSettings</code>
+and <code>modelSettings</code> to train and evaluate the model.</p>
 <div class="sourceCode" id="cb34"><pre class="downlit sourceCode r">
-<code class="sourceCode R">  <span class="va">gbmResults</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/runPlp.html">runPlp</a></span><span class="op">(</span>
-    plpData <span class="op">=</span> <span class="va">plpData</span>,
-    outcomeId <span class="op">=</span> <span class="fl">2</span>, 
-    analysisId <span class="op">=</span> <span class="st">'singleDemo2'</span>,
-    analysisName <span class="op">=</span> <span class="st">'Demonstration of runPlp for training single PLP models'</span>,
-    populationSettings <span class="op">=</span> <span class="va">populationSettings</span>, 
-    splitSettings <span class="op">=</span> <span class="va">splitSettings</span>,
-    sampleSettings <span class="op">=</span> <span class="va">sampleSettings</span>, 
-    featureEngineeringSettings <span class="op">=</span> <span class="va">featureEngineeringSettings</span>, 
-    preprocessSettings <span class="op">=</span> <span class="va">preprocessSettings</span>,
-    modelSettings <span class="op">=</span> <span class="va">gbmModel</span>,
-    logSettings <span class="op">=</span> <span class="fu"><a href="../reference/createLogSettings.html">createLogSettings</a></span><span class="op">(</span><span class="op">)</span>, 
-    executeSettings <span class="op">=</span> <span class="fu"><a href="../reference/createExecuteSettings.html">createExecuteSettings</a></span><span class="op">(</span>
-      runSplitData <span class="op">=</span> <span class="cn">T</span>, 
-      runSampleData <span class="op">=</span> <span class="cn">T</span>, 
-      runfeatureEngineering <span class="op">=</span> <span class="cn">T</span>, 
-      runPreprocessData <span class="op">=</span> <span class="cn">T</span>, 
-      runModelDevelopment <span class="op">=</span> <span class="cn">T</span>, 
-      runCovariateSummary <span class="op">=</span> <span class="cn">T</span>
-    <span class="op">)</span>, 
-    saveDirectory <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">'singlePlpExample2'</span><span class="op">)</span>
-    <span class="op">)</span></code></pre></div>
-<p>Under the hood the package will now use the R xgboost package to fit a a gradient boosting machine model using 75% of the data and will evaluate the model on the remaining 25%. A results data structure is returned containing information about the model, its performance etc.</p>
+<code class="sourceCode R"><span>  <span class="va">gbmResults</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/runPlp.html">runPlp</a></span><span class="op">(</span></span>
+<span>    plpData <span class="op">=</span> <span class="va">plpData</span>,</span>
+<span>    outcomeId <span class="op">=</span> <span class="fl">2</span>, </span>
+<span>    analysisId <span class="op">=</span> <span class="st">'singleDemo2'</span>,</span>
+<span>    analysisName <span class="op">=</span> <span class="st">'Demonstration of runPlp for training single PLP models'</span>,</span>
+<span>    populationSettings <span class="op">=</span> <span class="va">populationSettings</span>, </span>
+<span>    splitSettings <span class="op">=</span> <span class="va">splitSettings</span>,</span>
+<span>    sampleSettings <span class="op">=</span> <span class="va">sampleSettings</span>, </span>
+<span>    featureEngineeringSettings <span class="op">=</span> <span class="va">featureEngineeringSettings</span>, </span>
+<span>    preprocessSettings <span class="op">=</span> <span class="va">preprocessSettings</span>,</span>
+<span>    modelSettings <span class="op">=</span> <span class="va">gbmModel</span>,</span>
+<span>    logSettings <span class="op">=</span> <span class="fu"><a href="../reference/createLogSettings.html">createLogSettings</a></span><span class="op">(</span><span class="op">)</span>, </span>
+<span>    executeSettings <span class="op">=</span> <span class="fu"><a href="../reference/createExecuteSettings.html">createExecuteSettings</a></span><span class="op">(</span></span>
+<span>      runSplitData <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>      runSampleData <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>      runfeatureEngineering <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>      runPreprocessData <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>      runModelDevelopment <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>      runCovariateSummary <span class="op">=</span> <span class="cn">T</span></span>
+<span>    <span class="op">)</span>, </span>
+<span>    saveDirectory <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html" class="external-link">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">'singlePlpExample2'</span><span class="op">)</span></span>
+<span>    <span class="op">)</span></span></code></pre></div>
+<p>Under the hood the package will now use the R xgboost package to fit
+a a gradient boosting machine model using 75% of the data and will
+evaluate the model on the remaining 25%. A results data structure is
+returned containing information about the model, its performance
+etc.</p>
 <p>You can save the model using:</p>
 <div class="sourceCode" id="cb35"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="../reference/savePlpModel.html">savePlpModel</a></span><span class="op">(</span><span class="va">gbmResults</span><span class="op">$</span><span class="va">model</span>, dirPath <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"model"</span><span class="op">)</span><span class="op">)</span></code></pre></div>
+<code class="sourceCode R"><span><span class="fu"><a href="../reference/savePlpModel.html">savePlpModel</a></span><span class="op">(</span><span class="va">gbmResults</span><span class="op">$</span><span class="va">model</span>, dirPath <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html" class="external-link">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"model"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <p>You can load the model using:</p>
 <div class="sourceCode" id="cb36"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">plpModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/loadPlpModel.html">loadPlpModel</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"model"</span><span class="op">)</span><span class="op">)</span></code></pre></div>
+<code class="sourceCode R"><span><span class="va">plpModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/loadPlpModel.html">loadPlpModel</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html" class="external-link">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"model"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <p>You can also save the full results structure using:</p>
 <div class="sourceCode" id="cb37"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="../reference/savePlpResult.html">savePlpResult</a></span><span class="op">(</span><span class="va">gbmResults</span>, location <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"gbm"</span><span class="op">)</span><span class="op">)</span></code></pre></div>
+<code class="sourceCode R"><span><span class="fu"><a href="../reference/savePlpResult.html">savePlpResult</a></span><span class="op">(</span><span class="va">gbmResults</span>, location <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html" class="external-link">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"gbm"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <p>To load the full results structure use:</p>
 <div class="sourceCode" id="cb38"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">gbmResults</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/loadPlpResult.html">loadPlpResult</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"gbm"</span><span class="op">)</span><span class="op">)</span></code></pre></div>
+<code class="sourceCode R"><span><span class="va">gbmResults</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/loadPlpResult.html">loadPlpResult</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html" class="external-link">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"gbm"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <div style="page-break-after: always;"></div>
 </div>
 </div>
 </div>
-<div id="study-package-creation" class="section level1">
-<h1 class="hasAnchor">
-<a href="#study-package-creation" class="anchor"></a>Study package creation</h1>
-<p>The script we created manually above can also be automatically created using a powerful feature in ATLAS. By creating a new prediction study (left menu) you can select the Target and Outcome as created in ATLAS, set all the study parameters, and then you can download a R package that you can use to execute your study. What is really powerful is that you can add multiple Ts, Os, covariate settings etc. The package will then run all the combinations of automatically as separate analyses. The screenshots below explain this process.</p>
+<div class="section level2">
+<h2 id="study-package-creation">Study package creation<a class="anchor" aria-label="anchor" href="#study-package-creation"></a>
+</h2>
+<p>The script we created manually above can also be automatically
+created using a powerful feature in ATLAS. By creating a new prediction
+study (left menu) you can select the Target and Outcome as created in
+ATLAS, set all the study parameters, and then you can download a R
+package that you can use to execute your study. What is really powerful
+is that you can add multiple Ts, Os, covariate settings etc. The package
+will then run all the combinations of automatically as separate
+analyses. The screenshots below explain this process.</p>
 <ol style="list-style-type: decimal">
 <li><dl>
 <dt>
-Create a new prediction study and select your target and outcome cohorts.
+Create a new prediction study and select your target and outcome
+cohorts.
 </dt>
 <dd>
-<img src="atlasplp1.png">
+<img src="atlasplp1.webp">
 </dd>
 </dl></li>
-<li><dl>
+<li>
+<dl>
 <dt>
 Specify one or more analysis settings.
 </dt>
 <dd>
-<img src="atlasplp2.png">
+<img src="atlasplp2.web">
 </dd>
-</dl></li>
+</dl>
+<div style="page-break-after: always;"></div>
+</li>
 <li><dl>
 <dt>
 Specify the trainings settigns
 </dt>
 <dd>
-<img src="atlasplp3.png">
+<img src="atlasplp3.webp">
 </dd>
 </dl></li>
-<li><dl>
+<li>
+<dl>
 <dt>
 Specify the execution settings
 </dt>
 <dd>
-<img src="atlasplp4.png">
+<img src="atlasplp4.web">
 </dd>
-</dl></li>
+</dl>
+] </li>
 </ol>
-<div style="page-break-after: always;"></div>
-<p>ATLAS can build a R package for you that will execute the full study against you CDM. Below the steps are explained how to do this in ATLAS.</p>
+<p>ATLAS can build a R package for you that will execute the full study
+against you CDM. Below the steps are explained how to do this in
+ATLAS.</p>
 <ol style="list-style-type: decimal">
 <li><dl>
 <dt>
-<p>Under utilities you can find download. Click on the button to review the full study specification</p>
+<p>Under utilities you can find download. Click on the button to review
+the full study specification</p>
 </dt>
 <dd>
-<div class="figure">
-<img src="atlasdownload1.png" alt=""><p class="caption">R package download functionality in ATLAS</p>
+<div class="float">
+<img src="atlasdownload1.webp" alt="R package download functionality in ATLAS"><div class="figcaption">R package download functionality in ATLAS</div>
 </div>
 </dd>
 </dl></li>
 <li><dl>
 <dt>
-<p>You now have to review that you indeed want to run all these analyses (cartesian product of all the settings for each T and O combination.</p>
+<p>You now have to review that you indeed want to run all these analyses
+(cartesian product of all the settings for each T and O combination.</p>
 </dt>
 <dd>
-<div class="figure">
-<img src="atlasdownload2.png" alt=""><p class="caption">R package download functionality in ATLAS</p>
+<div class="float">
+<img src="atlasdownload2.webp" alt="R package download functionality in ATLAS"><div class="figcaption">R package download functionality in ATLAS</div>
 </div>
 </dd>
 </dl></li>
-<li><p>If you agree, you give the package a name, and download the package as a zipfile.</p></li>
-<li><p>By opening the R package in R studio and building the package you can run the study using the <code>execute</code> function. Theres is also an example CodeToRun.R script available in the extras folder of the package with extra instructions.</p></li>
+<li><p>If you agree, you give the package a name, and download the
+package as a zipfile.</p></li>
+<li><p>By opening the R package in R studio and building the package you
+can run the study using the <code>execute</code> function. Theres is
+also an example CodeToRun.R script available in the extras folder of the
+package with extra instructions.</p></li>
 </ol>
 </div>
-<div id="internal-validation" class="section level1">
-<h1 class="hasAnchor">
-<a href="#internal-validation" class="anchor"></a>Internal validation</h1>
-<p>Once we execute the study, the runPlp() function returns the trained model and the evaluation of the model on the train/test sets.</p>
-<p>You can interactively view the results by running: <code><a href="../reference/viewPlp.html">viewPlp(runPlp=lrResults)</a></code>. This will generate a Shiny App in your browser in which you can view all performance measures created by the framework as shown in the figure below.</p>
+<div class="section level2">
+<h2 id="internal-validation">Internal validation<a class="anchor" aria-label="anchor" href="#internal-validation"></a>
+</h2>
+<p>Once we execute the study, the runPlp() function returns the trained
+model and the evaluation of the model on the train/test sets.</p>
+<p>You can interactively view the results by running:
+<code>viewPlp(runPlp=lrResults)</code>. This will generate a Shiny App
+in your browser in which you can view all performance measures created
+by the framework as shown in the figure below.</p>
 <dd>
-<img src="shinysummary.PNG" title="fig:" alt="Summary of all the performance measures of the analyses">
+<div class="float">
+<img src="shinysummary.webp" alt="Summary of all the performance measures of the analyses"><div class="figcaption">Summary of all the performance measures of the
+analyses</div>
+</div>
 </dd>
-Furthermore, many interactive plots are available in the Shiny App, for example the ROC curve in which you can move over the plot to see the threshold and the corresponding sensitivity and specificity values.
+<p>Furthermore, many interactive plots are available in the Shiny App,
+for example the ROC curve in which you can move over the plot to see the
+threshold and the corresponding sensitivity and specificity values.</p>
 <dd>
-<img src="shinyroc.PNG" title="fig:" alt="Example of the interactive ROC curve">
+<div class="float">
+<img src="shinyroc.webp" alt="Example of the interactive ROC curve"><div class="figcaption">Example of the interactive ROC curve</div>
+</div>
 </dd>
-<p>To generate and save all the evaluation plots to a folder run the following code:</p>
+<p>To generate and save all the evaluation plots to a folder run the
+following code:</p>
 <div class="sourceCode" id="cb39"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="../reference/plotPlp.html">plotPlp</a></span><span class="op">(</span><span class="va">lrResults</span>, dirPath <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/getwd.html">getwd</a></span><span class="op">(</span><span class="op">)</span><span class="op">)</span></code></pre></div>
+<code class="sourceCode R"><span><span class="fu"><a href="../reference/plotPlp.html">plotPlp</a></span><span class="op">(</span><span class="va">lrResults</span>, dirPath <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <p>The plots are described in more detail in the next sections.</p>
 <div style="page-break-after: always;"></div>
-<div id="discrimination" class="section level2">
-<h2 class="hasAnchor">
-<a href="#discrimination" class="anchor"></a>Discrimination</h2>
-<p>The Receiver Operating Characteristics (ROC) plot shows the sensitivity against 1-specificity on the test set. The plot illustrates how well the model is able to discriminate between the people with the outcome and those without. The dashed diagonal line is the performance of a model that randomly assigns predictions. The higher the area under the ROC plot the better the discrimination of the model. The plot is created by changing the probability threshold to assign the positive class.</p>
+<div class="section level3">
+<h3 id="discrimination">Discrimination<a class="anchor" aria-label="anchor" href="#discrimination"></a>
+</h3>
+<p>The Receiver Operating Characteristics (ROC) plot shows the
+sensitivity against 1-specificity on the test set. The plot illustrates
+how well the model is able to discriminate between the people with the
+outcome and those without. The dashed diagonal line is the performance
+of a model that randomly assigns predictions. The higher the area under
+the ROC plot the better the discrimination of the model. The plot is
+created by changing the probability threshold to assign the positive
+class.</p>
 <dd>
-<img src="sparseROC.png" title="fig:" alt="Receiver Operating Characteristic Plot">
+<div class="float">
+<img src="sparseROC.webp" alt="Receiver Operating Characteristic Plot"><div class="figcaption">Receiver Operating Characteristic Plot</div>
+</div>
 </dd>
-<p> ## Calibration</p>
-<p>The calibration plot shows how close the predicted risk is to the observed risk. The diagonal dashed line thus indicates a perfectly calibrated model. The ten (or fewer) dots represent the mean predicted values for each quantile plotted against the observed fraction of people in that quantile who had the outcome (observed fraction). The straight black line is the linear regression using these 10 plotted quantile mean predicted vs observed fraction points. The straight vertical lines represented the 95% lower and upper confidence intervals of the slope of the fitted line.</p>
+<div style="page-break-after: always;"></div>
+<p>## Calibration</p>
+<p>The calibration plot shows how close the predicted risk is to the
+observed risk. The diagonal dashed line thus indicates a perfectly
+calibrated model. The ten (or fewer) dots represent the mean predicted
+values for each quantile plotted against the observed fraction of people
+in that quantile who had the outcome (observed fraction). The straight
+black line is the linear regression using these 10 plotted quantile mean
+predicted vs observed fraction points. The straight vertical lines
+represented the 95% lower and upper confidence intervals of the slope of
+the fitted line.</p>
 <dd>
-<img src="sparseCalibration.png" title="fig:" alt="Calibration Plot">
+<div class="float">
+<img src="sparseCalibration.webp" alt="Calibration Plot"><div class="figcaption">Calibration Plot</div>
+</div>
 </dd>
 <div style="page-break-after: always;"></div>
 </div>
-<div id="smooth-calibration" class="section level2">
-<h2 class="hasAnchor">
-<a href="#smooth-calibration" class="anchor"></a>Smooth Calibration</h2>
-<p>Similar to the traditional calibration shown above the Smooth Calibration plot shows the relationship between predicted and observed risk. the major difference is that the smooth fit allows for a more fine grained examination of this. Whereas the traditional plot will be heavily influenced by the areas with the highest density of data the smooth plot will provide the same information for this region as well as a more accurate interpretation of areas with lower density. the plot also contains information on the distribution of the outcomes relative to predicted risk.</p>
-<p>However, the increased information gain comes at a computational cost. It is recommended to use the traditional plot for examination and then to produce the smooth plot for final versions. To create the smooth calibarion plot you have to run the follow command:</p>
+<div class="section level3">
+<h3 id="smooth-calibration">Smooth Calibration<a class="anchor" aria-label="anchor" href="#smooth-calibration"></a>
+</h3>
+<p>Similar to the traditional calibration shown above the Smooth
+Calibration plot shows the relationship between predicted and observed
+risk. the major difference is that the smooth fit allows for a more fine
+grained examination of this. Whereas the traditional plot will be
+heavily influenced by the areas with the highest density of data the
+smooth plot will provide the same information for this region as well as
+a more accurate interpretation of areas with lower density. the plot
+also contains information on the distribution of the outcomes relative
+to predicted risk.</p>
+<p>However, the increased information gain comes at a computational
+cost. It is recommended to use the traditional plot for examination and
+then to produce the smooth plot for final versions. To create the smooth
+calibarion plot you have to run the follow command:</p>
 <div class="sourceCode" id="cb40"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="../reference/plotSmoothCalibration.html">plotSmoothCalibration</a></span><span class="op">(</span><span class="va">lrResults</span><span class="op">)</span></code></pre></div>
-<p>See the help function for more information, on how to set the smoothing method etc.</p>
-<p>The example below is from another study that better demonstrates the impact of using a smooth calibration plot. The default line fit would not highlight the miss-calibration at the lower predicted probability levels that well.</p>
+<code class="sourceCode R"><span><span class="fu"><a href="../reference/plotSmoothCalibration.html">plotSmoothCalibration</a></span><span class="op">(</span><span class="va">lrResults</span><span class="op">)</span></span></code></pre></div>
+<p>See the help function for more information, on how to set the
+smoothing method etc.</p>
+<p>The example below is from another study that better demonstrates the
+impact of using a smooth calibration plot. The default line fit would
+not highlight the miss-calibration at the lower predicted probability
+levels that well.</p>
 <dd>
-<img src="smoothCalibration.jpeg" title="fig:" alt="Smooth Calibration plot">
+<div class="float">
+<img src="smoothCalibration.jpeg" alt="Smooth Calibration plot"><div class="figcaption">Smooth Calibration plot</div>
+</div>
 </dd>
-<p> ## Preference distribution</p>
-<p>The preference distribution plots are the preference score distributions corresponding to i) people in the test set with the outcome (red) and ii) people in the test set without the outcome (blue).</p>
+<div style="page-break-after: always;"></div>
+<p>## Preference distribution</p>
+<p>The preference distribution plots are the preference score
+distributions corresponding to i) people in the test set with the
+outcome (red) and ii) people in the test set without the outcome
+(blue).</p>
 <dd>
-<img src="preferencePDF.png" title="fig:" alt="Preference Plot">
+<div class="float">
+<img src="preferencePDF.webp" alt="Preference Plot"><div class="figcaption">Preference Plot</div>
+</div>
 </dd>
-<p> ## Predicted probability distribution</p>
-<p>The prediction distribution box plots are for the predicted risks of the people in the test set with the outcome (class 1: blue) and without the outcome (class 0: red).</p>
-<p>The box plots in the Figure show that the predicted probability of the outcome is indeed higher for those with the outcome but there is also overlap between the two distribution which lead to an imperfect discrimination.</p>
+<div style="page-break-after: always;"></div>
+<p>## Predicted probability distribution</p>
+<p>The prediction distribution box plots are for the predicted risks of
+the people in the test set with the outcome (class 1: blue) and without
+the outcome (class 0: red).</p>
+<p>The box plots in the Figure show that the predicted probability of
+the outcome is indeed higher for those with the outcome but there is
+also overlap between the two distribution which lead to an imperfect
+discrimination.</p>
 <dd>
-<img src="predictionDistribution.png" title="fig:" alt="Prediction Distribution Box Plot">
+<div class="float">
+<img src="predictionDistribution.wwebp" alt="Prediction Distribution Box Plot"><div class="figcaption">Prediction Distribution Box Plot</div>
+</div>
 </dd>
-<p> ## Test-Train similarity</p>
-<p>The test-train similarity is assessed by plotting the mean covariate values in the train set against those in the test set for people with and without the outcome.</p>
-<p>The results for our example of look very promising since the mean values of the covariates are on the diagonal.</p>
+<div style="page-break-after: always;"></div>
+<p>## Test-Train similarity</p>
+<p>The test-train similarity is assessed by plotting the mean covariate
+values in the train set against those in the test set for people with
+and without the outcome.</p>
+<p>The results for our example of look very promising since the mean
+values of the covariates are on the diagonal.</p>
 <dd>
-<img src="generalizability.png" title="fig:" alt="Similarity plots of train and test set">
+<div class="float">
+<img src="generalizability.webp" alt="Similarity plots of train and test set"><div class="figcaption">Similarity plots of train and test set</div>
+</div>
 </dd>
-<p> ## Variable scatter plot</p>
-<p>The variable scatter plot shows the mean covariate value for the people with the outcome against the mean covariate value for the people without the outcome. The color of the dots corresponds to the inclusion (green) or exclusion in the model (blue), respectively. It is highly recommended to use the Shiny App since this allows you to hoover over a covariate to show more details (name, value etc).</p>
-<p>The plot shows that the mean of most of the covariates is higher for subjects with the outcome compared to those without.</p>
+<div style="page-break-after: always;"></div>
+<p>## Variable scatter plot</p>
+<p>The variable scatter plot shows the mean covariate value for the
+people with the outcome against the mean covariate value for the people
+without the outcome. The color of the dots corresponds to the inclusion
+(green) or exclusion in the model (blue), respectively. It is highly
+recommended to use the Shiny App since this allows you to hoover over a
+covariate to show more details (name, value etc).</p>
+<p>The plot shows that the mean of most of the covariates is higher for
+subjects with the outcome compared to those without.</p>
 <dd>
-<img src="variableScatterplot.png" title="fig:" alt="Variabel scatter Plot">
+<div class="float">
+<img src="variableScatterplot.webp" alt="Variabel scatter Plot"><div class="figcaption">Variabel scatter Plot</div>
+</div>
 </dd>
-<p> ## Precision recall</p>
-<p>Precision (P) is defined as the number of true positives (Tp) over the number of true positives plus the number of false positives (Fp).</p>
+<div style="page-break-after: always;"></div>
+<p>## Precision recall</p>
+<p>Precision (P) is defined as the number of true positives (Tp) over
+the number of true positives plus the number of false positives
+(Fp).</p>
 <div class="sourceCode" id="cb41"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">P</span> <span class="op">&lt;-</span> <span class="va">Tp</span><span class="op">/</span><span class="op">(</span><span class="va">Tp</span> <span class="op">+</span> <span class="va">Fp</span><span class="op">)</span></code></pre></div>
-<p>Recall (R) is defined as the number of true positives (Tp) over the number of true positives plus the number of false negatives (Fn).</p>
+<code class="sourceCode R"><span><span class="va">P</span> <span class="op">&lt;-</span> <span class="va">Tp</span><span class="op">/</span><span class="op">(</span><span class="va">Tp</span> <span class="op">+</span> <span class="va">Fp</span><span class="op">)</span></span></code></pre></div>
+<p>Recall (R) is defined as the number of true positives (Tp) over the
+number of true positives plus the number of false negatives (Fn).</p>
 <div class="sourceCode" id="cb42"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">R</span> <span class="op">&lt;-</span> <span class="va">Tp</span><span class="op">/</span><span class="op">(</span><span class="va">Tp</span> <span class="op">+</span> <span class="va">Fn</span><span class="op">)</span></code></pre></div>
-<p>These quantities are also related to the (F1) score, which is defined as the harmonic mean of precision and recall.</p>
+<code class="sourceCode R"><span><span class="va">R</span> <span class="op">&lt;-</span> <span class="va">Tp</span><span class="op">/</span><span class="op">(</span><span class="va">Tp</span> <span class="op">+</span> <span class="va">Fn</span><span class="op">)</span></span></code></pre></div>
+<p>These quantities are also related to the (F1) score, which is defined
+as the harmonic mean of precision and recall.</p>
 <div class="sourceCode" id="cb43"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">F1</span> <span class="op">&lt;-</span> <span class="fl">2</span> <span class="op">*</span> <span class="va">P</span> <span class="op">*</span> <span class="va">R</span><span class="op">/</span><span class="op">(</span><span class="va">P</span> <span class="op">+</span> <span class="va">R</span><span class="op">)</span></code></pre></div>
-<p>Note that the precision can either decrease or increase if the threshold is lowered. Lowering the threshold of a classifier may increase the denominator, by increasing the number of results returned. If the threshold was previously set too high, the new results may all be true positives, which will increase precision. If the previous threshold was about right or too low, further lowering the threshold will introduce false positives, decreasing precision.</p>
-<p>For Recall the denominator does not depend on the classifier threshold (Tp+Fn is a constant). This means that lowering the classifier threshold may increase recall, by increasing the number of true positive results. It is also possible that lowering the threshold may leave recall unchanged, while the precision fluctuates.</p>
+<code class="sourceCode R"><span><span class="va">F1</span> <span class="op">&lt;-</span> <span class="fl">2</span> <span class="op">*</span> <span class="va">P</span> <span class="op">*</span> <span class="va">R</span><span class="op">/</span><span class="op">(</span><span class="va">P</span> <span class="op">+</span> <span class="va">R</span><span class="op">)</span></span></code></pre></div>
+<p>Note that the precision can either decrease or increase if the
+threshold is lowered. Lowering the threshold of a classifier may
+increase the denominator, by increasing the number of results returned.
+If the threshold was previously set too high, the new results may all be
+true positives, which will increase precision. If the previous threshold
+was about right or too low, further lowering the threshold will
+introduce false positives, decreasing precision.</p>
+<p>For Recall the denominator does not depend on the classifier
+threshold (Tp+Fn is a constant). This means that lowering the classifier
+threshold may increase recall, by increasing the number of true positive
+results. It is also possible that lowering the threshold may leave
+recall unchanged, while the precision fluctuates.</p>
 <dd>
-<img src="precisionRecall.png" title="fig:" alt="Precision Recall Plot">
+<div class="float">
+<img src="precisionRecall.webp" alt="Precision Recall Plot"><div class="figcaption">Precision Recall Plot</div>
+</div>
 </dd>
-<p> ## Demographic summary</p>
-<p>This plot shows for females and males the expected and observed risk in different age groups together with a confidence area.</p>
-<p>The results show that our model is well calibrated across gender and age groups.</p>
+<div style="page-break-after: always;"></div>
+<p>## Demographic summary</p>
+<p>This plot shows for females and males the expected and observed risk
+in different age groups together with a confidence area.</p>
+<p>The results show that our model is well calibrated across gender and
+age groups.</p>
 <dd>
-<img src="demographicSummary.png" title="fig:" alt="Demographic Summary Plot">
+<div class="float">
+<img src="demographicSummary.webp" alt="Demographic Summary Plot"><div class="figcaption">Demographic Summary Plot</div>
+</div>
 </dd>
-<p> # External validation</p>
-<p>We recommend to always perform external validation, i.e. apply the final model on as much new datasets as feasible and evaluate its performance.</p>
+<div style="page-break-after: always;"></div>
+<p># External validation</p>
+<p>We recommend to always perform external validation, i.e. apply the
+final model on as much new datasets as feasible and evaluate its
+performance.</p>
 <div class="sourceCode" id="cb44"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="co"># load the trained model</span>
-<span class="va">plpModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/loadPlpModel.html">loadPlpModel</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html">getwd</a></span><span class="op">(</span><span class="op">)</span>,<span class="st">'model'</span><span class="op">)</span>
-
-<span class="co"># add details of new database</span>
-<span class="va">validationDatabaseDetails</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDatabaseDetails.html">createDatabaseDetails</a></span><span class="op">(</span><span class="op">)</span>
-
-<span class="co"># to externally validate the model and perform recalibration run:</span>
-<span class="fu"><a href="../reference/externalValidateDbPlp.html">externalValidateDbPlp</a></span><span class="op">(</span>
-  plpModel <span class="op">=</span> <span class="va">plpModel</span>,
-  validationDatabaseDetails <span class="op">=</span> <span class="va">validationDatabaseDetails</span>,
-  validationRestrictPlpDataSettings <span class="op">=</span> <span class="va">plpModel</span><span class="op">$</span><span class="va">settings</span><span class="op">$</span><span class="va">plpDataSettings</span>,
-  settings <span class="op">=</span> <span class="fu"><a href="../reference/createValidationSettings.html">createValidationSettings</a></span><span class="op">(</span>
-    recalibrate <span class="op">=</span> <span class="st">'weakRecalibration'</span>
-    <span class="op">)</span>,
-  outputFolder <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/getwd.html">getwd</a></span><span class="op">(</span><span class="op">)</span>
-<span class="op">)</span></code></pre></div>
-<p>This will extract the new plpData from the specified schemas and cohort tables. It will then apply the same population settings and the trained plp model. Finally, it will evaluate the performance and return the standard output as <code>validation$performanceEvaluation</code> and it will also return the prediction on the population as <code>validation$prediction</code>. They can be inserted into the shiny app for viewing the model and validation by running: <code><a href="../reference/viewPlp.html">viewPlp(runPlp=plpResult, validatePlp=validation )</a></code>.</p>
+<code class="sourceCode R"><span><span class="co"># load the trained model</span></span>
+<span><span class="va">plpModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/loadPlpModel.html">loadPlpModel</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span>,<span class="st">'model'</span><span class="op">)</span></span>
+<span></span>
+<span><span class="co"># add details of new database</span></span>
+<span><span class="va">validationDatabaseDetails</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDatabaseDetails.html">createDatabaseDetails</a></span><span class="op">(</span><span class="op">)</span></span>
+<span></span>
+<span><span class="co"># to externally validate the model and perform recalibration run:</span></span>
+<span><span class="fu"><a href="../reference/externalValidateDbPlp.html">externalValidateDbPlp</a></span><span class="op">(</span></span>
+<span>  plpModel <span class="op">=</span> <span class="va">plpModel</span>,</span>
+<span>  validationDatabaseDetails <span class="op">=</span> <span class="va">validationDatabaseDetails</span>,</span>
+<span>  validationRestrictPlpDataSettings <span class="op">=</span> <span class="va">plpModel</span><span class="op">$</span><span class="va">settings</span><span class="op">$</span><span class="va">plpDataSettings</span>,</span>
+<span>  settings <span class="op">=</span> <span class="fu"><a href="../reference/createValidationSettings.html">createValidationSettings</a></span><span class="op">(</span></span>
+<span>    recalibrate <span class="op">=</span> <span class="st">'weakRecalibration'</span></span>
+<span>    <span class="op">)</span>,</span>
+<span>  outputFolder <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+<p>This will extract the new plpData from the specified schemas and
+cohort tables. It will then apply the same population settings and the
+trained plp model. Finally, it will evaluate the performance and return
+the standard output as <code>validation$performanceEvaluation</code> and
+it will also return the prediction on the population as
+<code>validation$prediction</code>. They can be inserted into the shiny
+app for viewing the model and validation by running:
+<code>viewPlp(runPlp=plpResult, validatePlp=validation )</code>.</p>
 <div style="page-break-after: always;"></div>
 </div>
 </div>
-<div id="other-functionality" class="section level1">
-<h1 class="hasAnchor">
-<a href="#other-functionality" class="anchor"></a>Other functionality</h1>
-<p>The package has much more functionality than described in this vignette and contributions have been made my many persons in the OHDSI community. The table below provides an overview:</p>
+<div class="section level2">
+<h2 id="other-functionality">Other functionality<a class="anchor" aria-label="anchor" href="#other-functionality"></a>
+</h2>
+<p>The package has much more functionality than described in this
+vignette and contributions have been made my many persons in the OHDSI
+community. The table below provides an overview:</p>
 <table class="table">
 <colgroup>
-<col width="20%">
-<col width="60%">
-<col width="20%">
+<col width="23%">
+<col width="52%">
+<col width="23%">
 </colgroup>
 <thead><tr class="header">
 <th>Functionality</th>
@@ -1309,156 +2070,185 @@ <h1 class="hasAnchor">
 <tbody>
 <tr class="odd">
 <td>Builing Multiple Models</td>
-<td>This vignette describes how you can run multiple models automatically</td>
-<td><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingMultiplePredictiveModels.pdf"><code>Vignette</code></a></td>
+<td>This vignette describes how you can run multiple models
+automatically</td>
+<td><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingMultiplePredictiveModels.pdf" class="external-link"><code>Vignette</code></a></td>
 </tr>
 <tr class="even">
 <td>Custom Models</td>
-<td>This vignette describes how you can add your own custom algorithms in the framework</td>
-<td><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomModels.pdf"><code>Vignette</code></a></td>
+<td>This vignette describes how you can add your own custom algorithms
+in the framework</td>
+<td><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/AddingCustomModels.pdf" class="external-link"><code>Vignette</code></a></td>
 </tr>
 <tr class="odd">
 <td>Custom Splitting Functions</td>
-<td>This vignette describes how you can add your own custom training/validation/testing splitting functions in the framework</td>
-<td><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSplitting.pdf"><code>Vignette</code></a></td>
+<td>This vignette describes how you can add your own custom
+training/validation/testing splitting functions in the framework</td>
+<td><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/AddingCustomSplitting.pdf" class="external-link"><code>Vignette</code></a></td>
 </tr>
 <tr class="even">
 <td>Custom Sampling Functions</td>
-<td>This vignette describes how you can add your own custom sampling functions in the framework</td>
-<td><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSamples.pdf"><code>Vignette</code></a></td>
+<td>This vignette describes how you can add your own custom sampling
+functions in the framework</td>
+<td><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/AddingCustomSamples.pdf" class="external-link"><code>Vignette</code></a></td>
 </tr>
 <tr class="odd">
 <td>Custom Feature Engineering/Selection</td>
-<td>This vignette describes how you can add your own custom feature engineering and selection functions in the framework</td>
-<td><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomFeatureEngineering.pdf"><code>Vignette</code></a></td>
+<td>This vignette describes how you can add your own custom feature
+engineering and selection functions in the framework</td>
+<td><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/AddingCustomFeatureEngineering.pdf" class="external-link"><code>Vignette</code></a></td>
 </tr>
 <tr class="even">
 <td>Ensemble models</td>
-<td>This vignette describes how you can use the framework to build ensemble models, i.e combine multiple models in a super learner</td>
-<td><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingEnsembleModels.pdf"><code>Vignette</code></a></td>
+<td>This vignette describes how you can use the framework to build
+ensemble models, i.e combine multiple models in a super learner</td>
+<td><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingEnsembleModels.pdf" class="external-link"><code>Vignette</code></a></td>
 </tr>
 <tr class="odd">
 <td>Learning curves</td>
-<td>Learning curves assess the effect of training set size on model performance by training a sequence of prediction models on successively larger subsets of the training set. A learning curve plot can also help in diagnosing a bias or variance problem as explained below.</td>
-<td><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/GeneratingLearningCurves.pdf"><code>Vignette</code></a></td>
+<td>Learning curves assess the effect of training set size on model
+performance by training a sequence of prediction models on successively
+larger subsets of the training set. A learning curve plot can also help
+in diagnosing a bias or variance problem as explained below.</td>
+<td><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/GeneratingLearningCurves.pdf" class="external-link"><code>Vignette</code></a></td>
 </tr>
 </tbody>
 </table>
 </div>
-<div id="demos" class="section level1">
-<h1 class="hasAnchor">
-<a href="#demos" class="anchor"></a>Demos</h1>
-<p>We have added several demos in the package that run on simulated data:</p>
+<div class="section level2">
+<h2 id="demos">Demos<a class="anchor" aria-label="anchor" href="#demos"></a>
+</h2>
+<p>We have added several demos in the package that run on simulated
+data:</p>
 <div class="sourceCode" id="cb45"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="co"># Show all demos in our package: </span>
-<span class="fu"><a href="https://rdrr.io/r/utils/demo.html">demo</a></span><span class="op">(</span>package <span class="op">=</span> <span class="st">"PatientLevelPrediction"</span><span class="op">)</span>
-
-<span class="co"># For example, to run the SingleModelDemo that runs Lasso and shows you how to run the Shiny App use this call</span>
-<span class="fu"><a href="https://rdrr.io/r/utils/demo.html">demo</a></span><span class="op">(</span><span class="st">"SingleModelDemo"</span>, package <span class="op">=</span> <span class="st">"PatientLevelPrediction"</span><span class="op">)</span></code></pre></div>
+<code class="sourceCode R"><span><span class="co"># Show all demos in our package: </span></span>
+<span><span class="fu"><a href="https://rdrr.io/r/utils/demo.html" class="external-link">demo</a></span><span class="op">(</span>package <span class="op">=</span> <span class="st">"PatientLevelPrediction"</span><span class="op">)</span></span>
+<span></span>
+<span><span class="co"># For example, to run the SingleModelDemo that runs Lasso and shows you how to run the Shiny App use this call</span></span>
+<span><span class="fu"><a href="https://rdrr.io/r/utils/demo.html" class="external-link">demo</a></span><span class="op">(</span><span class="st">"SingleModelDemo"</span>, package <span class="op">=</span> <span class="st">"PatientLevelPrediction"</span><span class="op">)</span></span></code></pre></div>
 <div style="page-break-after: always;"></div>
 </div>
-<div id="acknowledgments" class="section level1">
-<h1 class="hasAnchor">
-<a href="#acknowledgments" class="anchor"></a>Acknowledgments</h1>
-<p>Considerable work has been dedicated to provide the <code>PatientLevelPrediction</code> package.</p>
+<div class="section level2">
+<h2 id="acknowledgments">Acknowledgments<a class="anchor" aria-label="anchor" href="#acknowledgments"></a>
+</h2>
+<p>Considerable work has been dedicated to provide the
+<code>PatientLevelPrediction</code> package.</p>
 <div class="sourceCode" id="cb46"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/utils/citation.html">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></code></pre></div>
-<pre><code>## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design
-## and implementation of a standardized framework to generate and evaluate
-## patient-level prediction models using observational healthcare data."
-## _Journal of the American Medical Informatics Association_, *25*(8),
-## 969-975. &lt;URL: https://doi.org/10.1093/jamia/ocy032&gt;.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }</code></pre>
-<p>Further, <code>PatientLevelPrediction</code> makes extensive use of the <code>Cyclops</code> package.</p>
+<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/utils/citation.html" class="external-link">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></span></code></pre></div>
+<pre><code><span><span class="co">## </span></span>
+<span><span class="co">## To cite PatientLevelPrediction in publications use:</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">##   Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design</span></span>
+<span><span class="co">##   and implementation of a standardized framework to generate and</span></span>
+<span><span class="co">##   evaluate patient-level prediction models using observational</span></span>
+<span><span class="co">##   healthcare data." _Journal of the American Medical Informatics</span></span>
+<span><span class="co">##   Association_, *25*(8), 969-975.</span></span>
+<span><span class="co">##   &lt;https://doi.org/10.1093/jamia/ocy032&gt;.</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">## A BibTeX entry for LaTeX users is</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">##   @Article{,</span></span>
+<span><span class="co">##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},</span></span>
+<span><span class="co">##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},</span></span>
+<span><span class="co">##     journal = {Journal of the American Medical Informatics Association},</span></span>
+<span><span class="co">##     volume = {25},</span></span>
+<span><span class="co">##     number = {8},</span></span>
+<span><span class="co">##     pages = {969-975},</span></span>
+<span><span class="co">##     year = {2018},</span></span>
+<span><span class="co">##     url = {https://doi.org/10.1093/jamia/ocy032},</span></span>
+<span><span class="co">##   }</span></span></code></pre>
+<p>Further, <code>PatientLevelPrediction</code> makes extensive use of
+the <code>Cyclops</code> package.</p>
 <div class="sourceCode" id="cb48"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/utils/citation.html">citation</a></span><span class="op">(</span><span class="st">"Cyclops"</span><span class="op">)</span></code></pre></div>
-<pre><code>## 
-## To cite Cyclops in publications use:
-## 
-## Suchard MA, Simpson SE, Zorych I, Ryan P, Madigan D (2013). "Massive
-## parallelization of serial inference algorithms for complex generalized
-## linear models." _ACM Transactions on Modeling and Computer Simulation_,
-## *23*, 10. &lt;URL: https://dl.acm.org/doi/10.1145/2414416.2414791&gt;.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {M. A. Suchard and S. E. Simpson and I. Zorych and P. Ryan and D. Madigan},
-##     title = {Massive parallelization of serial inference algorithms for complex generalized linear models},
-##     journal = {ACM Transactions on Modeling and Computer Simulation},
-##     volume = {23},
-##     pages = {10},
-##     year = {2013},
-##     url = {https://dl.acm.org/doi/10.1145/2414416.2414791},
-##   }</code></pre>
-<p><strong>Please reference this paper if you use the PLP Package in your work:</strong></p>
-<p><a href="http://dx.doi.org/10.1093/jamia/ocy032">Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.</a></p>
-<p>This work is supported in part through the National Science Foundation grant IIS 1251151.</p>
+<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/utils/citation.html" class="external-link">citation</a></span><span class="op">(</span><span class="st">"Cyclops"</span><span class="op">)</span></span></code></pre></div>
+<pre><code><span><span class="co">## </span></span>
+<span><span class="co">## To cite Cyclops in publications use:</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">##   Suchard MA, Simpson SE, Zorych I, Ryan P, Madigan D (2013). "Massive</span></span>
+<span><span class="co">##   parallelization of serial inference algorithms for complex</span></span>
+<span><span class="co">##   generalized linear models." _ACM Transactions on Modeling and</span></span>
+<span><span class="co">##   Computer Simulation_, *23*, 10.</span></span>
+<span><span class="co">##   &lt;https://dl.acm.org/doi/10.1145/2414416.2414791&gt;.</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">## A BibTeX entry for LaTeX users is</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">##   @Article{,</span></span>
+<span><span class="co">##     author = {M. A. Suchard and S. E. Simpson and I. Zorych and P. Ryan and D. Madigan},</span></span>
+<span><span class="co">##     title = {Massive parallelization of serial inference algorithms for complex generalized linear models},</span></span>
+<span><span class="co">##     journal = {ACM Transactions on Modeling and Computer Simulation},</span></span>
+<span><span class="co">##     volume = {23},</span></span>
+<span><span class="co">##     pages = {10},</span></span>
+<span><span class="co">##     year = {2013},</span></span>
+<span><span class="co">##     url = {https://dl.acm.org/doi/10.1145/2414416.2414791},</span></span>
+<span><span class="co">##   }</span></span></code></pre>
+<p><strong>Please reference this paper if you use the PLP Package in
+your work:</strong></p>
+<p><a href="http://dx.doi.org/10.1093/jamia/ocy032" class="external-link">Reps JM, Schuemie
+MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a
+standardized framework to generate and evaluate patient-level prediction
+models using observational healthcare data. J Am Med Inform Assoc.
+2018;25(8):969-975.</a></p>
+<p>This work is supported in part through the National Science
+Foundation grant IIS 1251151.</p>
 <div style="page-break-after: always;"></div>
 </div>
-<div id="appendix-1-study-population-settings-details" class="section level1 unnumbered">
-<h1 class="hasAnchor">
-<a href="#appendix-1-study-population-settings-details" class="anchor"></a>Appendix 1: Study population settings details</h1>
-<p>In the figures below the effect is shown of the removeSubjectsWithPriorOutcome, requireTimAtRisk, and includeAllOutcomes booleans on the final study population. We start with a Target Cohort with firstExposureOnly = false and we require a washout period = 1095. We then subset the target cohort based on additional constraints. The final study population in the Venn diagrams below are colored green.</p>
+<div class="section level2">
+<h2 class="unnumbered" id="appendix-1-study-population-settings-details">Appendix 1: Study population settings
+details<a class="anchor" aria-label="anchor" href="#appendix-1-study-population-settings-details"></a>
+</h2>
+<p>In the figures below the effect is shown of the
+removeSubjectsWithPriorOutcome, requireTimAtRisk, and includeAllOutcomes
+booleans on the final study population. We start with a Target Cohort
+with firstExposureOnly = false and we require a washout period = 1095.
+We then subset the target cohort based on additional constraints. The
+final study population in the Venn diagrams below are colored green.</p>
 <ol style="list-style-type: decimal">
 <li><dl>
 <dt>
 Require minimum time-at-risk for all person in the target cohort
 </dt>
 <dd>
-<img src="popdef1.png">
+<img src="popdef1.webp">
 </dd>
 </dl></li>
 <li><dl>
 <dt>
-Require minumum time-at-risk for target cohort, except for persons with outcomes during time-at-risk.
+Require minumum time-at-risk for target cohort, except for persons with
+outcomes during time-at-risk.
 </dt>
 <dd>
-<img src="popdef2.png">
+<img src="popdef2.webp">
 </dd>
 </dl></li>
 </ol>
-)
+<p>)</p>
 <dl>
 <dt>
-Include all persons in the target cohort exclude persons with prior outcomes
+Include all persons in the target cohort exclude persons with prior
+outcomes
 </dt>
 <dd>
-<img src="popdef3.png">
+<img src="popdef3.webp">
 </dd>
 </dl>
 <ol start="4" style="list-style-type: decimal">
 <li><dl>
 <dt>
-Require minimum time-at-risk for target cohort, except for persons with outcomes during time-at-risk, exclude persons with prior outcomes
+Require minimum time-at-risk for target cohort, except for persons with
+outcomes during time-at-risk, exclude persons with prior outcomes
 </dt>
 <dd>
-<img src="popdef4.png">
+<img src="popdef4.webp">
 </dd>
 </dl></li>
 </ol>
-)
+<p>)</p>
 <dl>
 <dt>
 Include all persons in target cohort exclude persons with prior outcomes
 </dt>
 <dd>
-<img src="popdef5.png">
+<img src="popdef5.webp">
 </dd>
 </dl>
 <ol start="6" style="list-style-type: decimal">
@@ -1467,7 +2257,7 @@ <h1 class="hasAnchor">
 Include all persons in target cohort
 </dt>
 <dd>
-<img src="popdef6.png">
+<img src="popdef6.webp">
 </dd>
 </dl></li>
 </ol>
@@ -1485,11 +2275,13 @@ <h1 class="hasAnchor">
 
 
       <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+  <p></p>
+<p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p>
+<p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
       </footer>
@@ -1498,5 +2290,7 @@ <h1 class="hasAnchor">
   
 
 
+  
+
   </body>
 </html>
diff --git a/docs/articles/BuildingPredictiveModels_files/header-attrs-2.11/header-attrs.js b/docs/articles/BuildingPredictiveModels_files/header-attrs-2.11/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/BuildingPredictiveModels_files/header-attrs-2.11/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/BuildingPredictiveModels_files/header-attrs-2.7/header-attrs.js b/docs/articles/BuildingPredictiveModels_files/header-attrs-2.7/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/BuildingPredictiveModels_files/header-attrs-2.7/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/ClinicalModels.html b/docs/articles/ClinicalModels.html
new file mode 100644
index 000000000..d3cd57891
--- /dev/null
+++ b/docs/articles/ClinicalModels.html
@@ -0,0 +1,273 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<meta charset="utf-8">
+<meta http-equiv="X-UA-Compatible" content="IE=edge">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>Clinical Models • PatientLevelPrediction</title>
+<!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous">
+<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css">
+<script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous">
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous">
+<!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet">
+<script src="../pkgdown.js"></script><meta property="og:title" content="Clinical Models">
+<meta property="og:description" content="PatientLevelPrediction">
+<!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]-->
+</head>
+<body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-article">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav">
+<li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu">
+<li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul>
+</li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul>
+<ul class="nav navbar-nav navbar-right">
+<li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul>
+</div>
+<!--/.nav-collapse -->
+  </div>
+<!--/.container -->
+</div>
+<!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header toc-ignore">
+      <h1 data-toc-skip>Clinical Models</h1>
+                        <h4 data-toc-skip class="author">Jenna Reps,
+Peter R. Rijnbeek</h4>
+            
+            <h4 data-toc-skip class="date">2024-09-09</h4>
+      
+      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/vignettes/ClinicalModels.rmd" class="external-link"><code>vignettes/ClinicalModels.rmd</code></a></small>
+      <div class="hidden name"><code>ClinicalModels.rmd</code></div>
+
+    </div>
+
+    
+    
+<!--
+%\VignetteEngine{knitr::knitr}
+%\VignetteIndexEntry{Clinical Models}
+-->
+<div class="section level2">
+<h2 id="clinical-models-developed-using-the-ohdsi-patientlevelprediction-framework">Clinical models developed using the OHDSI PatientLevelPrediction
+framework<a class="anchor" aria-label="anchor" href="#clinical-models-developed-using-the-ohdsi-patientlevelprediction-framework"></a>
+</h2>
+<table class="table">
+<colgroup>
+<col width="75%">
+<col width="24%">
+</colgroup>
+<thead><tr class="header">
+<th>Title</th>
+<th>Link</th>
+</tr></thead>
+<tbody>
+<tr class="odd">
+<td>Using Machine Learning Applied to Real-World Healthcare Data for
+Predictive Analytics: An Applied Example in Bariatric Surgery</td>
+<td><a href="https://www.sciencedirect.com/science/article/pii/S1098301519300737" class="external-link">Value
+in Health</a></td>
+</tr>
+<tr class="even">
+<td>Development and validation of a prognostic model predicting
+symptomatic hemorrhagic transformation in acute ischemic stroke at scale
+in the OHDSI network</td>
+<td><a href="https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0226718" class="external-link">PLoS
+One</a></td>
+</tr>
+<tr class="odd">
+<td>Wisdom of the CROUD: development and validation of a patient-level
+prediction model for opioid use disorder using population-level claims
+data</td>
+<td><a href="https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0228632" class="external-link">PLoS
+One</a></td>
+</tr>
+<tr class="even">
+<td>Developing predictive models to determine Patients in End-of-life
+Care in Administrative datasets</td>
+<td><a href="https://link.springer.com/article/10.1007/s40264-020-00906-7" class="external-link">Drug
+Safety</a></td>
+</tr>
+<tr class="odd">
+<td>Predictors of diagnostic transition from major depressive disorder
+to bipolar disorder: a retrospective observational network study</td>
+<td><a href="https://www.nature.com/articles/s41398-021-01760-6" class="external-link">Translational
+psychiatry</a></td>
+</tr>
+<tr class="even">
+<td>Seek COVER: using a disease proxy to rapidly develop and validate a
+personalized risk calculator for COVID-19 outcomes in an international
+network</td>
+<td><a href="https://link.springer.com/article/10.1186/s12874-022-01505-z" class="external-link">BMC
+Medical Research Methodology</a></td>
+</tr>
+<tr class="odd">
+<td>90-Day all-cause mortality can be predicted following a total knee
+replacement: an international, network study to develop and validate a
+prediction model</td>
+<td><a href="https://link.springer.com/article/10.1007/s00167-021-06799-y" class="external-link">Knee
+Surgery, Sports Traumatology, Arthroscopy</a></td>
+</tr>
+<tr class="even">
+<td>Machine learning and real-world data to predict lung cancer risk in
+routine care</td>
+<td><a href="https://aacrjournals.org/cebp/article-abstract/32/3/337/718495" class="external-link">Cancer
+Epidemiology, Biomarkers &amp; Prevention</a></td>
+</tr>
+<tr class="odd">
+<td>Development and validation of a patient-level model to predict
+dementia across a network of observational databases</td>
+<td><a href="https://link.springer.com/article/10.1186/s12916-024-03530-9" class="external-link">BMC
+medicine</a></td>
+</tr>
+</tbody>
+</table>
+</div>
+  </div>
+
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+
+        <nav id="toc" data-toggle="toc"><h2 data-toc-skip>Contents</h2>
+    </nav>
+</div>
+
+</div>
+
+
+
+      <footer><div class="copyright">
+  <p></p>
+<p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p>
+<p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer>
+</div>
+
+  
+
+
+  
+
+  </body>
+</html>
diff --git a/docs/articles/ConstrainedPredictors.html b/docs/articles/ConstrainedPredictors.html
new file mode 100644
index 000000000..24fa22693
--- /dev/null
+++ b/docs/articles/ConstrainedPredictors.html
@@ -0,0 +1,561 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<meta charset="utf-8">
+<meta http-equiv="X-UA-Compatible" content="IE=edge">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>Constrained predictors • PatientLevelPrediction</title>
+<!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous">
+<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css">
+<script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous">
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous">
+<!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet">
+<script src="../pkgdown.js"></script><meta property="og:title" content="Constrained predictors">
+<meta property="og:description" content="PatientLevelPrediction">
+<!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]-->
+</head>
+<body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-article">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav">
+<li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu">
+<li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul>
+</li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul>
+<ul class="nav navbar-nav navbar-right">
+<li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul>
+</div>
+<!--/.nav-collapse -->
+  </div>
+<!--/.container -->
+</div>
+<!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header toc-ignore">
+      <h1 data-toc-skip>Constrained predictors</h1>
+                        <h4 data-toc-skip class="author">Jenna Reps</h4>
+            
+            <h4 data-toc-skip class="date">2024-09-09</h4>
+      
+      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/vignettes/ConstrainedPredictors.Rmd" class="external-link"><code>vignettes/ConstrainedPredictors.Rmd</code></a></small>
+      <div class="hidden name"><code>ConstrainedPredictors.Rmd</code></div>
+
+    </div>
+
+    
+    
+<!--
+%\VignetteEngine{knitr::knitr}
+%\VignetteIndexEntry{Constrainted Predictors}
+-->
+<div class="section level2">
+<h2 id="constrained-predictors">Constrained Predictors<a class="anchor" aria-label="anchor" href="#constrained-predictors"></a>
+</h2>
+<div class="section level3">
+<h3 id="how-to-use-the-phenotypelibrary-r-package">How to use the PhenotypeLibrary R package<a class="anchor" aria-label="anchor" href="#how-to-use-the-phenotypelibrary-r-package"></a>
+</h3>
+<p>Here we provide a set of phenotypes that can be used as predictors in
+prediction models or best practice research.</p>
+<p>These phenotypes can be extracted from the PhenotypeLibrary R
+package. To install the R package run:</p>
+<div class="sourceCode" id="cb1"><pre class="downlit sourceCode r">
+<code class="sourceCode R"><span><span class="fu">remotes</span><span class="fu">::</span><span class="fu"><a href="https://remotes.r-lib.org/reference/install_github.html" class="external-link">install_github</a></span><span class="op">(</span><span class="st">'ohdsi/PhenotypeLibrary'</span><span class="op">)</span></span></code></pre></div>
+<p>To extract the cohort definition for Alcoholism with an id of 1165,
+just run:</p>
+<div class="sourceCode" id="cb2"><pre class="downlit sourceCode r">
+<code class="sourceCode R"><span><span class="fu">PhenotypeLibrary</span><span class="fu">::</span><span class="fu"><a href="https://ohdsi.github.io/PhenotypeLibrary/reference/getPlCohortDefinitionSet.html" class="external-link">getPlCohortDefinitionSet</a></span><span class="op">(</span><span class="fl">1165</span><span class="op">)</span></span></code></pre></div>
+<p>in general you can extract all the cohorts by running:</p>
+<div class="sourceCode" id="cb3"><pre class="downlit sourceCode r">
+<code class="sourceCode R"><span><span class="va">phenotypeDefinitions</span> <span class="op">&lt;-</span> <span class="fu">PhenotypeLibrary</span><span class="fu">::</span><span class="fu"><a href="https://ohdsi.github.io/PhenotypeLibrary/reference/getPlCohortDefinitionSet.html" class="external-link">getPlCohortDefinitionSet</a></span><span class="op">(</span><span class="fl">1152</span><span class="op">:</span><span class="fl">1215</span><span class="op">)</span></span></code></pre></div>
+</div>
+<div class="section level3">
+<h3 id="the-full-set-of-predictor-phenotypes">The full set of predictor phenotypes<a class="anchor" aria-label="anchor" href="#the-full-set-of-predictor-phenotypes"></a>
+</h3>
+<table class="table">
+<colgroup>
+<col width="45%">
+<col width="28%">
+<col width="26%">
+</colgroup>
+<thead><tr class="header">
+<th>Phenotype Name</th>
+<th>Disorder classification</th>
+<th>OHDSI Phenotype library ID</th>
+</tr></thead>
+<tbody>
+<tr class="odd">
+<td>Alcoholism</td>
+<td>Behavioral</td>
+<td>1165</td>
+</tr>
+<tr class="even">
+<td>Smoking</td>
+<td>Behavioral</td>
+<td>1166</td>
+</tr>
+<tr class="odd">
+<td>Anemia</td>
+<td>Blood</td>
+<td>1188</td>
+</tr>
+<tr class="even">
+<td>Osteoarthritis</td>
+<td>Bone</td>
+<td>1184</td>
+</tr>
+<tr class="odd">
+<td>Osteoporosis</td>
+<td>Bone</td>
+<td>1185</td>
+</tr>
+<tr class="even">
+<td>Cancer</td>
+<td>Cancer</td>
+<td>1215</td>
+</tr>
+<tr class="odd">
+<td>Atrial fibrillation</td>
+<td>Cardiovascular</td>
+<td>1160</td>
+</tr>
+<tr class="even">
+<td>Congestive heart failure</td>
+<td>Cardiovascular</td>
+<td>1154</td>
+</tr>
+<tr class="odd">
+<td>Coronary artery disease</td>
+<td>Cardiovascular</td>
+<td>1162</td>
+</tr>
+<tr class="even">
+<td>Heart valve disorder</td>
+<td>Cardiovascular</td>
+<td>1172</td>
+</tr>
+<tr class="odd">
+<td>Hyperlipidemia</td>
+<td>Cardiovascular</td>
+<td>1170</td>
+</tr>
+<tr class="even">
+<td>Hypertension</td>
+<td>Cardiovascular</td>
+<td>1198</td>
+</tr>
+<tr class="odd">
+<td>Angina</td>
+<td>Cardiovascular</td>
+<td>1159</td>
+</tr>
+<tr class="even">
+<td>Skin Ulcer</td>
+<td>Debility</td>
+<td>1168</td>
+</tr>
+<tr class="odd">
+<td>Diabetes type 1</td>
+<td>Endocrine</td>
+<td>1193</td>
+</tr>
+<tr class="even">
+<td>Diabetes type 2</td>
+<td>Endocrine</td>
+<td>1194</td>
+</tr>
+<tr class="odd">
+<td>Hypothyroidism</td>
+<td>Endocrine</td>
+<td>1171</td>
+</tr>
+<tr class="even">
+<td>Obesity</td>
+<td>Endocrine</td>
+<td>1179</td>
+</tr>
+<tr class="odd">
+<td>Gastroesophageal reflux disease (GERD)</td>
+<td>GI</td>
+<td>1178</td>
+</tr>
+<tr class="even">
+<td>Gastrointestinal (GI) bleed</td>
+<td>GI</td>
+<td>1197</td>
+</tr>
+<tr class="odd">
+<td>Inflammatory bowel disorder (IBD)</td>
+<td>GI/Rheumatology</td>
+<td>1180</td>
+</tr>
+<tr class="even">
+<td>Hormonal contraceptives</td>
+<td>Gynecologic</td>
+<td>1190</td>
+</tr>
+<tr class="odd">
+<td>Antibiotics Aminoglycosides</td>
+<td>Infection</td>
+<td>1201</td>
+</tr>
+<tr class="even">
+<td>Antibiotics Carbapenems</td>
+<td>Infection</td>
+<td>1202</td>
+</tr>
+<tr class="odd">
+<td>Antibiotics Cephalosporins</td>
+<td>Infection</td>
+<td>1203</td>
+</tr>
+<tr class="even">
+<td>Antibiotics Fluoroquinolones</td>
+<td>Infection</td>
+<td>1204</td>
+</tr>
+<tr class="odd">
+<td>Antibiotics Glycopeptides and lipoglycopeptides</td>
+<td>Infection</td>
+<td>1205</td>
+</tr>
+<tr class="even">
+<td>Antibiotics Macrolides</td>
+<td>Infection</td>
+<td>1206</td>
+</tr>
+<tr class="odd">
+<td>Antibiotics Monobactams</td>
+<td>Infection</td>
+<td>1207</td>
+</tr>
+<tr class="even">
+<td>Antibiotics Oxazolidinones</td>
+<td>Infection</td>
+<td>1208</td>
+</tr>
+<tr class="odd">
+<td>Antibiotics Penicillins</td>
+<td>Infection</td>
+<td>1209</td>
+</tr>
+<tr class="even">
+<td>Antibiotics Polypeptides</td>
+<td>Infection</td>
+<td>1210</td>
+</tr>
+<tr class="odd">
+<td>Antibiotics Rifamycins</td>
+<td>Infection</td>
+<td>1211</td>
+</tr>
+<tr class="even">
+<td>Antibiotics Sulfonamides</td>
+<td>Infection</td>
+<td>1212</td>
+</tr>
+<tr class="odd">
+<td>Antibiotics Streptogramins</td>
+<td>Infection</td>
+<td>1213</td>
+</tr>
+<tr class="even">
+<td>Antibiotics Tetracyclines</td>
+<td>Infection</td>
+<td>1214</td>
+</tr>
+<tr class="odd">
+<td>Pneumonia</td>
+<td>Infection/Respiratory</td>
+<td>1199</td>
+</tr>
+<tr class="even">
+<td>Sepsis</td>
+<td>Infection</td>
+<td>1176</td>
+</tr>
+<tr class="odd">
+<td>Urinary tract infection (UTI)</td>
+<td>Infection</td>
+<td>1186</td>
+</tr>
+<tr class="even">
+<td>Hepatitis</td>
+<td>Liver</td>
+<td>1169</td>
+</tr>
+<tr class="odd">
+<td>Anxiety</td>
+<td>Mood</td>
+<td>1189</td>
+</tr>
+<tr class="even">
+<td>Depression (MDD)</td>
+<td>Mood</td>
+<td>1161</td>
+</tr>
+<tr class="odd">
+<td>Psychotic disorder</td>
+<td>Mood</td>
+<td>1175</td>
+</tr>
+<tr class="even">
+<td>Antiepileptics (pain)</td>
+<td>Neurology/Pain</td>
+<td>1183</td>
+</tr>
+<tr class="odd">
+<td>Seizure</td>
+<td>Neurology</td>
+<td>1153</td>
+</tr>
+<tr class="even">
+<td>Hemorrhagic stroke</td>
+<td>Neurology/Vascular</td>
+<td>1156</td>
+</tr>
+<tr class="odd">
+<td>Non-hemorrhagic stroke</td>
+<td>Neurology/Vascular</td>
+<td>1155</td>
+</tr>
+<tr class="even">
+<td>Acetaminophen prescription</td>
+<td>Pain/Infection</td>
+<td>1187</td>
+</tr>
+<tr class="odd">
+<td>Low back pain</td>
+<td>Pain</td>
+<td>1173</td>
+</tr>
+<tr class="even">
+<td>Neuropathy</td>
+<td>Pain/Neurology</td>
+<td>1174</td>
+</tr>
+<tr class="odd">
+<td>Opioids</td>
+<td>Pain</td>
+<td>1182</td>
+</tr>
+<tr class="even">
+<td>Acute kidney injury</td>
+<td>Kidney</td>
+<td>1163</td>
+</tr>
+<tr class="odd">
+<td>Chronic kidney disease</td>
+<td>Kidney</td>
+<td>1191</td>
+</tr>
+<tr class="even">
+<td>Asthma</td>
+<td>Respiratory</td>
+<td>1164</td>
+</tr>
+<tr class="odd">
+<td>Chronic obstructive pulmonary disorder (COPD)</td>
+<td>Respiratory</td>
+<td>1192</td>
+</tr>
+<tr class="even">
+<td>Dyspnea</td>
+<td>Respiratory</td>
+<td>1195</td>
+</tr>
+<tr class="odd">
+<td>Respiratory failure</td>
+<td>Respiratory</td>
+<td>1177</td>
+</tr>
+<tr class="even">
+<td>Sleep apnea</td>
+<td>Respiratory</td>
+<td>1167</td>
+</tr>
+<tr class="odd">
+<td>Rheumatoid arthritis</td>
+<td>Rheumatology</td>
+<td>1200</td>
+</tr>
+<tr class="even">
+<td>Steroids</td>
+<td>Rheumatology/Pain/Pulmonary</td>
+<td>1181</td>
+</tr>
+<tr class="odd">
+<td>Peripheral vascular disease</td>
+<td>Vascular</td>
+<td>1157</td>
+</tr>
+<tr class="even">
+<td>Aspirin</td>
+<td>Vascular</td>
+<td>1158</td>
+</tr>
+<tr class="odd">
+<td>Deep vein thrombosis (DVT)</td>
+<td>Vascular</td>
+<td>1152</td>
+</tr>
+<tr class="even">
+<td>Edema</td>
+<td>Vascular</td>
+<td>1196</td>
+</tr>
+<tr class="odd">
+<td>Inpatient visit</td>
+<td>NA</td>
+<td>NA</td>
+</tr>
+</tbody>
+</table>
+</div>
+</div>
+  </div>
+
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+
+        <nav id="toc" data-toggle="toc"><h2 data-toc-skip>Contents</h2>
+    </nav>
+</div>
+
+</div>
+
+
+
+      <footer><div class="copyright">
+  <p></p>
+<p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p>
+<p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer>
+</div>
+
+  
+
+
+  
+
+  </body>
+</html>
diff --git a/docs/articles/CreatingLearningCurves.html b/docs/articles/CreatingLearningCurves.html
index 739749046..2f44ca17f 100644
--- a/docs/articles/CreatingLearningCurves.html
+++ b/docs/articles/CreatingLearningCurves.html
@@ -19,6 +19,8 @@
 <![endif]-->
 </head>
 <body data-spy="scroll" data-target="#toc">
+    
+
     <div class="container template-article">
       <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
@@ -31,7 +33,7 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
@@ -53,7 +55,7 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
@@ -72,10 +74,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -84,13 +86,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -101,21 +106,30 @@
   </ul>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
       </ul>
 <ul class="nav navbar-nav navbar-right">
 <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
@@ -130,15 +144,16 @@
 
       
 
-      </header><script src="CreatingLearningCurves_files/header-attrs-2.11/header-attrs.js"></script><div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header toc-ignore">
       <h1 data-toc-skip>Creating Learning Curves</h1>
-                        <h4 class="author">Luis H. John, Jenna M. Reps, Peter R. Rijnbeek</h4>
+                        <h4 data-toc-skip class="author">Luis H. John,
+Jenna M. Reps, Peter R. Rijnbeek</h4>
             
-            <h4 class="date">2022-03-09</h4>
+            <h4 data-toc-skip class="date">2024-09-09</h4>
       
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/CreatingLearningCurves.Rmd"><code>vignettes/CreatingLearningCurves.Rmd</code></a></small>
+      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/vignettes/CreatingLearningCurves.Rmd" class="external-link"><code>vignettes/CreatingLearningCurves.Rmd</code></a></small>
       <div class="hidden name"><code>CreatingLearningCurves.Rmd</code></div>
 
     </div>
@@ -146,162 +161,243 @@ <h4 class="date">2022-03-09</h4>
     
     
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Creating Learning Curves}
 -->
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
-<p>This vignette describes how you can use the Observational Health Data Sciences and Informatics (OHDSI) <a href="http://github.com/OHDSI/PatientLevelPrediction"><code>PatientLevelPrediction</code></a> package to create learning curves. This vignette assumes you have read and are comfortable with building patient level prediction models as described in the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf"><code>BuildingPredictiveModels</code> vignette</a>.</p>
-<p>Prediction models will show overly-optimistic performance when predicting on the same data as used for training. Therefore, best-practice is to partition our data into a training set and testing set. We then train our prediction model on the training set portion and asses its ability to generalize to unseen data by measuring its performance on the testing set.</p>
-<p>Learning curves assess the effect of training set size on model performance by training a sequence of prediction models on successively larger subsets of the training set. A learning curve plot can also help in diagnosing a bias or variance problem as explained below.</p>
-<div class="figure">
-<img src="learningCurve.png" alt=""><p class="caption">Learning curve example.</p>
+<div class="section level2">
+<h2 id="introduction">Introduction<a class="anchor" aria-label="anchor" href="#introduction"></a>
+</h2>
+<p>This vignette describes how you can use the Observational Health Data
+Sciences and Informatics (OHDSI) <a href="http://github.com/OHDSI/PatientLevelPrediction" class="external-link"><code>PatientLevelPrediction</code></a>
+package to create learning curves. This vignette assumes you have read
+and are comfortable with building patient level prediction models as
+described in the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingPredictiveModels.pdf" class="external-link"><code>BuildingPredictiveModels</code>
+vignette</a>.</p>
+<p>Prediction models will show overly-optimistic performance when
+predicting on the same data as used for training. Therefore,
+best-practice is to partition our data into a training set and testing
+set. We then train our prediction model on the training set portion and
+asses its ability to generalize to unseen data by measuring its
+performance on the testing set.</p>
+<p>Learning curves assess the effect of training set size on model
+performance by training a sequence of prediction models on successively
+larger subsets of the training set. A learning curve plot can also help
+in diagnosing a bias or variance problem as explained below.</p>
+<div class="float">
+<img src="learningCurve.png" alt="Learning curve example."><div class="figcaption">Learning curve example.</div>
 </div>
-<p>Figure 1, shows an example of learning curve plot in which the vertical axis represents the model performance and the horizontal axis the training set size. If training set size is small, the performance on the training set is high, because a model can often be fitted well to a limited number of training examples. At the same time, the performance on the testing set will be poor, because the model trained on such a limited number of training examples will not generalize well to unseen data in the testing set. As the training set size increases, the performance of the model on the training set will decrease. It becomes more difficult for the model to find a good fit through all the training examples. Also, the model will be trained on a more representative portion of training examples, making it generalize better to unseen data. This can be observed by the increasin testing set performance.</p>
-<p>The learning curve can help us in diagnosing bias and variance problems with our classifier which will provide guidance on how to further improve our model. We can observe high variance (overfitting) in a prediction model if it performs well on the training set, but poorly on the testing set (Figure 2). Adding additional data is a common approach to counteract high variance. From the learning curve it becomes apparent, that adding additional data may improve performance on the testing set a little further, as the learning curve has not yet plateaued and, thus, the model is not saturated yet. Therefore, adding more data will decrease the gap between training set and testing set, which is the main indicator for a high variance problem.</p>
-<div class="figure">
-<img src="learningCurveVariance.png" alt=""><p class="caption">Prediction model suffering from high variance.</p>
+<p>Figure 1, shows an example of learning curve plot in which the
+vertical axis represents the model performance and the horizontal axis
+the training set size. If training set size is small, the performance on
+the training set is high, because a model can often be fitted well to a
+limited number of training examples. At the same time, the performance
+on the testing set will be poor, because the model trained on such a
+limited number of training examples will not generalize well to unseen
+data in the testing set. As the training set size increases, the
+performance of the model on the training set will decrease. It becomes
+more difficult for the model to find a good fit through all the training
+examples. Also, the model will be trained on a more representative
+portion of training examples, making it generalize better to unseen
+data. This can be observed by the increasin testing set performance.</p>
+<p>The learning curve can help us in diagnosing bias and variance
+problems with our classifier which will provide guidance on how to
+further improve our model. We can observe high variance (overfitting) in
+a prediction model if it performs well on the training set, but poorly
+on the testing set (Figure 2). Adding additional data is a common
+approach to counteract high variance. From the learning curve it becomes
+apparent, that adding additional data may improve performance on the
+testing set a little further, as the learning curve has not yet
+plateaued and, thus, the model is not saturated yet. Therefore, adding
+more data will decrease the gap between training set and testing set,
+which is the main indicator for a high variance problem.</p>
+<div class="float">
+<img src="learningCurveVariance.png" alt="Prediction model suffering from high variance."><div class="figcaption">Prediction model suffering from high
+variance.</div>
 </div>
-<p>Furthermore, we can observe high bias (underfitting) if a prediction model performs poorly on the training set as well as on the testing set (Figure 3). The learning curves of training set and testing set have flattened on a low performance with only a small gap in between them. Adding additional data will in this case have little to no impact on the model performance. Choosing another prediction algorithm that can find more complex (for example non-linear) relationships in the data may be an alternative approach to consider in this high bias situation.</p>
-<div class="figure">
-<img src="learningCurveBias.png" alt=""><p class="caption">Prediction model suffering from high bias.</p>
+<p>Furthermore, we can observe high bias (underfitting) if a prediction
+model performs poorly on the training set as well as on the testing set
+(Figure 3). The learning curves of training set and testing set have
+flattened on a low performance with only a small gap in between them.
+Adding additional data will in this case have little to no impact on the
+model performance. Choosing another prediction algorithm that can find
+more complex (for example non-linear) relationships in the data may be
+an alternative approach to consider in this high bias situation.</p>
+<div class="float">
+<img src="learningCurveBias.png" alt="Prediction model suffering from high bias."><div class="figcaption">Prediction model suffering from high bias.</div>
 </div>
 </div>
-<div id="creating-the-learning-curve" class="section level1">
-<h1 class="hasAnchor">
-<a href="#creating-the-learning-curve" class="anchor"></a>Creating the learning curve</h1>
-<p>Use the <a href="http://github.com/OHDSI/PatientLevelPrediction"><code>PatientLevelPrediction</code></a> package to create a <code>plpData</code> object . Alternatively, you can make use of the data simulator. The following code snippet creates data for 12000 patients.</p>
+<div class="section level2">
+<h2 id="creating-the-learning-curve">Creating the learning curve<a class="anchor" aria-label="anchor" href="#creating-the-learning-curve"></a>
+</h2>
+<p>Use the <a href="http://github.com/OHDSI/PatientLevelPrediction" class="external-link"><code>PatientLevelPrediction</code></a>
+package to create a <code>plpData</code> object . Alternatively, you can
+make use of the data simulator. The following code snippet creates data
+for 12000 patients.</p>
 <div class="sourceCode" id="cb1"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/base/Random.html">set.seed</a></span><span class="op">(</span><span class="fl">1234</span><span class="op">)</span>
-<span class="fu"><a href="https://rdrr.io/r/utils/data.html">data</a></span><span class="op">(</span><span class="va">plpDataSimulationProfile</span><span class="op">)</span>
-<span class="va">sampleSize</span> <span class="op">&lt;-</span> <span class="fl">12000</span>
-<span class="va">plpData</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/simulatePlpData.html">simulatePlpData</a></span><span class="op">(</span>
-  <span class="va">plpDataSimulationProfile</span>,
-  n <span class="op">=</span> <span class="va">sampleSize</span>
-<span class="op">)</span></code></pre></div>
-<p>Specify the population settings (this does additional exclusions such as requiring minimum prior observation or no prior outcome as well as specifying the time-at-risk period to enable labels to be created):</p>
+<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/base/Random.html" class="external-link">set.seed</a></span><span class="op">(</span><span class="fl">1234</span><span class="op">)</span></span>
+<span><span class="fu"><a href="https://rdrr.io/r/utils/data.html" class="external-link">data</a></span><span class="op">(</span><span class="va">plpDataSimulationProfile</span><span class="op">)</span></span>
+<span><span class="va">sampleSize</span> <span class="op">&lt;-</span> <span class="fl">12000</span></span>
+<span><span class="va">plpData</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/simulatePlpData.html">simulatePlpData</a></span><span class="op">(</span></span>
+<span>  <span class="va">plpDataSimulationProfile</span>,</span>
+<span>  n <span class="op">=</span> <span class="va">sampleSize</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+<p>Specify the population settings (this does additional exclusions such
+as requiring minimum prior observation or no prior outcome as well as
+specifying the time-at-risk period to enable labels to be created):</p>
 <div class="sourceCode" id="cb2"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">populationSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createStudyPopulationSettings.html">createStudyPopulationSettings</a></span><span class="op">(</span>
-  binary <span class="op">=</span> <span class="cn">TRUE</span>,
-  firstExposureOnly <span class="op">=</span> <span class="cn">FALSE</span>,
-  washoutPeriod <span class="op">=</span> <span class="fl">0</span>,
-  removeSubjectsWithPriorOutcome <span class="op">=</span> <span class="cn">FALSE</span>,
-  priorOutcomeLookback <span class="op">=</span> <span class="fl">99999</span>,
-  requireTimeAtRisk <span class="op">=</span> <span class="cn">FALSE</span>,
-  minTimeAtRisk <span class="op">=</span> <span class="fl">0</span>,
-  riskWindowStart <span class="op">=</span> <span class="fl">0</span>,
-  riskWindowEnd <span class="op">=</span> <span class="fl">365</span>,
-  verbosity <span class="op">=</span> <span class="st">"INFO"</span>
-<span class="op">)</span></code></pre></div>
+<code class="sourceCode R"><span><span class="va">populationSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createStudyPopulationSettings.html">createStudyPopulationSettings</a></span><span class="op">(</span></span>
+<span>  binary <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>  firstExposureOnly <span class="op">=</span> <span class="cn">FALSE</span>,</span>
+<span>  washoutPeriod <span class="op">=</span> <span class="fl">0</span>,</span>
+<span>  removeSubjectsWithPriorOutcome <span class="op">=</span> <span class="cn">FALSE</span>,</span>
+<span>  priorOutcomeLookback <span class="op">=</span> <span class="fl">99999</span>,</span>
+<span>  requireTimeAtRisk <span class="op">=</span> <span class="cn">FALSE</span>,</span>
+<span>  minTimeAtRisk <span class="op">=</span> <span class="fl">0</span>,</span>
+<span>  riskWindowStart <span class="op">=</span> <span class="fl">0</span>,</span>
+<span>  riskWindowEnd <span class="op">=</span> <span class="fl">365</span>,</span>
+<span>  verbosity <span class="op">=</span> <span class="st">"INFO"</span></span>
+<span><span class="op">)</span></span></code></pre></div>
 <p>Specify the prediction algorithm to be used.</p>
 <div class="sourceCode" id="cb3"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="co"># Use LASSO logistic regression</span>
-<span class="va">modelSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span></code></pre></div>
-<p>Specify the split settings and a sequence of training set fractions (these over ride the splitSetting trainFraction). Alternatively, instead of <code>trainFractions</code>, you can provide a sequence of training events (<code>trainEvents</code>) instead of the training set fractions. This is recommended, because our research has shown that number of events is the important determinant of model performance. Make sure that your training set contains the number of events specified.</p>
+<code class="sourceCode R"><span><span class="co"># Use LASSO logistic regression</span></span>
+<span><span class="va">modelSettings</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span></span></code></pre></div>
+<p>Specify the split settings and a sequence of training set fractions
+(these over ride the splitSetting trainFraction). Alternatively, instead
+of <code>trainFractions</code>, you can provide a sequence of training
+events (<code>trainEvents</code>) instead of the training set fractions.
+This is recommended, because our research has shown that number of
+events is the important determinant of model performance. Make sure that
+your training set contains the number of events specified.</p>
 <div class="sourceCode" id="cb4"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">splitSettings</span> <span class="op">=</span> <span class="fu"><a href="../reference/createDefaultSplitSetting.html">createDefaultSplitSetting</a></span><span class="op">(</span>
-  testFraction <span class="op">=</span> <span class="fl">0.2</span>,  
-  type <span class="op">=</span> <span class="st">'stratified'</span>,
-  splitSeed <span class="op">=</span> <span class="fl">1000</span>
-  <span class="op">)</span>
-
-<span class="va">trainFractions</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/seq.html">seq</a></span><span class="op">(</span><span class="fl">0.1</span>, <span class="fl">0.8</span>, <span class="fl">0.1</span><span class="op">)</span> <span class="co"># Create eight training set fractions</span>
-
-<span class="co"># alternatively use a sequence of training events by uncommenting the line below.</span>
-<span class="co"># trainEvents &lt;- seq(100, 5000, 100)</span></code></pre></div>
+<code class="sourceCode R"><span><span class="va">splitSettings</span> <span class="op">=</span> <span class="fu"><a href="../reference/createDefaultSplitSetting.html">createDefaultSplitSetting</a></span><span class="op">(</span></span>
+<span>  testFraction <span class="op">=</span> <span class="fl">0.2</span>,  </span>
+<span>  type <span class="op">=</span> <span class="st">'stratified'</span>,</span>
+<span>  splitSeed <span class="op">=</span> <span class="fl">1000</span></span>
+<span>  <span class="op">)</span></span>
+<span></span>
+<span><span class="va">trainFractions</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/seq.html" class="external-link">seq</a></span><span class="op">(</span><span class="fl">0.1</span>, <span class="fl">0.8</span>, <span class="fl">0.1</span><span class="op">)</span> <span class="co"># Create eight training set fractions</span></span>
+<span></span>
+<span><span class="co"># alternatively use a sequence of training events by uncommenting the line below.</span></span>
+<span><span class="co"># trainEvents &lt;- seq(100, 5000, 100)</span></span></code></pre></div>
 <p>Create the learning curve object.</p>
 <div class="sourceCode" id="cb5"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">learningCurve</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createLearningCurve.html">createLearningCurve</a></span><span class="op">(</span>
-  plpData <span class="op">=</span> <span class="va">plpData</span>,
-  outcomeId <span class="op">=</span> <span class="fl">2</span>,  
-  parallel <span class="op">=</span> <span class="cn">T</span>,
-  cores <span class="op">=</span> <span class="fl">4</span>,
-  modelSettings <span class="op">=</span> <span class="va">modelSettings</span>,
-  saveDirectory <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/getwd.html">getwd</a></span><span class="op">(</span><span class="op">)</span>,
-  analysisId <span class="op">=</span> <span class="st">'learningCurve'</span>,
-  populationSettings <span class="op">=</span> <span class="va">populationSettings</span>,
-  splitSettings <span class="op">=</span> <span class="va">splitSettings</span>,
-  trainFractions <span class="op">=</span> <span class="va">trainFractions</span>,
-  trainEvents <span class="op">=</span> <span class="cn">NULL</span>,
-  preprocessSettings <span class="op">=</span> <span class="fu"><a href="../reference/createPreprocessSettings.html">createPreprocessSettings</a></span><span class="op">(</span>
-    minFraction <span class="op">=</span> <span class="fl">0.001</span>,
-    normalize <span class="op">=</span> <span class="cn">T</span>
-  <span class="op">)</span>,
-  executeSettings <span class="op">=</span> <span class="fu"><a href="../reference/createExecuteSettings.html">createExecuteSettings</a></span><span class="op">(</span>
-    runSplitData <span class="op">=</span> <span class="cn">T</span>, 
-    runSampleData <span class="op">=</span> <span class="cn">F</span>,
-    runfeatureEngineering <span class="op">=</span> <span class="cn">F</span>,
-    runPreprocessData <span class="op">=</span> <span class="cn">T</span>,
-    runModelDevelopment <span class="op">=</span> <span class="cn">T</span>,
-    runCovariateSummary <span class="op">=</span> <span class="cn">F</span>
-    <span class="op">)</span>
-<span class="op">)</span></code></pre></div>
-<p>Plot the learning curve object (Figure 4). Specify one of the available metrics: <code>AUROC</code>, <code>AUPRC</code>, <code>sBrier</code>. Moreover, you can specify what metric to put on the abscissa, number of <code>observations</code> or number of <code>events</code>. We recommend the latter, because <code>events</code> are determinant of model performance and allow you to better compare learning curves across different prediction problems and databases.</p>
+<code class="sourceCode R"><span><span class="va">learningCurve</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createLearningCurve.html">createLearningCurve</a></span><span class="op">(</span></span>
+<span>  plpData <span class="op">=</span> <span class="va">plpData</span>,</span>
+<span>  outcomeId <span class="op">=</span> <span class="fl">2</span>,  </span>
+<span>  parallel <span class="op">=</span> <span class="cn">T</span>,</span>
+<span>  cores <span class="op">=</span> <span class="fl">4</span>,</span>
+<span>  modelSettings <span class="op">=</span> <span class="va">modelSettings</span>,</span>
+<span>  saveDirectory <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  analysisId <span class="op">=</span> <span class="st">'learningCurve'</span>,</span>
+<span>  populationSettings <span class="op">=</span> <span class="va">populationSettings</span>,</span>
+<span>  splitSettings <span class="op">=</span> <span class="va">splitSettings</span>,</span>
+<span>  trainFractions <span class="op">=</span> <span class="va">trainFractions</span>,</span>
+<span>  trainEvents <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  preprocessSettings <span class="op">=</span> <span class="fu"><a href="../reference/createPreprocessSettings.html">createPreprocessSettings</a></span><span class="op">(</span></span>
+<span>    minFraction <span class="op">=</span> <span class="fl">0.001</span>,</span>
+<span>    normalize <span class="op">=</span> <span class="cn">T</span></span>
+<span>  <span class="op">)</span>,</span>
+<span>  executeSettings <span class="op">=</span> <span class="fu"><a href="../reference/createExecuteSettings.html">createExecuteSettings</a></span><span class="op">(</span></span>
+<span>    runSplitData <span class="op">=</span> <span class="cn">T</span>, </span>
+<span>    runSampleData <span class="op">=</span> <span class="cn">F</span>,</span>
+<span>    runfeatureEngineering <span class="op">=</span> <span class="cn">F</span>,</span>
+<span>    runPreprocessData <span class="op">=</span> <span class="cn">T</span>,</span>
+<span>    runModelDevelopment <span class="op">=</span> <span class="cn">T</span>,</span>
+<span>    runCovariateSummary <span class="op">=</span> <span class="cn">F</span></span>
+<span>    <span class="op">)</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+<p>Plot the learning curve object (Figure 4). Specify one of the
+available metrics: <code>AUROC</code>, <code>AUPRC</code>,
+<code>sBrier</code>. Moreover, you can specify what metric to put on the
+abscissa, number of <code>observations</code> or number of
+<code>events</code>. We recommend the latter, because
+<code>events</code> are determinant of model performance and allow you
+to better compare learning curves across different prediction problems
+and databases.</p>
 <div class="sourceCode" id="cb6"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="../reference/plotLearningCurve.html">plotLearningCurve</a></span><span class="op">(</span>
-  <span class="va">learningCurve</span>,
-  metric <span class="op">=</span> <span class="st">'AUROC'</span>,
-  abscissa <span class="op">=</span> <span class="st">'events'</span>,
-  plotTitle <span class="op">=</span> <span class="st">'Learning Curve'</span>,
-  plotSubtitle <span class="op">=</span> <span class="st">'AUROC performance'</span>
-<span class="op">)</span></code></pre></div>
-<div class="figure">
-<img src="learningCurvePlot.png" alt=""><p class="caption">Learning curve plot.</p>
+<code class="sourceCode R"><span><span class="fu"><a href="../reference/plotLearningCurve.html">plotLearningCurve</a></span><span class="op">(</span></span>
+<span>  <span class="va">learningCurve</span>,</span>
+<span>  metric <span class="op">=</span> <span class="st">'AUROC'</span>,</span>
+<span>  abscissa <span class="op">=</span> <span class="st">'events'</span>,</span>
+<span>  plotTitle <span class="op">=</span> <span class="st">'Learning Curve'</span>,</span>
+<span>  plotSubtitle <span class="op">=</span> <span class="st">'AUROC performance'</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+<div class="float">
+<img src="learningCurvePlot.png" alt="Learning curve plot."><div class="figcaption">Learning curve plot.</div>
 </div>
 </div>
-<div id="parallel-processing" class="section level1">
-<h1 class="hasAnchor">
-<a href="#parallel-processing" class="anchor"></a>Parallel processing</h1>
-<p>The learning curve object can be created in parallel, which can reduce computation time significantly. Whether to run the code in parallel or not is specified using the <code>parallel</code> input. Currently this functionality is only available for LASSO logistic regression and gradient boosting machines. Depending on the number of parallel workers it may require a significant amount of memory. We advise to use the parallelized learning curve function for parameter search and exploratory data analysis.</p>
-<p>When running in parrallel, R will find the number of available processing cores automatically and register the required parallel backend. Alternatively, you can provide the number of cores you wish to use via the <code>cores</code> input.</p>
+<div class="section level2">
+<h2 id="parallel-processing">Parallel processing<a class="anchor" aria-label="anchor" href="#parallel-processing"></a>
+</h2>
+<p>The learning curve object can be created in parallel, which can
+reduce computation time significantly. Whether to run the code in
+parallel or not is specified using the <code>parallel</code> input.
+Currently this functionality is only available for LASSO logistic
+regression and gradient boosting machines. Depending on the number of
+parallel workers it may require a significant amount of memory. We
+advise to use the parallelized learning curve function for parameter
+search and exploratory data analysis.</p>
+<p>When running in parrallel, R will find the number of available
+processing cores automatically and register the required parallel
+backend. Alternatively, you can provide the number of cores you wish to
+use via the <code>cores</code> input.</p>
 </div>
-<div id="demo" class="section level1">
-<h1 class="hasAnchor">
-<a href="#demo" class="anchor"></a>Demo</h1>
+<div class="section level2">
+<h2 id="demo">Demo<a class="anchor" aria-label="anchor" href="#demo"></a>
+</h2>
 <p>We have added a demo of the learningcurve:</p>
 <div class="sourceCode" id="cb7"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="co"># Show all demos in our package: </span>
- <span class="fu"><a href="https://rdrr.io/r/utils/demo.html">demo</a></span><span class="op">(</span>package <span class="op">=</span> <span class="st">"PatientLevelPrediction"</span><span class="op">)</span>
-
-<span class="co"># Run the learning curve</span>
- <span class="fu"><a href="https://rdrr.io/r/utils/demo.html">demo</a></span><span class="op">(</span><span class="st">"LearningCurveDemo"</span>, package <span class="op">=</span> <span class="st">"PatientLevelPrediction"</span><span class="op">)</span></code></pre></div>
-<p>Do note that running this demo can take a considerable amount of time (15 min on Quad core running in parallel)!</p>
+<code class="sourceCode R"><span><span class="co"># Show all demos in our package: </span></span>
+<span> <span class="fu"><a href="https://rdrr.io/r/utils/demo.html" class="external-link">demo</a></span><span class="op">(</span>package <span class="op">=</span> <span class="st">"PatientLevelPrediction"</span><span class="op">)</span></span>
+<span></span>
+<span><span class="co"># Run the learning curve</span></span>
+<span> <span class="fu"><a href="https://rdrr.io/r/utils/demo.html" class="external-link">demo</a></span><span class="op">(</span><span class="st">"LearningCurveDemo"</span>, package <span class="op">=</span> <span class="st">"PatientLevelPrediction"</span><span class="op">)</span></span></code></pre></div>
+<p>Do note that running this demo can take a considerable amount of time
+(15 min on Quad core running in parallel)!</p>
 </div>
-<div id="publication" class="section level1">
-<h1 class="hasAnchor">
-<a href="#publication" class="anchor"></a>Publication</h1>
-<p>A publication titled ‘How little data do we need for patient-level prediction?’ uses the learning curve functionality in this package and can be accessed as preprint in the arXiv archives at <a href="https://arxiv.org/abs/2008.07361">https://arxiv.org/abs/2008.07361</a>.</p>
+<div class="section level2">
+<h2 id="publication">Publication<a class="anchor" aria-label="anchor" href="#publication"></a>
+</h2>
+<p>A publication titled ‘How little data do we need for patient-level
+prediction?’ uses the learning curve functionality in this package and
+can be accessed as preprint in the arXiv archives at <a href="https://arxiv.org/abs/2008.07361" class="external-link uri">https://arxiv.org/abs/2008.07361</a>.</p>
 </div>
-<div id="acknowledgments" class="section level1">
-<h1 class="hasAnchor">
-<a href="#acknowledgments" class="anchor"></a>Acknowledgments</h1>
-<p>Considerable work has been dedicated to provide the <code>PatientLevelPrediction</code> package.</p>
+<div class="section level2">
+<h2 id="acknowledgments">Acknowledgments<a class="anchor" aria-label="anchor" href="#acknowledgments"></a>
+</h2>
+<p>Considerable work has been dedicated to provide the
+<code>PatientLevelPrediction</code> package.</p>
 <div class="sourceCode" id="cb8"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/utils/citation.html">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></code></pre></div>
-<pre><code>## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design
-## and implementation of a standardized framework to generate and evaluate
-## patient-level prediction models using observational healthcare data."
-## _Journal of the American Medical Informatics Association_, *25*(8),
-## 969-975. &lt;URL: https://doi.org/10.1093/jamia/ocy032&gt;.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }</code></pre>
-<p><strong>Please reference this paper if you use the PLP Package in your work:</strong></p>
-<p><a href="http://dx.doi.org/10.1093/jamia/ocy032">Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.</a></p>
+<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/utils/citation.html" class="external-link">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></span></code></pre></div>
+<pre><code><span><span class="co">## </span></span>
+<span><span class="co">## To cite PatientLevelPrediction in publications use:</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">##   Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design</span></span>
+<span><span class="co">##   and implementation of a standardized framework to generate and</span></span>
+<span><span class="co">##   evaluate patient-level prediction models using observational</span></span>
+<span><span class="co">##   healthcare data." _Journal of the American Medical Informatics</span></span>
+<span><span class="co">##   Association_, *25*(8), 969-975.</span></span>
+<span><span class="co">##   &lt;https://doi.org/10.1093/jamia/ocy032&gt;.</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">## A BibTeX entry for LaTeX users is</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">##   @Article{,</span></span>
+<span><span class="co">##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},</span></span>
+<span><span class="co">##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},</span></span>
+<span><span class="co">##     journal = {Journal of the American Medical Informatics Association},</span></span>
+<span><span class="co">##     volume = {25},</span></span>
+<span><span class="co">##     number = {8},</span></span>
+<span><span class="co">##     pages = {969-975},</span></span>
+<span><span class="co">##     year = {2018},</span></span>
+<span><span class="co">##     url = {https://doi.org/10.1093/jamia/ocy032},</span></span>
+<span><span class="co">##   }</span></span></code></pre>
+<p><strong>Please reference this paper if you use the PLP Package in
+your work:</strong></p>
+<p><a href="http://dx.doi.org/10.1093/jamia/ocy032" class="external-link">Reps JM, Schuemie
+MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a
+standardized framework to generate and evaluate patient-level prediction
+models using observational healthcare data. J Am Med Inform Assoc.
+2018;25(8):969-975.</a></p>
 </div>
   </div>
 
@@ -316,11 +412,13 @@ <h1 class="hasAnchor">
 
 
       <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+  <p></p>
+<p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p>
+<p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
       </footer>
@@ -329,5 +427,7 @@ <h1 class="hasAnchor">
   
 
 
+  
+
   </body>
 </html>
diff --git a/docs/articles/CreatingLearningCurves_files/header-attrs-2.11/header-attrs.js b/docs/articles/CreatingLearningCurves_files/header-attrs-2.11/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/CreatingLearningCurves_files/header-attrs-2.11/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/CreatingLearningCurves_files/header-attrs-2.7/header-attrs.js b/docs/articles/CreatingLearningCurves_files/header-attrs-2.7/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/CreatingLearningCurves_files/header-attrs-2.7/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/CreatingNetworkStudies.html b/docs/articles/CreatingNetworkStudies.html
index c5febb13c..bcf3648fa 100644
--- a/docs/articles/CreatingNetworkStudies.html
+++ b/docs/articles/CreatingNetworkStudies.html
@@ -19,6 +19,8 @@
 <![endif]-->
 </head>
 <body data-spy="scroll" data-target="#toc">
+    
+
     <div class="container template-article">
       <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
@@ -31,7 +33,7 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
@@ -53,7 +55,7 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
@@ -72,10 +74,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -84,13 +86,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -101,21 +106,30 @@
   </ul>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
       </ul>
 <ul class="nav navbar-nav navbar-right">
 <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
@@ -130,15 +144,17 @@
 
       
 
-      </header><script src="CreatingNetworkStudies_files/header-attrs-2.11/header-attrs.js"></script><div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header toc-ignore">
-      <h1 data-toc-skip>Making patient-level predictive network study packages</h1>
-                        <h4 class="author">Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek</h4>
+      <h1 data-toc-skip>Making patient-level predictive network study
+packages</h1>
+                        <h4 data-toc-skip class="author">Jenna Reps,
+Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek</h4>
             
-            <h4 class="date">2022-03-09</h4>
+            <h4 data-toc-skip class="date">2024-09-09</h4>
       
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/CreatingNetworkStudies.Rmd"><code>vignettes/CreatingNetworkStudies.Rmd</code></a></small>
+      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/vignettes/CreatingNetworkStudies.Rmd" class="external-link"><code>vignettes/CreatingNetworkStudies.Rmd</code></a></small>
       <div class="hidden name"><code>CreatingNetworkStudies.Rmd</code></div>
 
     </div>
@@ -146,68 +162,106 @@ <h4 class="date">2022-03-09</h4>
     
     
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Making  patient-level predictive network study packages}
 -->
 <div style="page-break-after: always;"></div>
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
-<p>The OHDSI Patient Level Prediction (PLP) package provides the framework to implement prediction models at scale. This can range from developing a large number of models across sites (methodology and study design insight) to extensive external validation of existing models in the OHDSI PLP framework (model insight). This vignette describes how you can use the <code>PatientLevelPrediction</code> package to create a network study package.</p>
+<div class="section level2">
+<h2 id="introduction">Introduction<a class="anchor" aria-label="anchor" href="#introduction"></a>
+</h2>
+<p>The OHDSI Patient Level Prediction (PLP) package provides the
+framework to implement prediction models at scale. This can range from
+developing a large number of models across sites (methodology and study
+design insight) to extensive external validation of existing models in
+the OHDSI PLP framework (model insight). This vignette describes how you
+can use the <code>PatientLevelPrediction</code> package to create a
+network study package.</p>
 </div>
-<div id="useful-publication" class="section level1">
-<h1 class="hasAnchor">
-<a href="#useful-publication" class="anchor"></a>Useful publication</h1>
-<p>The open access publication <a href="https://pubmed.ncbi.nlm.nih.gov/34560604/">A standardized analytics pipeline for reliable and rapid development and validation of prediction models using observational health data</a> details the process used to develop and validate prediction models using the OHDSI prediction framework and tools. This publication describes each of the steps and then demonstrates these by focusing on predicting death in those who have covid-19.</p>
+<div class="section level2">
+<h2 id="useful-publication">Useful publication<a class="anchor" aria-label="anchor" href="#useful-publication"></a>
+</h2>
+<p>The open access publication <a href="https://pubmed.ncbi.nlm.nih.gov/34560604/" class="external-link">A standardized
+analytics pipeline for reliable and rapid development and validation of
+prediction models using observational health data</a> details the
+process used to develop and validate prediction models using the OHDSI
+prediction framework and tools. This publication describes each of the
+steps and then demonstrates these by focusing on predicting death in
+those who have covid-19.</p>
 </div>
-<div id="main-steps-for-running-a-network-study" class="section level1">
-<h1 class="hasAnchor">
-<a href="#main-steps-for-running-a-network-study" class="anchor"></a>Main steps for running a network study</h1>
-<div id="step-1-developing-the-study" class="section level2">
-<h2 class="hasAnchor">
-<a href="#step-1-developing-the-study" class="anchor"></a>Step 1 – developing the study</h2>
+<div class="section level2">
+<h2 id="main-steps-for-running-a-network-study">Main steps for running a network study<a class="anchor" aria-label="anchor" href="#main-steps-for-running-a-network-study"></a>
+</h2>
+<div class="section level3">
+<h3 id="step-1-developing-the-study">Step 1 – developing the study<a class="anchor" aria-label="anchor" href="#step-1-developing-the-study"></a>
+</h3>
 <ul>
-<li>Design the study: target/outcome cohort logic, concept sets for medical definitions, settings for developing new model or validation of adding existing models to framework. Suggestion: look in literature for validated definitions.</li>
-<li>Write a protocol that motivates the study and provides full details (sufficient for people to replicate the study in the future).</li>
-<li>Write an R package for implementing the study across diverse computational environments [see guidance below for structure of package and use the skeleton github package here: <a href="https://github.com/OHDSI/SkeletonPredictionStudy" class="uri">https://github.com/OHDSI/SkeletonPredictionStudy</a> ]</li>
+<li>Design the study: target/outcome cohort logic, concept sets for
+medical definitions, settings for developing new model or validation of
+adding existing models to framework. Suggestion: look in literature for
+validated definitions.</li>
+<li>Write a protocol that motivates the study and provides full details
+(sufficient for people to replicate the study in the future).</li>
+<li>Write an R package for implementing the study across diverse
+computational environments [see guidance below for structure of package
+and use the skeleton github package here: <a href="https://github.com/OHDSI/SkeletonPredictionStudy" class="external-link uri">https://github.com/OHDSI/SkeletonPredictionStudy</a> ]</li>
 </ul>
 </div>
-<div id="step-2-implementing-the-study-part-1" class="section level2">
-<h2 class="hasAnchor">
-<a href="#step-2-implementing-the-study-part-1" class="anchor"></a>Step 2 – implementing the study part 1</h2>
+<div class="section level3">
+<h3 id="step-2-implementing-the-study-part-1">Step 2 – implementing the study part 1<a class="anchor" aria-label="anchor" href="#step-2-implementing-the-study-part-1"></a>
+</h3>
 <ul>
-<li>Get contributors to install the package and dependencies. Ensure the package is installed correctly for each contributor by asking them to run the checkInstall functions (as specified in the InstallationGuide).</li>
-<li>Get contributors to run the createCohort function to inspect the target/outcome definitions. If the definitions are not suitable for a site, go back to step 1 and revise the cohort definitions.</li>
+<li>Get contributors to install the package and dependencies. Ensure the
+package is installed correctly for each contributor by asking them to
+run the checkInstall functions (as specified in the
+InstallationGuide).</li>
+<li>Get contributors to run the createCohort function to inspect the
+target/outcome definitions. If the definitions are not suitable for a
+site, go back to step 1 and revise the cohort definitions.</li>
 </ul>
 </div>
-<div id="step-3-implementing-the-study-part-2-make-sure-the-package-is-functioning-as-planned-and-the-definitions-are-valid-across-sites" class="section level2">
-<h2 class="hasAnchor">
-<a href="#step-3-implementing-the-study-part-2-make-sure-the-package-is-functioning-as-planned-and-the-definitions-are-valid-across-sites" class="anchor"></a>Step 3 – implementing the study part 2 (make sure the package is functioning as planned and the definitions are valid across sites)</h2>
+<div class="section level3">
+<h3 id="step-3-implementing-the-study-part-2-make-sure-the-package-is-functioning-as-planned-and-the-definitions-are-valid-across-sites">Step 3 – implementing the study part 2 (make sure the package is
+functioning as planned and the definitions are valid across sites)<a class="anchor" aria-label="anchor" href="#step-3-implementing-the-study-part-2-make-sure-the-package-is-functioning-as-planned-and-the-definitions-are-valid-across-sites"></a>
+</h3>
 <ul>
-<li>Get contributors to run the main.R with the settings configured to their environment</li>
+<li>Get contributors to run the main.R with the settings configured to
+their environment</li>
 <li>Get the contributors to submit the results</li>
 </ul>
 </div>
-<div id="step-4-publication" class="section level2">
-<h2 class="hasAnchor">
-<a href="#step-4-publication" class="anchor"></a>Step 4 – Publication</h2>
-<p>The study creator has the first option to be first author, if he/she does not wish to be first author then he/she can pick the most suitable person from the contributors. All contributors will be listed as authors on the paper. The last author will be the person who lead/managed the study, if this was the first author then the first author can pick the most suitable last author. All authors between the first and last author will be alphabetical by last name.</p>
+<div class="section level3">
+<h3 id="step-4-publication">Step 4 – Publication<a class="anchor" aria-label="anchor" href="#step-4-publication"></a>
+</h3>
+<p>The study creator has the first option to be first author, if he/she
+does not wish to be first author then he/she can pick the most suitable
+person from the contributors. All contributors will be listed as authors
+on the paper. The last author will be the person who lead/managed the
+study, if this was the first author then the first author can pick the
+most suitable last author. All authors between the first and last author
+will be alphabetical by last name.</p>
 </div>
 </div>
-<div id="package-skeleton---file-structure" class="section level1">
-<h1 class="hasAnchor">
-<a href="#package-skeleton---file-structure" class="anchor"></a>Package Skeleton - File Structure</h1>
+<div class="section level2">
+<h2 id="package-skeleton---file-structure">Package Skeleton - File Structure<a class="anchor" aria-label="anchor" href="#package-skeleton---file-structure"></a>
+</h2>
 <ul>
-<li>DESCRIPTION: This file describes the R package and the dependencies</li>
+<li>DESCRIPTION: This file describes the R package and the
+dependencies</li>
 <li>NAMESPACE: This file is created automatically by Roxygen</li>
-<li>Readme.md: This file should provide the step by step guidance on implementing the package</li>
+<li>Readme.md: This file should provide the step by step guidance on
+implementing the package</li>
 <li>R</li>
-<li>helpers.r: all the custom functions used by the package should be in this file (e.g., checkInstall)</li>
-<li>main.r: this file will call the functions in helpers.r to execute the full study</li>
-<li>submit.r: this file will be called at the end to submit the compressed folder to the study creator/manager.</li>
-<li>Man: this folder will contain the documentation for the functions in helpers.r (this should be automatically generated by roxygen)</li>
+<li>helpers.r: all the custom functions used by the package should be in
+this file (e.g., checkInstall)</li>
+<li>main.r: this file will call the functions in helpers.r to execute
+the full study</li>
+<li>submit.r: this file will be called at the end to submit the
+compressed folder to the study creator/manager.</li>
+<li>Man: this folder will contain the documentation for the functions in
+helpers.r (this should be automatically generated by roxygen)</li>
 <li>Inst</li>
-<li>sql/sql_sever * targetCohort: the target cohort parameterised sql code * outcomeCohort: the outcome cohort parameterised sql code</li>
+<li>sql/sql_sever * targetCohort: the target cohort parameterised sql
+code * outcomeCohort: the outcome cohort parameterised sql code</li>
 <li>plp_models: place any PLP models here</li>
 <li>Extras</li>
 </ul>
@@ -225,11 +279,13 @@ <h1 class="hasAnchor">
 
 
       <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+  <p></p>
+<p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p>
+<p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
       </footer>
@@ -238,5 +294,7 @@ <h1 class="hasAnchor">
   
 
 
+  
+
   </body>
 </html>
diff --git a/docs/articles/CreatingNetworkStudies_files/header-attrs-2.11/header-attrs.js b/docs/articles/CreatingNetworkStudies_files/header-attrs-2.11/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/CreatingNetworkStudies_files/header-attrs-2.11/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/CreatingNetworkStudies_files/header-attrs-2.7/header-attrs.js b/docs/articles/CreatingNetworkStudies_files/header-attrs-2.7/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/CreatingNetworkStudies_files/header-attrs-2.7/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/CreatingShinyApp.html b/docs/articles/CreatingShinyApp.html
deleted file mode 100644
index 838fedfa8..000000000
--- a/docs/articles/CreatingShinyApp.html
+++ /dev/null
@@ -1,456 +0,0 @@
-<!DOCTYPE html>
-<!-- Generated by pkgdown: do not edit by hand --><html lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-<title>Creating Shiny App • PatientLevelPrediction</title>
-<!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous">
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous">
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous">
-<!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script><meta property="og:title" content="Creating Shiny App">
-<meta property="og:description" content="PatientLevelPrediction">
-<!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-</head>
-<body data-spy="scroll" data-target="#toc">
-    <div class="container template-article">
-      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-<li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/InstallationGuide.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-<li>
-      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
-    </li>
-    <li>
-      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
-    </li>
-    <li>
-      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-<ul class="nav navbar-nav navbar-right">
-<li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-</div>
-<!--/.nav-collapse -->
-  </div>
-<!--/.container -->
-</div>
-<!--/.navbar -->
-
-      
-
-      </header><script src="CreatingShinyApp_files/header-attrs-2.11/header-attrs.js"></script><div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header toc-ignore">
-      <h1 data-toc-skip>Creating Shiny App</h1>
-                        <h4 class="author">Jenna Reps</h4>
-            
-            <h4 class="date">2022-03-09</h4>
-      
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/CreatingShinyApp.Rmd"><code>vignettes/CreatingShinyApp.Rmd</code></a></small>
-      <div class="hidden name"><code>CreatingShinyApp.Rmd</code></div>
-
-    </div>
-
-    
-    
-<!--
-%\VignetteEngine{knitr}
-%\VignetteIndexEntry{Creating Shiny App}
--->
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
-<p>In this vignette we will show with example code how to create a shiny app and add the shiny app online for other researcher around the whole to explore.</p>
-<p>There are two ways to create the shiny app: 1) Using the atlas R generated prediction package 2) Manually using the PatientLevelPrediction functions in a script</p>
-<p>We assume you have experience with using the OHDSI PatientLevelPrediction package to develop and externally validate prediction models using data in the OMOP CDM. If you do not have experience with this then please first read our general vignette <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf"><code>BuildingPredictiveModels</code> vignette</a>.</p>
-</div>
-<div id="atlas-development-shiny-app" class="section level1">
-<h1 class="hasAnchor">
-<a href="#atlas-development-shiny-app" class="anchor"></a>Atlas Development Shiny App</h1>
-<div id="step-1-run-the-model-development-package-to-get-results" class="section level2">
-<h2 class="hasAnchor">
-<a href="#step-1-run-the-model-development-package-to-get-results" class="anchor"></a>Step 1: Run the model development package to get results</h2>
-<p>To create a shiny app project via the Atlas auto-generated prediction R package you named ‘myPackage’ you need to run the execute function:</p>
-<div class="sourceCode" id="cb1"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="kw"><a href="https://rdrr.io/r/base/library.html">library</a></span><span class="op">(</span><span class="va">myPackage</span><span class="op">)</span>
-<span class="fu">myPackage</span><span class="fu">::</span><span class="fu">execute</span><span class="op">(</span>connectionDetails <span class="op">=</span> <span class="va">connectionDetails</span>,
-        cdmDatabaseSchema <span class="op">=</span> <span class="st">'myDatabaseSchema.dbo'</span>,
-        cdmDatabaseName <span class="op">=</span> <span class="st">'MyDatabase'</span>,
-        cohortDatabaseSchema <span class="op">=</span> <span class="st">'myDatabaseSchema.ohdsi_results'</span>,
-        cohortTable <span class="op">=</span> <span class="st">'cohort'</span>,
-        outputFolder <span class="op">=</span> <span class="st">'C:/myResults'</span>,
-        createProtocol <span class="op">=</span> <span class="cn">F</span>,
-        createCohorts <span class="op">=</span> <span class="cn">F</span>,
-        runAnalyses <span class="op">=</span> <span class="cn">T</span>,
-        createResultsDoc <span class="op">=</span> <span class="cn">F</span>,
-        packageResults <span class="op">=</span> <span class="cn">F</span>,
-        createValidationPackage <span class="op">=</span> <span class="cn">F</span>, 
-        minCellCount<span class="op">=</span> <span class="fl">5</span>,
-        createShiny <span class="op">=</span> <span class="cn">F</span>,
-        createJournalDocument <span class="op">=</span> <span class="cn">F</span>,
-        analysisIdDocument <span class="op">=</span> <span class="fl">1</span><span class="op">)</span></code></pre></div>
-<p>This will extract data based on the settings you supplied in the Atlas prediction design from cohort tables already generated in your CDM database schema. The PatientLevelPrediction framework will then run and develop/evaluate models saving the results to the location specified by outputFolder (e.g., ‘C:/myResults’).</p>
-</div>
-<div id="step-2-create-the-shiny-app" class="section level2">
-<h2 class="hasAnchor">
-<a href="#step-2-create-the-shiny-app" class="anchor"></a>Step 2: Create the shiny app</h2>
-<p>To create a shiny app project with these results you can then simply run:</p>
-<div class="sourceCode" id="cb2"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu">myPackage</span><span class="fu">::</span><span class="fu">execute</span><span class="op">(</span>connectionDetails <span class="op">=</span> <span class="va">connectionDetails</span>,
-        cdmDatabaseSchema <span class="op">=</span> <span class="st">'myDatabaseSchema.dbo'</span>,
-        cdmDatabaseName <span class="op">=</span> <span class="st">'MyDatabase'</span>,
-        cohortDatabaseSchema <span class="op">=</span> <span class="st">'myDatabaseSchema.ohdsi_results'</span>,
-        cohortTable <span class="op">=</span> <span class="st">'cohort'</span>,
-        outputFolder <span class="op">=</span> <span class="st">'C:/myResults'</span>,
-        minCellCount<span class="op">=</span> <span class="fl">5</span>,
-        createShiny <span class="op">=</span> <span class="cn">T</span><span class="op">)</span></code></pre></div>
-<p>making sure the outputFolder is the same location used when you ran the analysis. This code populates a shiny app project with the results but removes sensitive date such as connection settings, the cdmDatabaseSchema setting, the predicton matrix and any sensitive counts less than ‘minCellCount’ from the covariate summary and performance evalaution.</p>
-<p>The shiny app project populated with the model development results can then be found at ‘[outputFolder]/ShinyApp’ e.g., ‘C:/myResults/ShinyApp’.</p>
-<div id="testing-optional-but-recommended" class="section level3">
-<h3 class="hasAnchor">
-<a href="#testing-optional-but-recommended" class="anchor"></a>Testing (Optional but recommended)</h3>
-<p>You can test the app by opening the shiny project within the [outputFolder]/ShinyApp’ folder, double click on the file named ‘PLPViewer.Rproj’. This will open an R studio session with the shiny app project loaded. Now load the ‘ui.R’ files within this R studio session and you will see a green arrow with the words ‘Run App’ at the top right of the script. Click on this and the shiny app with open. Note: You may need to install some R pacakge dependancies for the shiny app to work.</p>
-</div>
-</div>
-<div id="step-3-sharing-the-shiny-app" class="section level2">
-<h2 class="hasAnchor">
-<a href="#step-3-sharing-the-shiny-app" class="anchor"></a>Step 3: Sharing the shiny app</h2>
-<p>Once you are happy with your app, you can publish it onto <a href="https://data.ohdsi.org" class="uri">https://data.ohdsi.org</a> by adding the folder ‘ShinyApp’ to the OHDSI githib ShinyDeploy (<a href="https://github.com/OHDSI/ShinyDeploy/" class="uri">https://github.com/OHDSI/ShinyDeploy/</a>). Continuing the example, we would copy the folder ‘[outputFolder]/ShinyApp’ and paste it to the local github clone of ShinyDeploy. We recommend renaming the folder from ‘ShinyApp’ to a name that describes your prediction, e.g., ‘StrokeinAF’. Then commit the changes and make a pull request to ShinyDeploy. Once accepted your shiny app will be viewable at ‘<a href="https://data.ohdsi.org" class="uri">https://data.ohdsi.org</a>’. If you commited the folder named ‘StrokeInAF’ then the shiny app will be hosted at ‘<a href="https://data.ohdsi.org/StrokeInAF" class="uri">https://data.ohdsi.org/StrokeInAF</a>’.</p>
-</div>
-</div>
-<div id="atlas-external-validation" class="section level1">
-<h1 class="hasAnchor">
-<a href="#atlas-external-validation" class="anchor"></a>Atlas External Validation</h1>
-<p>To include external validation results you can use the Atlas generated R study package to create the external validation package:</p>
-<div class="sourceCode" id="cb3"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu">myPackage</span><span class="fu">::</span><span class="fu">execute</span><span class="op">(</span>connectionDetails <span class="op">=</span> <span class="va">connectionDetails</span>,
-        cdmDatabaseSchema <span class="op">=</span> <span class="st">'myDatabaseSchema.dbo'</span>,
-        cdmDatabaseName <span class="op">=</span> <span class="st">'MyDatabase'</span>,
-        cohortDatabaseSchema <span class="op">=</span> <span class="st">'myDatabaseSchema.ohdsi_results'</span>,
-        cohortTable <span class="op">=</span> <span class="st">'cohort'</span>,
-        outputFolder <span class="op">=</span> <span class="st">'C:/myResults'</span>,
-        createValidationPackage <span class="op">=</span> <span class="cn">T</span><span class="op">)</span></code></pre></div>
-<p>This will create a new R package inside the ‘outputFolder’ location with the word ‘Validation’ appended the name of your development package. For example, if your ‘outputFolder’ was ‘C:/myResults’ and your development package was named ‘myPackage’ then the validation package will be found at: ‘C:/myResults/myPackageValidation’. When running the valdiation package make sure to set the ‘outputFolder’ to the Validation folder within your model development outputFolder location:</p>
-<div class="sourceCode" id="cb4"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu">myPackageValidation</span><span class="fu">::</span><span class="fu">execute</span><span class="op">(</span>connectionDetails <span class="op">=</span> <span class="va">connectionDetails</span>,
-                 databaseName <span class="op">=</span> <span class="va">databaseName</span>,
-                 cdmDatabaseSchema <span class="op">=</span> <span class="va">cdmDatabaseSchema</span>,
-                 cohortDatabaseSchema <span class="op">=</span> <span class="va">cohortDatabaseSchema</span>,
-                 oracleTempSchema <span class="op">=</span> <span class="va">oracleTempSchema</span>,
-                 cohortTable <span class="op">=</span> <span class="va">cohortTable</span>,
-                 outputFolder <span class="op">=</span> <span class="st">'C:/myResults/Validation'</span>,
-                 createCohorts <span class="op">=</span> <span class="cn">T</span>,
-                 runValidation <span class="op">=</span> <span class="cn">T</span>,
-                 packageResults <span class="op">=</span> <span class="cn">F</span>,
-                 minCellCount <span class="op">=</span> <span class="fl">5</span>,
-                 sampleSize <span class="op">=</span> <span class="cn">NULL</span><span class="op">)</span></code></pre></div>
-<p>Now you can rerun Steps 2-3 to populate the shiny app project that will also include the validation results (as long as the validation results are in the Validation folder found in the Step 1 outputFolder location e.g., in ‘C:/myResults/Validation’).</p>
-</div>
-<div id="combining-multiple-atlas-results-into-one-shiny-app" class="section level1">
-<h1 class="hasAnchor">
-<a href="#combining-multiple-atlas-results-into-one-shiny-app" class="anchor"></a>Combining multiple atlas results into one shiny app:</h1>
-<p>The code below can be used to combine multiple Atlas packages’ results into one shiny app:</p>
-<div class="sourceCode" id="cb5"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="va">populateMultipleShinyApp</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">shinyDirectory</span>,
-                             <span class="va">resultDirectory</span>,
-                             <span class="va">minCellCount</span> <span class="op">=</span> <span class="fl">10</span>,
-                             <span class="va">databaseName</span> <span class="op">=</span> <span class="st">'sharable name of development data'</span><span class="op">)</span><span class="op">{</span>
-  
-  <span class="co">#check inputs</span>
-  <span class="kw">if</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/missing.html">missing</a></span><span class="op">(</span><span class="va">shinyDirectory</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-    <span class="va">shinyDirectory</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/system.file.html">system.file</a></span><span class="op">(</span><span class="st">"shiny"</span>, <span class="st">"PLPViewer"</span>, package <span class="op">=</span> <span class="st">"SkeletonPredictionStudy"</span><span class="op">)</span>
-  <span class="op">}</span>
-  <span class="kw">if</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/missing.html">missing</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-    <span class="kw"><a href="https://rdrr.io/r/base/stop.html">stop</a></span><span class="op">(</span><span class="st">'Need to enter the resultDirectory'</span><span class="op">)</span>
-  <span class="op">}</span>
-  
-
-    <span class="kw">for</span><span class="op">(</span><span class="va">i</span> <span class="kw">in</span> <span class="fl">1</span><span class="op">:</span><span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-      <span class="kw">if</span><span class="op">(</span><span class="op">!</span><span class="fu"><a href="https://rdrr.io/r/base/files2.html">dir.exists</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">[</span><span class="va">i</span><span class="op">]</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-        <span class="kw"><a href="https://rdrr.io/r/base/stop.html">stop</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste</a></span><span class="op">(</span><span class="st">'resultDirectory '</span>,<span class="va">i</span>,<span class="st">' does not exist'</span><span class="op">)</span><span class="op">)</span>
-      <span class="op">}</span>
-    <span class="op">}</span>
-  
-  <span class="va">outputDirectory</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">shinyDirectory</span>,<span class="st">'data'</span><span class="op">)</span>
-  
-  <span class="co"># create the shiny data folder</span>
-  <span class="kw">if</span><span class="op">(</span><span class="op">!</span><span class="fu"><a href="https://rdrr.io/r/base/files2.html">dir.exists</a></span><span class="op">(</span><span class="va">outputDirectory</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-    <span class="fu"><a href="https://rdrr.io/r/base/files2.html">dir.create</a></span><span class="op">(</span><span class="va">outputDirectory</span>, recursive <span class="op">=</span> <span class="cn">T</span><span class="op">)</span>
-  <span class="op">}</span>
-  
-  
-  <span class="co"># need to edit settings ...</span>
-  <span class="va">files</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="op">)</span>
-  <span class="kw">for</span><span class="op">(</span><span class="va">i</span> <span class="kw">in</span> <span class="fl">1</span><span class="op">:</span><span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-  <span class="co"># copy the settings csv</span>
-  <span class="va">file</span> <span class="op">&lt;-</span> <span class="fu">utils</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/r/utils/read.table.html">read.csv</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">[</span><span class="va">i</span><span class="op">]</span>,<span class="st">'settings.csv'</span><span class="op">)</span><span class="op">)</span>
-  <span class="va">file</span><span class="op">$</span><span class="va">analysisId</span> <span class="op">&lt;-</span> <span class="fl">1000</span><span class="op">*</span><span class="fu"><a href="https://rdrr.io/r/base/double.html">as.double</a></span><span class="op">(</span><span class="va">file</span><span class="op">$</span><span class="va">analysisId</span><span class="op">)</span><span class="op">+</span><span class="va">i</span>
-  <span class="va">files</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/cbind.html">rbind</a></span><span class="op">(</span><span class="va">files</span>, <span class="va">file</span><span class="op">)</span>
-  <span class="op">}</span>
-  <span class="fu">utils</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/r/utils/write.table.html">write.csv</a></span><span class="op">(</span><span class="va">files</span>, <span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">outputDirectory</span>,<span class="st">'settings.csv'</span><span class="op">)</span>, row.names <span class="op">=</span> <span class="cn">F</span><span class="op">)</span>
-  
-  <span class="kw">for</span><span class="op">(</span><span class="va">i</span> <span class="kw">in</span> <span class="fl">1</span><span class="op">:</span><span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-  <span class="co"># copy each analysis as a rds file and copy the log</span>
-  <span class="va">files</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.files.html">dir</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">[</span><span class="va">i</span><span class="op">]</span>, full.names <span class="op">=</span> <span class="cn">F</span><span class="op">)</span>
-  <span class="va">files</span> <span class="op">&lt;-</span> <span class="va">files</span><span class="op">[</span><span class="fu"><a href="https://rdrr.io/r/base/grep.html">grep</a></span><span class="op">(</span><span class="st">'Analysis'</span>, <span class="va">files</span><span class="op">)</span><span class="op">]</span>
-  <span class="kw">for</span><span class="op">(</span><span class="va">file</span> <span class="kw">in</span> <span class="va">files</span><span class="op">)</span><span class="op">{</span>
-    
-    <span class="kw">if</span><span class="op">(</span><span class="op">!</span><span class="fu"><a href="https://rdrr.io/r/base/files2.html">dir.exists</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">outputDirectory</span>,<span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste0</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="fl">1000</span><span class="op">*</span><span class="fu"><a href="https://rdrr.io/r/base/double.html">as.double</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/grep.html">gsub</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="st">''</span>,<span class="va">file</span><span class="op">)</span><span class="op">)</span><span class="op">+</span><span class="va">i</span><span class="op">)</span><span class="op">)</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-      <span class="fu"><a href="https://rdrr.io/r/base/files2.html">dir.create</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">outputDirectory</span>,<span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste0</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="fl">1000</span><span class="op">*</span><span class="fu"><a href="https://rdrr.io/r/base/double.html">as.double</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/grep.html">gsub</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="st">''</span>,<span class="va">file</span><span class="op">)</span><span class="op">)</span><span class="op">+</span><span class="va">i</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>
-    <span class="op">}</span>
-    
-    <span class="kw">if</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/files2.html">dir.exists</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">[</span><span class="va">i</span><span class="op">]</span>,<span class="va">file</span>, <span class="st">'plpResult'</span><span class="op">)</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-      <span class="va">res</span> <span class="op">&lt;-</span> <span class="fu">PatientLevelPrediction</span><span class="fu">::</span><span class="fu"><a href="../reference/loadPlpResult.html">loadPlpResult</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">[</span><span class="va">i</span><span class="op">]</span>,<span class="va">file</span>, <span class="st">'plpResult'</span><span class="op">)</span><span class="op">)</span>
-      <span class="va">res</span> <span class="op">&lt;-</span> <span class="fu">PatientLevelPrediction</span><span class="fu">::</span><span class="fu">transportPlp</span><span class="op">(</span><span class="va">res</span>, n<span class="op">=</span> <span class="va">minCellCount</span>, 
-                                                  save <span class="op">=</span> <span class="cn">F</span>, dataName <span class="op">=</span> <span class="va">databaseName</span><span class="op">[</span><span class="va">i</span><span class="op">]</span><span class="op">)</span>
-      
-      <span class="va">res</span><span class="op">$</span><span class="va">covariateSummary</span> <span class="op">&lt;-</span> <span class="va">res</span><span class="op">$</span><span class="va">covariateSummary</span><span class="op">[</span><span class="va">res</span><span class="op">$</span><span class="va">covariateSummary</span><span class="op">$</span><span class="va">covariateValue</span><span class="op">!=</span><span class="fl">0</span>,<span class="op">]</span>
-      <span class="va">covSet</span> <span class="op">&lt;-</span> <span class="va">res</span><span class="op">$</span><span class="va">model</span><span class="op">$</span><span class="va">metaData</span><span class="op">$</span><span class="va">call</span><span class="op">$</span><span class="va">covariateSettings</span>
-      <span class="va">res</span><span class="op">$</span><span class="va">model</span><span class="op">$</span><span class="va">metaData</span> <span class="op">&lt;-</span> <span class="cn">NULL</span>
-      <span class="va">res</span><span class="op">$</span><span class="va">model</span><span class="op">$</span><span class="va">metaData</span><span class="op">$</span><span class="va">call</span><span class="op">$</span><span class="va">covariateSettings</span> <span class="op">&lt;-</span> <span class="va">covSet</span>
-      <span class="va">res</span><span class="op">$</span><span class="va">model</span><span class="op">$</span><span class="va">predict</span> <span class="op">&lt;-</span> <span class="cn">NULL</span>
-      <span class="kw">if</span><span class="op">(</span><span class="op">!</span><span class="fu"><a href="https://rdrr.io/r/base/NULL.html">is.null</a></span><span class="op">(</span><span class="va">res</span><span class="op">$</span><span class="va">performanceEvaluation</span><span class="op">$</span><span class="va">evaluationStatistics</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-      <span class="va">res</span><span class="op">$</span><span class="va">performanceEvaluation</span><span class="op">$</span><span class="va">evaluationStatistics</span><span class="op">[</span>,<span class="fl">1</span><span class="op">]</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste0</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="fl">1000</span><span class="op">*</span><span class="fu"><a href="https://rdrr.io/r/base/double.html">as.double</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/grep.html">gsub</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="st">''</span>,<span class="va">file</span><span class="op">)</span><span class="op">)</span><span class="op">+</span><span class="va">i</span><span class="op">)</span>
-      <span class="op">}</span> <span class="kw">else</span><span class="op">{</span>
-        <span class="fu"><a href="https://rdrr.io/r/base/writeLines.html">writeLines</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste0</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">[</span><span class="va">i</span><span class="op">]</span>,<span class="va">file</span>, <span class="st">'-ev'</span><span class="op">)</span><span class="op">)</span>
-      <span class="op">}</span>
-      <span class="kw">if</span><span class="op">(</span><span class="op">!</span><span class="fu"><a href="https://rdrr.io/r/base/NULL.html">is.null</a></span><span class="op">(</span><span class="va">res</span><span class="op">$</span><span class="va">performanceEvaluation</span><span class="op">$</span><span class="va">thresholdSummary</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-      <span class="va">res</span><span class="op">$</span><span class="va">performanceEvaluation</span><span class="op">$</span><span class="va">thresholdSummary</span><span class="op">[</span>,<span class="fl">1</span><span class="op">]</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste0</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="fl">1000</span><span class="op">*</span><span class="fu"><a href="https://rdrr.io/r/base/double.html">as.double</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/grep.html">gsub</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="st">''</span>,<span class="va">file</span><span class="op">)</span><span class="op">)</span><span class="op">+</span><span class="va">i</span><span class="op">)</span>
-      <span class="op">}</span><span class="kw">else</span><span class="op">{</span>
-        <span class="fu"><a href="https://rdrr.io/r/base/writeLines.html">writeLines</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste0</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">[</span><span class="va">i</span><span class="op">]</span>,<span class="va">file</span>, <span class="st">'-thres'</span><span class="op">)</span><span class="op">)</span>
-      <span class="op">}</span>
-      <span class="kw">if</span><span class="op">(</span><span class="op">!</span><span class="fu"><a href="https://rdrr.io/r/base/NULL.html">is.null</a></span><span class="op">(</span><span class="va">res</span><span class="op">$</span><span class="va">performanceEvaluation</span><span class="op">$</span><span class="va">demographicSummary</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-      <span class="va">res</span><span class="op">$</span><span class="va">performanceEvaluation</span><span class="op">$</span><span class="va">demographicSummary</span><span class="op">[</span>,<span class="fl">1</span><span class="op">]</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste0</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="fl">1000</span><span class="op">*</span><span class="fu"><a href="https://rdrr.io/r/base/double.html">as.double</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/grep.html">gsub</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="st">''</span>,<span class="va">file</span><span class="op">)</span><span class="op">)</span><span class="op">+</span><span class="va">i</span><span class="op">)</span>
-      <span class="op">}</span> <span class="kw">else</span><span class="op">{</span>
-        <span class="fu"><a href="https://rdrr.io/r/base/writeLines.html">writeLines</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste0</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">[</span><span class="va">i</span><span class="op">]</span>,<span class="va">file</span>, <span class="st">'-dem'</span><span class="op">)</span><span class="op">)</span>
-      <span class="op">}</span>
-      <span class="kw">if</span><span class="op">(</span><span class="op">!</span><span class="fu"><a href="https://rdrr.io/r/base/NULL.html">is.null</a></span><span class="op">(</span><span class="va">res</span><span class="op">$</span><span class="va">performanceEvaluation</span><span class="op">$</span><span class="va">calibrationSummary</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-      <span class="va">res</span><span class="op">$</span><span class="va">performanceEvaluation</span><span class="op">$</span><span class="va">calibrationSummary</span><span class="op">[</span>,<span class="fl">1</span><span class="op">]</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste0</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="fl">1000</span><span class="op">*</span><span class="fu"><a href="https://rdrr.io/r/base/double.html">as.double</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/grep.html">gsub</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="st">''</span>,<span class="va">file</span><span class="op">)</span><span class="op">)</span><span class="op">+</span><span class="va">i</span><span class="op">)</span>
-      <span class="op">}</span><span class="kw">else</span><span class="op">{</span>
-        <span class="fu"><a href="https://rdrr.io/r/base/writeLines.html">writeLines</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste0</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">[</span><span class="va">i</span><span class="op">]</span>,<span class="va">file</span>, <span class="st">'-cal'</span><span class="op">)</span><span class="op">)</span>
-      <span class="op">}</span>
-      <span class="kw">if</span><span class="op">(</span><span class="op">!</span><span class="fu"><a href="https://rdrr.io/r/base/NULL.html">is.null</a></span><span class="op">(</span><span class="va">res</span><span class="op">$</span><span class="va">performanceEvaluation</span><span class="op">$</span><span class="va">predictionDistribution</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-      <span class="va">res</span><span class="op">$</span><span class="va">performanceEvaluation</span><span class="op">$</span><span class="va">predictionDistribution</span><span class="op">[</span>,<span class="fl">1</span><span class="op">]</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste0</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="fl">1000</span><span class="op">*</span><span class="fu"><a href="https://rdrr.io/r/base/double.html">as.double</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/grep.html">gsub</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="st">''</span>,<span class="va">file</span><span class="op">)</span><span class="op">)</span><span class="op">+</span><span class="va">i</span><span class="op">)</span>
-      <span class="op">}</span><span class="kw">else</span><span class="op">{</span>
-        <span class="fu"><a href="https://rdrr.io/r/base/writeLines.html">writeLines</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste0</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">[</span><span class="va">i</span><span class="op">]</span>,<span class="va">file</span>, <span class="st">'-dist'</span><span class="op">)</span><span class="op">)</span>
-      <span class="op">}</span>
-      <span class="fu"><a href="https://rdrr.io/r/base/readRDS.html">saveRDS</a></span><span class="op">(</span><span class="va">res</span>, <span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">outputDirectory</span>,<span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste0</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="fl">1000</span><span class="op">*</span><span class="fu"><a href="https://rdrr.io/r/base/double.html">as.double</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/grep.html">gsub</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="st">''</span>,<span class="va">file</span><span class="op">)</span><span class="op">)</span><span class="op">+</span><span class="va">i</span><span class="op">)</span>, <span class="st">'plpResult.rds'</span><span class="op">)</span><span class="op">)</span>
-    <span class="op">}</span>
-    <span class="kw">if</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/files.html">file.exists</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">[</span><span class="va">i</span><span class="op">]</span>,<span class="va">file</span>, <span class="st">'plpLog.txt'</span><span class="op">)</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-      <span class="fu"><a href="https://rdrr.io/r/base/files.html">file.copy</a></span><span class="op">(</span>from <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">[</span><span class="va">i</span><span class="op">]</span>,<span class="va">file</span>, <span class="st">'plpLog.txt'</span><span class="op">)</span>, 
-                to <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">outputDirectory</span>,<span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste0</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="fl">1000</span><span class="op">*</span><span class="fu"><a href="https://rdrr.io/r/base/double.html">as.double</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/grep.html">gsub</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="st">''</span>,<span class="va">file</span><span class="op">)</span><span class="op">)</span><span class="op">+</span><span class="va">i</span><span class="op">)</span>, <span class="st">'plpLog.txt'</span><span class="op">)</span><span class="op">)</span>
-    <span class="op">}</span>
-  <span class="op">}</span>
-  <span class="op">}</span>
-  
-  
-  
-  <span class="kw">for</span><span class="op">(</span><span class="va">i</span> <span class="kw">in</span> <span class="fl">1</span><span class="op">:</span><span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-  <span class="co"># copy any validation results</span>
-  <span class="kw">if</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/files2.html">dir.exists</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">[</span><span class="va">i</span><span class="op">]</span>,<span class="st">'Validation'</span><span class="op">)</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-    <span class="va">valFolders</span> <span class="op">&lt;-</span>  <span class="fu"><a href="https://rdrr.io/r/base/list.files.html">dir</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">[</span><span class="va">i</span><span class="op">]</span>,<span class="st">'Validation'</span><span class="op">)</span>, full.names <span class="op">=</span> <span class="cn">F</span><span class="op">)</span>
-    
-    <span class="kw">if</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="va">valFolders</span><span class="op">)</span><span class="op">&gt;</span><span class="fl">0</span><span class="op">)</span><span class="op">{</span>
-      <span class="co"># move each of the validation rds</span>
-      <span class="kw">for</span><span class="op">(</span><span class="va">valFolder</span> <span class="kw">in</span> <span class="va">valFolders</span><span class="op">)</span><span class="op">{</span>
-        
-        <span class="co"># get the analysisIds</span>
-        <span class="va">valSubfolders</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.files.html">dir</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">[</span><span class="va">i</span><span class="op">]</span>,<span class="st">'Validation'</span>,<span class="va">valFolder</span><span class="op">)</span>, full.names <span class="op">=</span> <span class="cn">F</span><span class="op">)</span>
-        <span class="kw">if</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/length.html">length</a></span><span class="op">(</span><span class="va">valSubfolders</span><span class="op">)</span><span class="op">!=</span><span class="fl">0</span><span class="op">)</span><span class="op">{</span>
-          <span class="kw">for</span><span class="op">(</span><span class="va">valSubfolder</span> <span class="kw">in</span> <span class="va">valSubfolders</span> <span class="op">)</span><span class="op">{</span>
-            <span class="va">valSubfolderUpdate</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste0</a></span><span class="op">(</span><span class="st">'Analysis_'</span>, <span class="fu"><a href="https://rdrr.io/r/base/double.html">as.double</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/grep.html">gsub</a></span><span class="op">(</span><span class="st">'Analysis_'</span>,<span class="st">''</span>, <span class="va">valSubfolder</span><span class="op">)</span><span class="op">)</span><span class="op">*</span><span class="fl">1000</span><span class="op">+</span><span class="va">i</span><span class="op">)</span>
-            <span class="va">valOut</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">valFolder</span>,<span class="va">valSubfolderUpdate</span><span class="op">)</span>
-            <span class="va">valOutOld</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">valFolder</span>,<span class="va">valSubfolder</span><span class="op">)</span>
-            <span class="kw">if</span><span class="op">(</span><span class="op">!</span><span class="fu"><a href="https://rdrr.io/r/base/files2.html">dir.exists</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">outputDirectory</span>,<span class="st">'Validation'</span>,<span class="va">valOut</span><span class="op">)</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-              <span class="fu"><a href="https://rdrr.io/r/base/files2.html">dir.create</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">outputDirectory</span>,<span class="st">'Validation'</span>,<span class="va">valOut</span><span class="op">)</span>, recursive <span class="op">=</span> <span class="cn">T</span><span class="op">)</span>
-            <span class="op">}</span>
-            
-            
-            <span class="kw">if</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/files.html">file.exists</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">[</span><span class="va">i</span><span class="op">]</span>,<span class="st">'Validation'</span>,<span class="va">valOutOld</span>, <span class="st">'validationResult.rds'</span><span class="op">)</span><span class="op">)</span><span class="op">)</span><span class="op">{</span>
-              <span class="va">res</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/readRDS.html">readRDS</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">resultDirectory</span><span class="op">[</span><span class="va">i</span><span class="op">]</span>,<span class="st">'Validation'</span>,<span class="va">valOutOld</span>, <span class="st">'validationResult.rds'</span><span class="op">)</span><span class="op">)</span>
-              <span class="va">res</span> <span class="op">&lt;-</span> <span class="fu">PatientLevelPrediction</span><span class="fu">::</span><span class="fu">transportPlp</span><span class="op">(</span><span class="va">res</span>, n<span class="op">=</span> <span class="va">minCellCount</span>, 
-                                                          save <span class="op">=</span> <span class="cn">F</span>, dataName <span class="op">=</span> <span class="va">databaseName</span><span class="op">[</span><span class="va">i</span><span class="op">]</span><span class="op">)</span>
-              <span class="va">res</span><span class="op">$</span><span class="va">covariateSummary</span> <span class="op">&lt;-</span> <span class="va">res</span><span class="op">$</span><span class="va">covariateSummary</span><span class="op">[</span><span class="va">res</span><span class="op">$</span><span class="va">covariateSummary</span><span class="op">$</span><span class="va">covariateValue</span><span class="op">!=</span><span class="fl">0</span>,<span class="op">]</span>
-              <span class="fu"><a href="https://rdrr.io/r/base/readRDS.html">saveRDS</a></span><span class="op">(</span><span class="va">res</span>, <span class="fu"><a href="https://rdrr.io/r/base/file.path.html">file.path</a></span><span class="op">(</span><span class="va">outputDirectory</span>,<span class="st">'Validation'</span>,<span class="va">valOut</span>, <span class="st">'validationResult.rds'</span><span class="op">)</span><span class="op">)</span>
-            <span class="op">}</span>
-          <span class="op">}</span>
-        <span class="op">}</span>
-        
-      <span class="op">}</span>
-      
-    <span class="op">}</span>
-    
-  <span class="op">}</span>
-  <span class="op">}</span>
-  
-  <span class="kw"><a href="https://rdrr.io/r/base/function.html">return</a></span><span class="op">(</span><span class="va">outputDirectory</span><span class="op">)</span>
-  
-<span class="op">}</span></code></pre></div>
-<div id="example-code-to-combine-multiple-results" class="section level2">
-<h2 class="hasAnchor">
-<a href="#example-code-to-combine-multiple-results" class="anchor"></a>Example code to combine multiple results</h2>
-<p>The following code will combine the results found in ‘C:/myResults’, ‘C:/myResults2’ and ‘C:/myResults3’ into the shiny project at ‘C:/R/library/myPackage/shiny/PLPViewer’:</p>
-<div class="sourceCode" id="cb6"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu">populateMultipleShinyApp</span><span class="op">(</span>shinyDirectory <span class="op">=</span> <span class="st">'C:/R/library/myPackage/shiny/PLPViewer'</span>,
-                                     resultDirectory <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="st">'C:/myResults'</span>,
-                                                         <span class="st">'C:/myResults2'</span>,
-                                                         <span class="st">'C:/myResults3'</span><span class="op">)</span>,
-                                     minCellCount <span class="op">=</span> <span class="fl">0</span>,
-                                     databaseName <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="st">'database1'</span>,<span class="st">'database2'</span>,<span class="st">'database3'</span><span class="op">)</span><span class="op">)</span></code></pre></div>
-</div>
-</div>
-<div id="manual-app-creation" class="section level1">
-<h1 class="hasAnchor">
-<a href="#manual-app-creation" class="anchor"></a>Manual App Creation</h1>
-<p>[instructions coming soon]</p>
-</div>
-<div id="acknowledgments" class="section level1">
-<h1 class="hasAnchor">
-<a href="#acknowledgments" class="anchor"></a>Acknowledgments</h1>
-<p>Considerable work has been dedicated to provide the <code>PatientLevelPrediction</code> package.</p>
-<div class="sourceCode" id="cb7"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/utils/citation.html">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></code></pre></div>
-<pre><code>## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design
-## and implementation of a standardized framework to generate and evaluate
-## patient-level prediction models using observational healthcare data."
-## _Journal of the American Medical Informatics Association_, *25*(8),
-## 969-975. &lt;URL: https://doi.org/10.1093/jamia/ocy032&gt;.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }</code></pre>
-<p><strong>Please reference this paper if you use the PLP Package in your work:</strong></p>
-<p><a href="http://dx.doi.org/10.1093/jamia/ocy032">Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.</a></p>
-</div>
-  </div>
-
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-
-        <nav id="toc" data-toggle="toc"><h2 data-toc-skip>Contents</h2>
-    </nav>
-</div>
-
-</div>
-
-
-
-      <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-</div>
-
-  
-
-
-  </body>
-</html>
diff --git a/docs/articles/CreatingShinyApp_files/header-attrs-2.11/header-attrs.js b/docs/articles/CreatingShinyApp_files/header-attrs-2.11/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/CreatingShinyApp_files/header-attrs-2.11/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/CreatingShinyApp_files/header-attrs-2.7/header-attrs.js b/docs/articles/CreatingShinyApp_files/header-attrs-2.7/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/CreatingShinyApp_files/header-attrs-2.7/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/CustomPredictionAlgorithms.html b/docs/articles/CustomPredictionAlgorithms.html
deleted file mode 100644
index c38730513..000000000
--- a/docs/articles/CustomPredictionAlgorithms.html
+++ /dev/null
@@ -1,463 +0,0 @@
-<!DOCTYPE html>
-<!-- Generated by pkgdown: do not edit by hand --><html lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-<title>Custom patient-level prediction algorithms • PatientLevelPrediction</title>
-<!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous">
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous">
-<!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script><!-- sticky kit --><script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script><meta property="og:title" content="Custom patient-level prediction algorithms">
-<meta property="og:description" content="">
-<meta name="twitter:card" content="summary">
-<!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-</head>
-<body>
-    <div class="container template-article">
-      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.1</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-<li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-<li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/AddingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction framework</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CustomPredictionAlgorithms.html">Custom patient-level prediction algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/PlottingLearningCurves.html">Plotting learning curves</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-<ul class="nav navbar-nav navbar-right">
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-</div>
-<!--/.nav-collapse -->
-  </div>
-<!--/.container -->
-</div>
-<!--/.navbar -->
-
-      
-      </header><div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header toc-ignore">
-      <h1>Custom patient-level prediction algorithms</h1>
-                        <h4 class="author">Jenna Reps</h4>
-            
-            <h4 class="date">2019-03-04</h4>
-      
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/CustomPredictionAlgorithms.Rmd"><code>vignettes/CustomPredictionAlgorithms.Rmd</code></a></small>
-      <div class="hidden name"><code>CustomPredictionAlgorithms.Rmd</code></div>
-
-    </div>
-
-    
-    
-<!--
-%\VignetteEngine{knitr}
-%\VignetteIndexEntry{Custom patient-level prediction algorithms}
--->
-
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
-<p>The <code>PatientLevelPrediction</code> package enables data extraction, model building, and model evaluation using data from databases that are translated into the Observational Medical Outcomes Partnership Common Data Model (OMOP CDM).</p>
-<p>This vignette describes how you can add custom algorithms to the list of available algorithms in the <code>PatientLevelPrediction</code> package. This would allow you to fully leverage the OHDSI PatientLevelPrediction model development process with your own favourite algorithm.</p>
-<p>Of course, we invite you to share your new algorithms with the community through the GitHub repository.</p>
-</div>
-<div id="general-structure" class="section level1">
-<h1 class="hasAnchor">
-<a href="#general-structure" class="anchor"></a>General Structure</h1>
-<p>To add a custom classifier to the package you need to add the set, the fit functions into a R file with the model name. You also need to ensure there is a corresponding predict function in predict.R. For example, if you were to make a made up model, then in MadeUp.R you would add the following models:</p>
-<div id="set" class="section level2">
-<h2 class="hasAnchor">
-<a href="#set" class="anchor"></a>set</h2>
-<p>The setNewModel is a function that takes as input the different hyper-parameter values to do a grid search when training the model. The output of the model a list as class ‘modelSettings’ containing: + param - all the combinations of the hyper-parameter values input + model - a string specifying what function to call to fit the model + name - a string containing the name of the model.<br>
-For example, if you were adding a model call madeUp that had two hyper-parameters then the set function would be:</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">setMadeUp &lt;-<span class="st"> </span><span class="cf">function</span>(<span class="dt">a=</span><span class="dv">1</span>, <span class="dt">b=</span><span class="dv">2</span>, <span class="dt">seed=</span><span class="ot">NULL</span>){
-  <span class="co"># add input checks here...</span>
-  
-  <span class="co"># now create list of all combinations:</span>
-  result &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/list">list</a></span>(<span class="dt">model=</span><span class="st">'fitMadeUp'</span>, <span class="co"># this will be calle to train the made up model</span>
-                 <span class="dt">param=</span> <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/split">split</a></span>(<span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/expand.grid">expand.grid</a></span>(<span class="dt">a=</span>a, 
-                                          <span class="dt">b=</span>b,
-                                          <span class="dt">seed=</span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/ifelse">ifelse</a></span>(<span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/NULL">is.null</a></span>(seed),<span class="st">'NULL'</span>, seed)),
-                              <span class="dv">1</span><span class="op">:</span>(<span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/length">length</a></span>(a)<span class="op">*</span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/length">length</a></span>(b)  )),
-                 <span class="dt">name=</span><span class="st">'Made Up Algorithm'</span>
-  )
-  <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/class">class</a></span>(result) &lt;-<span class="st"> 'modelSettings'</span> 
-  
-  <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/function">return</a></span>(result)
-}</code></pre></div>
-</div>
-<div id="fit" class="section level2">
-<h2 class="hasAnchor">
-<a href="#fit" class="anchor"></a>fit</h2>
-<p>fitNewModel this function takes as input: * population - the study popualation the model is being developed on * plpData - the plpData object * param - the hyper-parameters as a list of all combinations * quiet - T or F indicating whether to output progress * outcomeId - the outcome id * cohortId - the target population id</p>
-<p>then it trains a model for each param entry, picks the best param entry and trains a final model for that setting. The fit function returns a list of class plpModel with the following objects: * model - a trained model * modelSettings - a list containing the model and input param * trainCVAac - a value with the train AUC value * hyperParamSearch - a dataframe with the hyperparameter grid and corresponding AUCs * metaData - the metaData from the plpData object * populationSettings - the settings used to create the population and define the time-at-risk * outcomeId - the outcomeId being predicted * cohortId - the cohortId corresponding to the target cohort * varImp - a dataframe with the covaraites and a measure of importance * trainingTime - how long it took to develop/evaluate the model * covariateMap - if the plpData are converted to a matrix for model compatibility this tells us what covariate each row in the matrix correpsonds to and is need when implementing the model on new data The plpModle returned by fit also has a type attribute, this points to the predict function, for example <code>attr(result, 'type') &lt;- 'madeup'</code> means when the model is applied to new data, the ‘predict.madeup’ function in Predict.R is called. if this doesnt exist, then the model will fail. Another attribute is the predictionType <code>attr(result, 'predictionType') &lt;- 'binary'</code> this is currently not needed but may be important in the future when we expand to regression or multiclass classification.</p>
-<p>The fit shell is:</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">fitMadeUp &lt;-<span class="st"> </span><span class="cf">function</span>(population, plpData, param, <span class="dt">quiet=</span>F,
-                        outcomeId, cohortId, ...){
-  
-  <span class="co"># **************** code to train the model here</span>
-  <span class="co"># trainedModel &lt;- this code should apply each hyper-param using the cross validation</span>
-  <span class="co">#                 then pick out the best hyper-param setting</span>
-  <span class="co">#                 and finally fit a model on the whole train data using the </span>
-  <span class="co">#                 optimal hyper-param settings</span>
-  <span class="co"># ****************</span>
-  
-  <span class="co"># construct the standard output for a model:</span>
-  result &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/list">list</a></span>(<span class="dt">model =</span> trainedModel,
-                 <span class="dt">modelSettings =</span> <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/list">list</a></span>(<span class="dt">model=</span><span class="st">'made_up'</span>, <span class="dt">modelParameters=</span>param),
-                 <span class="dt">trainCVAuc =</span> <span class="ot">NULL</span>,
-                 <span class="dt">hyperParamSearch =</span> hyperSummary,
-                 <span class="dt">metaData =</span> plpData<span class="op">$</span>metaData,
-                 <span class="dt">populationSettings =</span> <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/attr">attr</a></span>(population, <span class="st">'metaData'</span>),
-                 <span class="dt">outcomeId=</span>outcomeId,<span class="co"># can use populationSettings$outcomeId?</span>
-                 <span class="dt">cohortId=</span>cohortId,
-                 <span class="dt">varImp =</span> <span class="ot">NULL</span>,
-                 <span class="dt">trainingTime=</span>comp,
-                 <span class="dt">covariateMap=</span>result<span class="op">$</span>map
-  )
-  <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/class">class</a></span>(result) &lt;-<span class="st"> 'plpModel'</span>
-  <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/attr">attr</a></span>(result, <span class="st">'type'</span>) &lt;-<span class="st"> 'madeup'</span>
-  <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/attr">attr</a></span>(result, <span class="st">'predictionType'</span>) &lt;-<span class="st"> 'binary'</span>
-  <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/function">return</a></span>(result)
-    
-}</code></pre></div>
-<p>You can wish to make the fitMadeUp function cleaner by adding helper function in the MadeUp.R file that are called by the fit function. As the end of the fit function specified <code>attr(result, 'type') &lt;- 'madeup'</code> we also need to make sure there is a <code>predict.madeup</code> function in Predict.R:</p>
-</div>
-<div id="predict" class="section level2">
-<h2 class="hasAnchor">
-<a href="#predict" class="anchor"></a>predict</h2>
-<p>The prediction function takes as input the plpModel returned by fit, a population and corresponding plpData. It returns a data.frame with the columns: * rowId - the id for each person in the population * value - the predicted risk from the plpModel If the population contains the columns outcomeCount and indexes, then these are also output.</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">predict.madeup &lt;-<span class="st"> </span><span class="cf">function</span>(plpModel, population, plpData, ...) {
-    
-    <span class="co"># ************* code to do prediction for each rowId in population</span>
-    <span class="co"># prediction &lt;- code to do prediction here returning columns: rowId and</span>
-    <span class="co"># value (predicted risk) **************</span>
-    
-    prediction &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/merge">merge</a></span>(population, prediction, <span class="dt">by =</span> <span class="st">"rowId"</span>)
-    prediction &lt;-<span class="st"> </span>prediction[, <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/colnames">colnames</a></span>(prediction) <span class="op">%in%</span><span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/c">c</a></span>(<span class="st">"rowId"</span>, <span class="st">"outcomeCount"</span>, 
-        <span class="st">"indexes"</span>, <span class="st">"value"</span>)]
-    <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/attr">attr</a></span>(prediction, <span class="st">"metaData"</span>) &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/list">list</a></span>(<span class="dt">predictionType =</span> <span class="st">"binary"</span>)
-    <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/function">return</a></span>(prediction)
-    
-}</code></pre></div>
-</div>
-</div>
-<div id="r-model-example" class="section level1">
-<h1 class="hasAnchor">
-<a href="#r-model-example" class="anchor"></a>R Model Example</h1>
-<div id="set-1" class="section level2">
-<h2 class="hasAnchor">
-<a href="#set-1" class="anchor"></a>set</h2>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">setMadeUp &lt;-<span class="st"> </span><span class="cf">function</span>(<span class="dt">a=</span><span class="dv">1</span>, <span class="dt">b=</span><span class="dv">2</span>, <span class="dt">seed=</span><span class="ot">NULL</span>){
-  <span class="co"># check a is valid positive value</span>
-  <span class="cf">if</span>(<span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/missing">missing</a></span>(a)){
-    <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/stop">stop</a></span>(<span class="st">'a must be input'</span>)
-  }
-  <span class="cf">if</span>(<span class="op">!</span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/class">class</a></span>(a)<span class="op">%in%</span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/c">c</a></span>(<span class="st">'numeric'</span>,<span class="st">'integer'</span>){
-    <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/stop">stop</a></span>(<span class="st">'a must be numeric'</span>)
-  }
-  <span class="cf">if</span>(a <span class="op">&lt;</span><span class="st"> </span><span class="dv">0</span>){
-    <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/stop">stop</a></span>(<span class="st">'a must be positive'</span>)
-  }
-  <span class="co"># check b is numeric</span>
-  <span class="cf">if</span>(<span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/missing">missing</a></span>(b)){
-    <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/stop">stop</a></span>(<span class="st">'b must be input'</span>)
-  }
-  <span class="cf">if</span>(<span class="op">!</span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/class">class</a></span>(b)<span class="op">%in%</span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/c">c</a></span>(<span class="st">'numeric'</span>,<span class="st">'integer'</span>){
-    <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/stop">stop</a></span>(<span class="st">'b must be numeric'</span>)
-  }
-  
-  <span class="co"># now create list of all combinations:</span>
-  result &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/list">list</a></span>(<span class="dt">model=</span><span class="st">'fitMadeUp'</span>, 
-                 <span class="dt">param=</span> <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/split">split</a></span>(<span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/expand.grid">expand.grid</a></span>(<span class="dt">a=</span>a, 
-                                          <span class="dt">b=</span>b,
-                                          <span class="dt">seed=</span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/ifelse">ifelse</a></span>(<span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/NULL">is.null</a></span>(seed),<span class="st">'NULL'</span>, seed)),
-                              <span class="dv">1</span><span class="op">:</span>(<span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/length">length</a></span>(a)<span class="op">*</span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/length">length</a></span>(b)  )),
-                 <span class="dt">name=</span><span class="st">'Made Up Algorithm'</span>
-  )
-  <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/class">class</a></span>(result) &lt;-<span class="st"> 'modelSettings'</span> 
-  
-  <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/function">return</a></span>(result)
-    
-  
-<span class="er">}</span></code></pre></div>
-</div>
-<div id="fit-1" class="section level2">
-<h2 class="hasAnchor">
-<a href="#fit-1" class="anchor"></a>fit</h2>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">fitMadeUp &lt;-<span class="st"> </span><span class="cf">function</span>(population, plpData, param, <span class="dt">quiet=</span>F,
-                        outcomeId, cohortId, ...){
-    <span class="cf">if</span>(<span class="op">!</span>quiet)
-    <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/writeLines">writeLines</a></span>(<span class="st">'Training Made Up model'</span>)
-  
-  <span class="cf">if</span>(param[[<span class="dv">1</span>]]<span class="op">$</span>seed<span class="op">!=</span><span class="st">'NULL'</span>)
-    <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/Random">set.seed</a></span>(param[[<span class="dv">1</span>]]<span class="op">$</span>seed)
-  
-    <span class="co"># check plpData is coo format:</span>
-  <span class="cf">if</span>(<span class="op">!</span><span class="st">'ffdf'</span><span class="op">%in%</span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/class">class</a></span>(plpData<span class="op">$</span>covariates) )
-    <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/stop">stop</a></span>(<span class="st">'This algorithm requires plpData in coo format'</span>)
-  
-  metaData &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/attr">attr</a></span>(population, <span class="st">'metaData'</span>)
-  <span class="cf">if</span>(<span class="op">!</span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/NULL">is.null</a></span>(population<span class="op">$</span>indexes))
-    population &lt;-<span class="st"> </span>population[population<span class="op">$</span>indexes<span class="op">&gt;</span><span class="dv">0</span>,]
-  <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/attr">attr</a></span>(population, <span class="st">'metaData'</span>) &lt;-<span class="st"> </span>metaData
-  <span class="co">#TODO - how to incorporate indexes?</span>
-  
-  <span class="co"># convert data into sparse R Matrix:</span>
-  result &lt;-<span class="st"> </span><span class="kw"><a href="../reference/toSparseM.html">toSparseM</a></span>(plpData,population,<span class="dt">map=</span><span class="ot">NULL</span>)
-  data &lt;-<span class="st"> </span>result<span class="op">$</span>data
-  
-  data &lt;-<span class="st"> </span>data[population<span class="op">$</span>rowId,]
-  
-  <span class="co"># set test/train sets (for printing performance as it trains)</span>
-  <span class="cf">if</span>(<span class="op">!</span>quiet)
-    <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/writeLines">writeLines</a></span>(<span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/paste">paste0</a></span>(<span class="st">'Training made up model on train set containing '</span>, <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/nrow">nrow</a></span>(population), <span class="st">' people with '</span>,<span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/sum">sum</a></span>(population<span class="op">$</span>outcomeCount<span class="op">&gt;</span><span class="dv">0</span>), <span class="st">' outcomes'</span>))
-  start &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/Sys.time">Sys.time</a></span>()
-  
-  <span class="co">#============= STEP 1 ======================================</span>
-  <span class="co"># pick the best hyper-params and then do final training on all data...</span>
-  <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/writeLines">writeLines</a></span>(<span class="st">'train'</span>)
-  datas &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/list">list</a></span>(<span class="dt">population=</span>population, <span class="dt">data=</span>data)
-  param.sel &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/lapply">lapply</a></span>(param, <span class="cf">function</span>(x) <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/do.call">do.call</a></span>(made_up_model, <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/c">c</a></span>(x,datas)  ))
-  hyperSummary &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/do.call">do.call</a></span>(rbind, <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/lapply">lapply</a></span>(param.sel, <span class="cf">function</span>(x) x<span class="op">$</span>hyperSum))
-  hyperSummary &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/as.data.frame">as.data.frame</a></span>(hyperSummary)
-  hyperSummary<span class="op">$</span>auc &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/unlist">unlist</a></span>(<span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/lapply">lapply</a></span>(param.sel, <span class="cf">function</span>(x) x<span class="op">$</span>auc)) 
-  param.sel &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/unlist">unlist</a></span>(<span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/lapply">lapply</a></span>(param.sel, <span class="cf">function</span>(x) x<span class="op">$</span>auc))
-  param &lt;-<span class="st"> </span>param[[<span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/which.min">which.max</a></span>(param.sel)]]
-  
-  <span class="co"># set this so you do a final model train </span>
-  param<span class="op">$</span>final=T
-  
-  <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/writeLines">writeLines</a></span>(<span class="st">'final train'</span>)
-  trainedModel &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/do.call">do.call</a></span>(made_up_model, <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/c">c</a></span>(param,datas)  )<span class="op">$</span>model
-  
-  comp &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/Sys.time">Sys.time</a></span>() <span class="op">-</span><span class="st"> </span>start
-  <span class="cf">if</span>(<span class="op">!</span>quiet)
-    <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/writeLines">writeLines</a></span>(<span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/paste">paste0</a></span>(<span class="st">'Model Made Up trained - took:'</span>,  <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/format">format</a></span>(comp, <span class="dt">digits=</span><span class="dv">3</span>)))
-  
-  <span class="co"># construct the standard output for a model:</span>
-  result &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/list">list</a></span>(<span class="dt">model =</span> trainedModel,
-                 <span class="dt">modelSettings =</span> <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/list">list</a></span>(<span class="dt">model=</span><span class="st">'made_up'</span>, <span class="dt">modelParameters=</span>param),
-                 <span class="dt">trainCVAuc =</span> <span class="ot">NULL</span>,
-                 <span class="dt">hyperParamSearch =</span> hyperSummary,
-                 <span class="dt">metaData =</span> plpData<span class="op">$</span>metaData,
-                 <span class="dt">populationSettings =</span> <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/attr">attr</a></span>(population, <span class="st">'metaData'</span>),
-                 <span class="dt">outcomeId=</span>outcomeId,<span class="co"># can use populationSettings$outcomeId?</span>
-                 <span class="dt">cohortId=</span>cohortId,
-                 <span class="dt">varImp =</span> <span class="ot">NULL</span>,
-                 <span class="dt">trainingTime=</span>comp,
-                 <span class="dt">covariateMap=</span>result<span class="op">$</span>map
-  )
-  <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/class">class</a></span>(result) &lt;-<span class="st"> 'plpModel'</span>
-  <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/attr">attr</a></span>(result, <span class="st">'type'</span>) &lt;-<span class="st"> 'madeup'</span>
-  <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/attr">attr</a></span>(result, <span class="st">'predictionType'</span>) &lt;-<span class="st"> 'binary'</span>
-  <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/function">return</a></span>(result)
-    
-}</code></pre></div>
-</div>
-<div id="helpers" class="section level2">
-<h2 class="hasAnchor">
-<a href="#helpers" class="anchor"></a>helpers</h2>
-<p>In the fit model I specified calling <code>made_up_model</code>, this is the function that trains a model given the data and population (where the popualtion contains a column outcomeCount corresponding to the outcome). Both the data and population are ordered the same way:</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">made_up_model &lt;-<span class="st"> </span><span class="cf">function</span>(data, population, <span class="dt">a =</span> <span class="dv">1</span>, <span class="dt">b =</span> <span class="dv">1</span>, <span class="dt">final =</span> F, ...) {
-    
-    <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/writeLines">writeLines</a></span>(<span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/paste">paste</a></span>(<span class="st">"Training Made Up model with "</span>, <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/length">length</a></span>(<span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/unique">unique</a></span>(population<span class="op">$</span>indexes)), 
-        <span class="st">" fold CV"</span>))
-    <span class="cf">if</span> (<span class="op">!</span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/NULL">is.null</a></span>(population<span class="op">$</span>indexes) <span class="op">&amp;&amp;</span><span class="st"> </span>final <span class="op">==</span><span class="st"> </span>F) {
-        index_vect &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/unique">unique</a></span>(population<span class="op">$</span>indexes)
-        perform &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/c">c</a></span>()
-        
-        <span class="co"># create prediction matrix to store all predictions</span>
-        predictionMat &lt;-<span class="st"> </span>population
-        predictionMat<span class="op">$</span>value &lt;-<span class="st"> </span><span class="dv">0</span>
-        <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/attr">attr</a></span>(predictionMat, <span class="st">"metaData"</span>) &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/list">list</a></span>(<span class="dt">predictionType =</span> <span class="st">"binary"</span>)
-        
-        <span class="cf">for</span> (index <span class="cf">in</span> <span class="dv">1</span><span class="op">:</span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/length">length</a></span>(index_vect)) {
-            <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/writeLines">writeLines</a></span>(<span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/paste">paste</a></span>(<span class="st">"Fold "</span>, index, <span class="st">" -- with "</span>, <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/sum">sum</a></span>(population<span class="op">$</span>indexes <span class="op">!=</span><span class="st"> </span>
-<span class="st">                </span>index), <span class="st">"train rows"</span>))
-            model &lt;-<span class="st"> </span>madeup<span class="op">::</span><span class="kw">model</span>(<span class="dt">x =</span> data[population<span class="op">$</span>indexes <span class="op">!=</span><span class="st"> </span>index, ], 
-                <span class="dt">y =</span> population<span class="op">$</span>outcomeCount[population<span class="op">$</span>indexes <span class="op">!=</span><span class="st"> </span>index], <span class="dt">a =</span> a, 
-                <span class="dt">b =</span> b)
-            
-            pred &lt;-<span class="st"> </span>stats<span class="op">::</span><span class="kw"><a href="https://www.rdocumentation.org/packages/stats/topics/predict">predict</a></span>(model, data[population<span class="op">$</span>indexes <span class="op">==</span><span class="st"> </span>index, 
-                ])
-            prediction &lt;-<span class="st"> </span>population[population<span class="op">$</span>indexes <span class="op">==</span><span class="st"> </span>index, ]
-            prediction<span class="op">$</span>value &lt;-<span class="st"> </span>pred
-            <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/attr">attr</a></span>(prediction, <span class="st">"metaData"</span>) &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/list">list</a></span>(<span class="dt">predictionType =</span> <span class="st">"binary"</span>)
-            aucVal &lt;-<span class="st"> </span><span class="kw"><a href="../reference/computeAuc.html">computeAuc</a></span>(prediction)
-            perform &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/c">c</a></span>(perform, aucVal)
-            
-            <span class="co"># add the fold predictions and compute AUC after loop</span>
-            predictionMat<span class="op">$</span>value[population<span class="op">$</span>indexes <span class="op">==</span><span class="st"> </span>index] &lt;-<span class="st"> </span>pred
-            
-        }
-        ## auc &lt;- mean(perform) # want overal rather than mean
-        auc &lt;-<span class="st"> </span><span class="kw"><a href="../reference/computeAuc.html">computeAuc</a></span>(predictionMat)
-        
-        foldPerm &lt;-<span class="st"> </span>perform
-    } <span class="cf">else</span> {
-        model &lt;-<span class="st"> </span>madeup<span class="op">::</span><span class="kw">model</span>(<span class="dt">x =</span> data, <span class="dt">y =</span> population<span class="op">$</span>outcomeCount, <span class="dt">a =</span> a, 
-            <span class="dt">b =</span> b)
-        
-        pred &lt;-<span class="st"> </span>stats<span class="op">::</span><span class="kw"><a href="https://www.rdocumentation.org/packages/stats/topics/predict">predict</a></span>(model, data)
-        prediction &lt;-<span class="st"> </span>population
-        prediction<span class="op">$</span>value &lt;-<span class="st"> </span>pred
-        <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/attr">attr</a></span>(prediction, <span class="st">"metaData"</span>) &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/list">list</a></span>(<span class="dt">predictionType =</span> <span class="st">"binary"</span>)
-        auc &lt;-<span class="st"> </span><span class="kw"><a href="../reference/computeAuc.html">computeAuc</a></span>(prediction)
-        foldPerm &lt;-<span class="st"> </span>auc
-    }
-    
-    result &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/list">list</a></span>(<span class="dt">model =</span> model, <span class="dt">auc =</span> auc, <span class="dt">hyperSum =</span> <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/unlist">unlist</a></span>(<span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/list">list</a></span>(<span class="dt">a =</span> a, <span class="dt">b =</span> b, 
-        <span class="dt">fold_auc =</span> foldPerm)))
-    <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/function">return</a></span>(result)
-}</code></pre></div>
-</div>
-<div id="predict-1" class="section level2">
-<h2 class="hasAnchor">
-<a href="#predict-1" class="anchor"></a>Predict</h2>
-<p>The final step is to create a predict function for the model. This gets added to the predict.R file. In the example above the type <code>attr(result, 'type') &lt;- 'madeup'</code> was madeup, so a <code>predict.madeup</code> function is required to be added into the predict.R. The predict function needs to take as input the plpModel returned by the fit function, the population to apply the model on and the plpData specifying the covariates of the population.</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">predict.madeup &lt;-<span class="st"> </span><span class="cf">function</span>(plpModel, population, plpData, ...) {
-    result &lt;-<span class="st"> </span><span class="kw"><a href="../reference/toSparseM.html">toSparseM</a></span>(plpData, population, <span class="dt">map =</span> plpModel<span class="op">$</span>covariateMap)
-    data &lt;-<span class="st"> </span>result<span class="op">$</span>data[population<span class="op">$</span>rowId, ]
-    prediction &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/data.frame">data.frame</a></span>(<span class="dt">rowId =</span> population<span class="op">$</span>rowId, <span class="dt">value =</span> stats<span class="op">::</span><span class="kw"><a href="https://www.rdocumentation.org/packages/stats/topics/predict">predict</a></span>(plpModel<span class="op">$</span>model, 
-        data))
-    
-    prediction &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/merge">merge</a></span>(population, prediction, <span class="dt">by =</span> <span class="st">"rowId"</span>)
-    prediction &lt;-<span class="st"> </span>prediction[, <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/colnames">colnames</a></span>(prediction) <span class="op">%in%</span><span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/c">c</a></span>(<span class="st">"rowId"</span>, <span class="st">"outcomeCount"</span>, 
-        <span class="st">"indexes"</span>, <span class="st">"value"</span>)]  <span class="co"># need to fix no index issue</span>
-    <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/attr">attr</a></span>(prediction, <span class="st">"metaData"</span>) &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/list">list</a></span>(<span class="dt">predictionType =</span> <span class="st">"binary"</span>)
-    <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/function">return</a></span>(prediction)
-    
-}</code></pre></div>
-<p>As the madeup model uses the standard R prediction, it has the same prediction function as xgboost, so we could have not added a new prediction function and instead made the tpye of the result returned by fitMadeUpModel to <code>attr(result, 'type') &lt;- 'xgboost'</code>.</p>
-</div>
-</div>
-  </div>
-
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-        <div id="tocnav">
-      <h2 class="hasAnchor">
-<a href="#tocnav" class="anchor"></a>Contents</h2>
-      <ul class="nav nav-pills nav-stacked">
-<li><a href="#introduction">Introduction</a></li>
-      <li>
-<a href="#general-structure">General Structure</a><ul class="nav nav-pills nav-stacked">
-<li><a href="#set">set</a></li>
-      <li><a href="#fit">fit</a></li>
-      <li><a href="#predict">predict</a></li>
-      </ul>
-</li>
-      <li>
-<a href="#r-model-example">R Model Example</a><ul class="nav nav-pills nav-stacked">
-<li><a href="#set-1">set</a></li>
-      <li><a href="#fit-1">fit</a></li>
-      <li><a href="#helpers">helpers</a></li>
-      <li><a href="#predict-1">Predict</a></li>
-      </ul>
-</li>
-      </ul>
-</div>
-      </div>
-
-</div>
-
-
-      <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-</div>
-
-  
-
-  </body>
-</html>
diff --git a/docs/articles/Figure1.png b/docs/articles/Figure1.png
deleted file mode 100644
index 878a509e2..000000000
Binary files a/docs/articles/Figure1.png and /dev/null differ
diff --git a/docs/articles/Figure1.webp b/docs/articles/Figure1.webp
new file mode 100644
index 000000000..42ad71d7f
Binary files /dev/null and b/docs/articles/Figure1.webp differ
diff --git a/docs/articles/GeneratingLearningCurves.html b/docs/articles/GeneratingLearningCurves.html
deleted file mode 100644
index cb9467f1d..000000000
--- a/docs/articles/GeneratingLearningCurves.html
+++ /dev/null
@@ -1,294 +0,0 @@
-<!DOCTYPE html>
-<!-- Generated by pkgdown: do not edit by hand --><html lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-<title>Generating Learning Curves • PatientLevelPrediction</title>
-<!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous">
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous">
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous">
-<!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script><meta property="og:title" content="Generating Learning Curves">
-<meta property="og:description" content="PatientLevelPrediction">
-<!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-</head>
-<body data-spy="scroll" data-target="#toc">
-    <div class="container template-article">
-      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.0.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-<li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-<li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-<ul class="nav navbar-nav navbar-right">
-<li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-</div>
-<!--/.nav-collapse -->
-  </div>
-<!--/.container -->
-</div>
-<!--/.navbar -->
-
-      
-
-      </header><div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header toc-ignore">
-      <h1 data-toc-skip>Generating Learning Curves</h1>
-                        <h4 class="author">Luis H. John, Xiaoyong Pan, Jenna Reps, Peter R. Rijnbeek</h4>
-            
-            <h4 class="date">2020-06-08</h4>
-      
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/GeneratingLearningCurves.Rmd"><code>vignettes/GeneratingLearningCurves.Rmd</code></a></small>
-      <div class="hidden name"><code>GeneratingLearningCurves.Rmd</code></div>
-
-    </div>
-
-    
-    
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
-<p>This vignette describes how you can use the Observational Health Data Sciences and Informatics (OHDSI) <a href="http://github.com/OHDSI/PatientLevelPrediction"><code>PatientLevelPrediction</code></a> package to generate learning curves. This vignette assumes you have read and are comfortable with building patient level prediction models as described in the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf"><code>BuildingPredictiveModels</code> vignette</a>.</p>
-<p>Prediction models will show overly-optimistic performance when predicting on the same data as used for training. Therefore, best-practice is to partition our data into a training set and testing set. We then train our prediction model on the training set portion and asses its ability to generalize to unseen data by measuring its performance on the testing set.</p>
-<p>Learning curves assess the effect of training set size on model performance by training a sequence of prediction models on successively larger subsets of the training set. A learning curve plot can also help in diagnosing a bias or variance problem as explained below.</p>
-<div class="figure">
-<img src="learningCurve.png" alt=""><p class="caption">Learning curve example.</p>
-</div>
-<p>Figure 1, shows an example of learning curve plot in which the vertical axis represents the model performance and the horizontal axis the training set size. If training set size is small, the performance on the training set is high, because a model can often be fitted well to a limited number of training examples. At the same time, the performance on the testing set will be poor, because the model trained on such a limited number of training examples will not generalize well to unseen data in the testing set. As the training set size increases, the performance of the model on the training set will decrease. It becomes more difficult for the model to find a good fit through all the training examples. Also, the model will be trained on a more representative portion of training examples, making it generalize better to unseen data. This can be observed by the increasin testing set performance.</p>
-<p>The learning curve can help us in diagnosing bias and variance problems with our classifier which will provide guidance on how to further improve our model. We can observe high variance (overfitting) in a prediction model if it performs well on the training set, but poorly on the testing set (Figure 2). Adding additional data is a common approach to counteract high variance. From the learning curve it becomes apparent, that adding additional data may improve performance on the testing set a little further, as the learning curve has not yet plateaued and, thus, the model is not saturated yet. Therefore, adding more data will decrease the gap between training set and testing set, which is the main indicator for a high variance problem.</p>
-<div class="figure">
-<img src="learningCurveVariance.png" alt=""><p class="caption">Prediction model suffering from high variance.</p>
-</div>
-<p>Furthermore, we can observe high bias (underfitting) if a prediction model performs poorly on the training set as well as on the testing set (Figure 3). The learning curves of training set and testing set have flattened on a low performance with only a small gap in between them. Adding additional data will in this case have little to no impact on the model performance. Choosing another prediction algorithm that can find more complex (for example non-linear) relationships in the data may be an alternative approach to consider in this high bias situation.</p>
-<div class="figure">
-<img src="learningCurveBias.png" alt=""><p class="caption">Prediction model suffering from high bias.</p>
-</div>
-</div>
-<div id="generating-the-learning-curve" class="section level1">
-<h1 class="hasAnchor">
-<a href="#generating-the-learning-curve" class="anchor"></a>Generating the learning curve</h1>
-<p>Use the <a href="http://github.com/OHDSI/PatientLevelPrediction"><code>PatientLevelPrediction</code></a> package to generate a <code>population</code> and <code>plpData</code> object. Alternatively, you can make use of the data simulator. The following code snippet creates a population of 12000 patients.</p>
-<div class="sourceCode" id="cb1"><html><body><pre class="r"><span class="fu"><a href="https://rdrr.io/r/base/Random.html">set.seed</a></span>(<span class="fl">1234</span>)
-<span class="fu"><a href="https://rdrr.io/r/utils/data.html">data</a></span>(<span class="no">plpDataSimulationProfile</span>)
-<span class="no">sampleSize</span> <span class="kw">&lt;-</span> <span class="fl">12000</span>
-<span class="no">plpData</span> <span class="kw">&lt;-</span> <span class="fu"><a href="../reference/simulatePlpData.html">simulatePlpData</a></span>(
-  <span class="no">plpDataSimulationProfile</span>,
-  <span class="kw">n</span> <span class="kw">=</span> <span class="no">sampleSize</span>
-)
-
-<span class="no">population</span> <span class="kw">&lt;-</span> <span class="fu"><a href="../reference/createStudyPopulation.html">createStudyPopulation</a></span>(
-  <span class="no">plpData</span>,
-  <span class="kw">outcomeId</span> <span class="kw">=</span> <span class="fl">2</span>,
-  <span class="kw">binary</span> <span class="kw">=</span> <span class="fl">TRUE</span>,
-  <span class="kw">firstExposureOnly</span> <span class="kw">=</span> <span class="fl">FALSE</span>,
-  <span class="kw">washoutPeriod</span> <span class="kw">=</span> <span class="fl">0</span>,
-  <span class="kw">removeSubjectsWithPriorOutcome</span> <span class="kw">=</span> <span class="fl">FALSE</span>,
-  <span class="kw">priorOutcomeLookback</span> <span class="kw">=</span> <span class="fl">99999</span>,
-  <span class="kw">requireTimeAtRisk</span> <span class="kw">=</span> <span class="fl">FALSE</span>,
-  <span class="kw">minTimeAtRisk</span> <span class="kw">=</span> <span class="fl">0</span>,
-  <span class="kw">riskWindowStart</span> <span class="kw">=</span> <span class="fl">0</span>,
-  <span class="kw">addExposureDaysToStart</span> <span class="kw">=</span> <span class="fl">FALSE</span>,
-  <span class="kw">riskWindowEnd</span> <span class="kw">=</span> <span class="fl">365</span>,
-  <span class="kw">addExposureDaysToEnd</span> <span class="kw">=</span> <span class="fl">FALSE</span>,
-  <span class="kw">verbosity</span> <span class="kw">=</span> <span class="st">"INFO"</span>
-)</pre></body></html></div>
-<p>Specify the prediction algorithm to be used.</p>
-<div class="sourceCode" id="cb2"><html><body><pre class="r"><span class="co"># Use LASSO logistic regression</span>
-<span class="no">modelSettings</span> <span class="kw">&lt;-</span> <span class="fu"><a href="../reference/setLassoLogisticRegression.html">setLassoLogisticRegression</a></span>()</pre></body></html></div>
-<p>Specify a test fraction and a sequence of training set fractions.</p>
-<div class="sourceCode" id="cb3"><html><body><pre class="r"><span class="no">testFraction</span> <span class="kw">&lt;-</span> <span class="fl">0.2</span>
-<span class="no">trainFractions</span> <span class="kw">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/seq.html">seq</a></span>(<span class="fl">0.1</span>, <span class="fl">0.8</span>, <span class="fl">0.1</span>) <span class="co"># Create eight training set fractions</span></pre></body></html></div>
-<p>Specify the test split to be used.</p>
-<div class="sourceCode" id="cb4"><html><body><pre class="r"><span class="co"># Use a split by person, alternatively a time split is possible</span>
-<span class="no">testSplit</span> <span class="kw">&lt;-</span> <span class="st">'stratified'</span></pre></body></html></div>
-<p>Create the learning curve object.</p>
-<div class="sourceCode" id="cb5"><html><body><pre class="r"><span class="no">learningCurve</span> <span class="kw">&lt;-</span> <span class="fu"><a href="../reference/createLearningCurve.html">createLearningCurve</a></span>(<span class="no">population</span>,
-                                     <span class="kw">plpData</span> <span class="kw">=</span> <span class="no">plpData</span>,
-                                     <span class="kw">modelSettings</span> <span class="kw">=</span> <span class="no">modelSettings</span>,
-                                     <span class="kw">testFraction</span> <span class="kw">=</span> <span class="fl">0.2</span>,
-                                     <span class="kw">verbosity</span> <span class="kw">=</span> <span class="st">"TRACE"</span>,
-                                     <span class="kw">trainFractions</span> <span class="kw">=</span> <span class="no">trainFractions</span>,
-                                     <span class="kw">splitSeed</span> <span class="kw">=</span> <span class="fl">1000</span>,
-                                     <span class="kw">saveModel</span> <span class="kw">=</span> <span class="fl">TRUE</span>)</pre></body></html></div>
-<p>Plot the learning curve object (Figure 4). Specify one of the available metrics: <code>AUROC</code>, <code>AUPRC</code>, <code>sBrier</code>.</p>
-<div class="sourceCode" id="cb6"><html><body><pre class="r"><span class="fu"><a href="../reference/plotLearningCurve.html">plotLearningCurve</a></span>(
-  <span class="no">learningCurve</span>,
-  <span class="kw">metric</span><span class="kw">=</span><span class="st">'AUROC'</span>,
-  <span class="kw">plotTitle</span> <span class="kw">=</span> <span class="st">'Learning Curve'</span>,
-  <span class="kw">plotSubtitle</span> <span class="kw">=</span> <span class="st">'AUROC performance'</span>
-)</pre></body></html></div>
-<div class="figure">
-<img src="learningCurvePlot.png" alt=""><p class="caption">Learning curve plot.</p>
-</div>
-</div>
-<div id="parallel-processing" class="section level1">
-<h1 class="hasAnchor">
-<a href="#parallel-processing" class="anchor"></a>Parallel processing</h1>
-<p>The learning curve object can be created in parallel, which can reduce computation time significantly. Currently this functionality is only available for LASSO logistic regression. Depending on the number of parallel workers it may require a significant amount of memory. We advise to use the parallelized learning curve function for parameter search and exploratory data analysis. Logging and saving functionality is unavailable.</p>
-<p>Use the parallelized version of the learning curve function to create the learning curve object in parallel. R will find the number of available processing cores automatically and register the required parallel backend.</p>
-<div class="sourceCode" id="cb7"><html><body><pre class="r"><span class="no">learningCurvePar</span> <span class="kw">&lt;-</span> <span class="fu"><a href="../reference/createLearningCurvePar.html">createLearningCurvePar</a></span>(
-  <span class="no">population</span>,
-  <span class="kw">plpData</span> <span class="kw">=</span>  <span class="no">plpData</span>,
-  <span class="kw">modelSettings</span> <span class="kw">=</span> <span class="no">modelSettings</span>,
-  <span class="kw">testSplit</span> <span class="kw">=</span> <span class="no">testSplit</span>,
-  <span class="kw">testFraction</span> <span class="kw">=</span> <span class="no">testFraction</span>,
-  <span class="kw">trainFractions</span> <span class="kw">=</span> <span class="no">trainFractions</span>,
-  <span class="kw">splitSeed</span> <span class="kw">=</span> <span class="fl">1000</span>
-)</pre></body></html></div>
-</div>
-<div id="demo" class="section level1">
-<h1 class="hasAnchor">
-<a href="#demo" class="anchor"></a>Demo</h1>
-<p>We have added a demo of the learningcurve:</p>
-<div class="sourceCode" id="cb8"><html><body><pre class="r"><span class="co"># Show all demos in our package: </span>
- <span class="fu"><a href="https://rdrr.io/r/utils/demo.html">demo</a></span>(<span class="kw">package</span> <span class="kw">=</span> <span class="st">"PatientLevelPrediction"</span>)
-
-<span class="co"># Run the learning curve</span>
- <span class="fu"><a href="https://rdrr.io/r/utils/demo.html">demo</a></span>(<span class="st">"LearningCurveDemo"</span>, <span class="kw">package</span> <span class="kw">=</span> <span class="st">"PatientLevelPrediction"</span>)</pre></body></html></div>
-<p>Do note that running this demo can take a considerable amount of time (15 min on Quad core running in parallel)!</p>
-</div>
-<div id="acknowledgments" class="section level1">
-<h1 class="hasAnchor">
-<a href="#acknowledgments" class="anchor"></a>Acknowledgments</h1>
-<p>Considerable work has been dedicated to provide the <code>PatientLevelPrediction</code> package.</p>
-<div class="sourceCode" id="cb9"><html><body><pre class="r"><span class="fu"><a href="https://rdrr.io/r/utils/citation.html">citation</a></span>(<span class="st">"PatientLevelPrediction"</span>)</pre></body></html></div>
-<pre><code>## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design
-## and implementation of a standardized framework to generate and evaluate
-## patient-level prediction models using observational healthcare data."
-## _Journal of the American Medical Informatics Association_, *25*(8),
-## 969-975. &lt;URL: https://doi.org/10.1093/jamia/ocy032&gt;.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }</code></pre>
-<p><strong>Please reference this paper if you use the PLP Package in your work:</strong></p>
-<p><a href="http://dx.doi.org/10.1093/jamia/ocy032">Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.</a></p>
-</div>
-  </div>
-
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-
-        <nav id="toc" data-toggle="toc"><h2 data-toc-skip>Contents</h2>
-    </nav>
-</div>
-
-</div>
-
-
-
-      <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.5.1.</p>
-</div>
-
-      </footer>
-</div>
-
-  
-
-
-  </body>
-</html>
diff --git a/docs/articles/ImplementingExistingModels.html b/docs/articles/ImplementingExistingModels.html
deleted file mode 100644
index 2bb38a32e..000000000
--- a/docs/articles/ImplementingExistingModels.html
+++ /dev/null
@@ -1,359 +0,0 @@
-<!DOCTYPE html>
-<!-- Generated by pkgdown: do not edit by hand --><html lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-<title>Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework • PatientLevelPrediction</title>
-<!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous">
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous">
-<!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script><!-- sticky kit --><script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script><meta property="og:title" content="Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework">
-<meta property="og:description" content="">
-<meta name="twitter:card" content="summary">
-<!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-</head>
-<body>
-    <div class="container template-article">
-      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.5</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-<li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-<li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-<ul class="nav navbar-nav navbar-right">
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-</div>
-<!--/.nav-collapse -->
-  </div>
-<!--/.container -->
-</div>
-<!--/.navbar -->
-
-      
-      </header><div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header toc-ignore">
-      <h1>Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</h1>
-                        <h4 class="author">Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek</h4>
-            
-            <h4 class="date">2019-05-30</h4>
-      
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/ImplementingExistingModels.Rmd"><code>vignettes/ImplementingExistingModels.Rmd</code></a></small>
-      <div class="hidden name"><code>ImplementingExistingModels.Rmd</code></div>
-
-    </div>
-
-    
-    
-<!--
-%\VignetteEngine{knitr}
-%\VignetteIndexEntry{Implementing existing patient-level predictive models}
--->
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
-<p>This vignette describes how you can implement existing logistic regression models in the Observational Health Data Sciences (OHDSI) <a href="http://www.github.com/OHDSI/PatientLevelPrediction"><code>PatientLevelPrediction</code></a> framework. This allows you to for example externally validate them at scale in the OHDSI data network.</p>
-<p>As an example we are going to implement the CHADS2 model as described in:</p>
-<p>Gage BF, Waterman AD, Shannon W, Boechler M, Rich MW, Radford MJ. Validation of clinical classification schemes for predicting stroke: results from the National Registry of Atrial Fibrillation. JAMA. 2001 Jun 13;285(22):2864-70</p>
-<p>To implement the model you need to create three tables: the model table, the covariate table, and the intercept table. The model table specifies the modelId (sequence number), the modelCovariateId (sequence number) and the covariateValue (beta for the covariate). The covariate table specifies the mapping between the covariates from the published model and the standard covariates, i.e. it maps to a combination of an analysisid and a concept_id (see below). The intercept table specifies per modelId the intercept.</p>
-</div>
-<div id="model-implementation" class="section level1">
-<h1 class="hasAnchor">
-<a href="#model-implementation" class="anchor"></a>Model implementation</h1>
-<div id="define-the-model" class="section level2">
-<h2 class="hasAnchor">
-<a href="#define-the-model" class="anchor"></a>Define the model</h2>
-<p>The CHADS2 is a score based model with:</p>
-<pre><code>##   Points                        Covariate
-## 1      1         Congestive heart failure
-## 2      1                     Hypertension
-## 3      1                  Age &gt;= 75 years
-## 4      1                Diabetes mellitus
-## 5      2 Stroke/transient ischemic attack</code></pre>
-<p>The model table should therefore be defined as:</p>
-<pre><code>##   modelId modelCovariateId covariateValue
-## 1       1                1              1
-## 2       1                2              1
-## 3       1                3              1
-## 4       1                4              1
-## 5       1                5              2</code></pre>
-<p>The covariateTable will then specify what standard covariates need to be included in the model.</p>
-<p>In this case we choose the following Standard SNOMED concept_ids: 319835 for congestive heart failure, 316866 for hypertensive disorder, 201820 for diabetes, and 381591 for cerebrovascular disease. It is allowed to add multiple concept_ids as separate rows for the same modelCovariateId if concept sets are needed. These concept_ids can be found using the vocabulary search in ATLAS.</p>
-<p>The standard covariates are of the form: conceptid*1000 + analysisid. The analysisid specifies the domain of the covariate and its lookback window. Examples can be found here: <a href="https://github.com/OHDSI/FeatureExtraction/blob/master/inst/csv/PrespecAnalyses.csv" class="uri">https://github.com/OHDSI/FeatureExtraction/blob/master/inst/csv/PrespecAnalyses.csv</a></p>
-<p>Our example of CHADS2 uses agegroup and conditions in the full history. Therefore we need to define the standard covariates using the FeatureExtraction::createCovariateSettings as follows:</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/library">library</a></span>(PatientLevelPrediction)
-covSet &lt;-<span class="st"> </span>FeatureExtraction<span class="op">::</span><span class="kw"><a href="https://www.rdocumentation.org/packages/FeatureExtraction/topics/createCovariateSettings">createCovariateSettings</a></span>(<span class="dt">useDemographicsAgeGroup =</span> T,                             
-                                                     <span class="dt">useConditionOccurrenceLongTerm =</span> T,
-                                                     <span class="dt">includedCovariateIds =</span> <span class="ot">NULL</span>,
-                                                     <span class="dt">longTermStartDays =</span> <span class="op">-</span><span class="dv">9999</span>, 
-                                                     <span class="dt">endDays =</span> <span class="dv">0</span>)</code></pre></div>
-<p>In the above code we used the useConditionOccurrenceLongTerm (these have an analysis id of 102) and we defined the longTermStartDays to be -9999 days relative to index (so we get the full history). We include the index date in our lookback period by specifying endDays = 0. The includeCovariateIds is set to NULL here, but this will be updated automatically later on. As we picked analysis id 102, the standard covariate for anytime prior congestive heart failure is 319835102. The same logic follows for the other conditions, so the covariate table will be:</p>
-<pre><code>##   modelCovariateId covariateId
-## 1                1   319835102
-## 2                2   316866102
-## 3                3       15003
-## 4                3       16003
-## 5                3       17003
-## 6                3       18003
-## 7                3       19003
-## 8                4   201820102
-## 9                5   381591102</code></pre>
-<p>modelCovariateId 3 was age&gt;= 75, as the standard covariate age groups are in 5 year groups, we needed to add the age groups 75-80, 80-85, 85-90, 90-95 and 95-100, these correspond to the covaraiteIds 15003, 16003, 17003, 18003 and 19003 respectively.</p>
-<p>To create the tables in R for CHADS2 you need to make the following dataframes:</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">model_table &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/data.frame">data.frame</a></span>(<span class="dt">modelId =</span> <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/c">c</a></span>(<span class="dv">1</span>,<span class="dv">1</span>,<span class="dv">1</span>,<span class="dv">1</span>,<span class="dv">1</span>),
-                          <span class="dt">modelCovariateId =</span> <span class="dv">1</span><span class="op">:</span><span class="dv">5</span>, 
-                          <span class="dt">coefficientValue =</span> <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/c">c</a></span>(<span class="dv">1</span>, <span class="dv">1</span>, <span class="dv">1</span>, <span class="dv">1</span>, <span class="dv">2</span>)
-                          )
-
-covariate_table &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/data.frame">data.frame</a></span>(<span class="dt">modelCovariateId =</span> <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/c">c</a></span>(<span class="dv">1</span>,<span class="dv">2</span>,<span class="dv">3</span>,<span class="dv">3</span>,<span class="dv">3</span>,<span class="dv">3</span>,<span class="dv">3</span>,<span class="dv">4</span>,<span class="dv">5</span>),
-                              <span class="dt">covariateId =</span> <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/c">c</a></span>(<span class="dv">319835102</span>, <span class="dv">316866102</span>, 
-                                            <span class="dv">15003</span>, <span class="dv">16003</span>, <span class="dv">17003</span>, <span class="dv">18003</span>, <span class="dv">19003</span>, 
-                                            <span class="dv">201820102</span>, <span class="dv">381591102</span>)
-                              )
-
-interceptTable &lt;-<span class="st">  </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/data.frame">data.frame</a></span>(<span class="dt">modelId =</span> <span class="dv">1</span>, 
-                              <span class="dt">interceptValue =</span> <span class="dv">0</span>)</code></pre></div>
-</div>
-<div id="create-the-model" class="section level2">
-<h2 class="hasAnchor">
-<a href="#create-the-model" class="anchor"></a>Create the model</h2>
-<p>Now you have everything in place to actually create the existing model. First specify the current environment as executing createExistingModelSql creates two functions for running the existing model into the specified environment. You need to specify the type of model (either logistic or score), in our example we are calculating a score. We finally need to specify the analysisId for the newly created CHADS2 covariate.</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">e &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/environment">environment</a></span>()
-PatientLevelPrediction<span class="op">::</span><span class="kw"><a href="https://www.rdocumentation.org/packages/PatientLevelPrediction/topics/createExistingModelSql">createExistingModelSql</a></span>(<span class="dt">modelTable =</span> model_table, 
-                       <span class="dt">modelNames =</span> <span class="st">'CHADS2'</span>, 
-                       <span class="dt">interceptTable =</span> <span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/data.frame">data.frame</a></span>(<span class="dt">modelId =</span> <span class="dv">1</span>, <span class="dt">interceptValue =</span> <span class="dv">0</span>),
-                       <span class="dt">covariateTable =</span> covariate_table, 
-                       <span class="dt">type =</span> <span class="st">'score'</span>,
-                       <span class="dt">analysisId =</span> <span class="dv">112</span>, <span class="dt">covariateSettings =</span> covSettings, <span class="dt">e =</span> e)</code></pre></div>
-<p>Once run you will find two new functions in your environment:</p>
-<ul>
-<li>createExistingmodelsCovariateSettings()</li>
-<li>getExistingmodelsCovariateSettings()</li>
-</ul>
-</div>
-<div id="run-the-model" class="section level2">
-<h2 class="hasAnchor">
-<a href="#run-the-model" class="anchor"></a>Run the model</h2>
-<p>Now you can use the functions you previously created to extract the existing model risk scores for a target population:</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">plpData &lt;-<span class="st"> </span>PatientLevelPrediction<span class="op">::</span><span class="kw"><a href="https://www.rdocumentation.org/packages/PatientLevelPrediction/topics/getPlpData">getPlpData</a></span>(connectionDetails, 
-                      <span class="dt">cdmDatabaseSchema =</span> <span class="st">'databasename.dbo'</span>,
-                      <span class="dt">cohortId =</span> <span class="dv">1</span>,
-                      <span class="dt">outcomeIds =</span> <span class="dv">2</span>, 
-                      <span class="dt">cohortDatabaseSchema =</span> <span class="st">'databasename.dbo'</span>, 
-                      <span class="dt">cohortTable =</span>  <span class="st">'cohort'</span> , 
-                      <span class="dt">outcomeDatabaseSchema =</span> <span class="st">'databasename.dbo'</span>, 
-                      <span class="dt">outcomeTable =</span> <span class="st">'cohort'</span>, 
-                      <span class="dt">covariateSettings =</span>  <span class="kw">createExistingmodelsCovariateSettings</span>(),
-                      <span class="dt">sampleSize =</span> <span class="dv">20000</span>
-                      )</code></pre></div>
-<p>To implement and evaluate an existing model you can use the function:</p>
-<p><code><a href="../reference/evaluateExistingModel.html">PatientLevelPrediction::evaluateExistingModel()</a></code></p>
-<p>with the following parameters:</p>
-<ul>
-<li>modelTable - a data.frame containing the columns: modelId, covariateId and coefficientValue</li>
-<li>covariateTable - a data.frame containing the columns: covariateId and standardCovariateId - this provides a set of standardCovariateId to define each model covariate.</li>
-<li>interceptTable - a data.frame containing the columns modelId and interceptValue or NULL if the model doesn’t have an intercept (equal to zero).</li>
-<li>type - the type of model (either: score or logistic)</li>
-<li>covariateSettings - this is used to determine the startDay and endDay for the standard covariates</li>
-<li>customCovariates - a data.frame with the covariateId and sql to generate the covariate value.</li>
-<li>riskWindowStart - the time at risk starts at target cohort start date + riskWindowStart</li>
-<li>addExposureDaysToEnd - if true then the time at risk window ends a the cohort end date + riskWindowEnd rather than cohort start date + riskWindowEnd</li>
-<li>riskWindowEnd - the time at risk ends at target cohort start/end date + riskWindowStart</li>
-<li>requireTimeAtRisk - whether to add a constraint to the number of days observed during the time at risk period in including people into the study</li>
-<li>minTimeAtRisk - the minimum number of days observation during the time at risk a target population person needs to be included</li>
-<li>includeAllOutcomes - Include outcomes even if they do not satisfy the minTimeAtRisk? (useful if the outcome is associated to death or rare)</li>
-<li>removeSubjectsWithPriorOutcome - remove target population people who have the outcome prior to the time at tisk period?</li>
-<li>connectionDetails - the connection to the CDM database</li>
-</ul>
-<p>Finally you need to add the settings for downloading the new data:</p>
-<ul>
-<li>cdmDatabaseSchema</li>
-<li>cohortDatabaseSchema</li>
-<li>cohortTable</li>
-<li>cohortId</li>
-<li>outcomeDatabaseSchema</li>
-<li>outcomeTable</li>
-<li>outcomeId</li>
-<li>oracleTempSchema</li>
-</ul>
-<p>To run the external validation of an existing model where the target population are those in the cohort table with id 1 and the outcome is those in the cohort table with id 2 and we are looking to predict first time occurrence of the outcome 1 day to 365 days after the target cohort start date (assuming you have the modelTable, covariateTable and interceptTable in the format explained above):</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="co"># in our example the existing model uses gender and condition groups looking back 200 days:</span>
-covSet &lt;-<span class="st"> </span>FeatureExtraction<span class="op">::</span><span class="kw"><a href="https://www.rdocumentation.org/packages/FeatureExtraction/topics/createCovariateSettings">createCovariateSettings</a></span>(<span class="dt">useDemographicsGender =</span> T,
-                                                     <span class="dt">useConditionGroupEraMediumTerm =</span> T, 
-                                                     <span class="dt">mediumTermStartDays =</span> <span class="op">-</span><span class="dv">200</span>)
-
-result &lt;-<span class="st"> </span><span class="kw"><a href="../reference/evaluateExistingModel.html">evaluateExistingModel</a></span>(<span class="dt">modelTable =</span> modelTable,
-                                <span class="dt">covariateTable =</span> covariateTable,
-                                <span class="dt">interceptTable =</span> <span class="ot">NULL</span>,
-                                <span class="dt">type =</span> <span class="st">'score'</span>, 
-                                <span class="dt">covariateSettings =</span>  covSet,
-                                <span class="dt">riskWindowStart =</span> <span class="dv">1</span>, 
-                                <span class="dt">addExposureDaysToEnd =</span> F, 
-                                <span class="dt">riskWindowEnd =</span> <span class="dv">365</span>, 
-                                <span class="dt">requireTimeAtRisk =</span> T, 
-                                <span class="dt">minTimeAtRisk =</span> <span class="dv">364</span>, 
-                                <span class="dt">includeAllOutcomes =</span> T, 
-                                <span class="dt">removeSubjectsWithPriorOutcome =</span> T, 
-                                <span class="dt">connectionDetails =</span> connectionDetails, 
-                                <span class="dt">cdmDatabaseSchema =</span> <span class="st">'databasename.dbo'</span>,
-                                <span class="dt">cohortId =</span> <span class="dv">1</span>,
-                                <span class="dt">outcomeId =</span> <span class="dv">2</span>, 
-                                <span class="dt">cohortDatabaseSchema =</span> <span class="st">'databasename.dbo'</span>, 
-                                <span class="dt">cohortTable =</span>  <span class="st">'cohort'</span> , 
-                                <span class="dt">outcomeDatabaseSchema =</span> <span class="st">'databasename.dbo'</span>, 
-                                <span class="dt">outcomeTable =</span> <span class="st">'cohort'</span>
-                      )</code></pre></div>
-<p><code>Result</code> will contain the performance and the predictions made by the model.</p>
-</div>
-</div>
-<div id="acknowledgments" class="section level1">
-<h1 class="hasAnchor">
-<a href="#acknowledgments" class="anchor"></a>Acknowledgments</h1>
-<p>Considerable work has been dedicated to provide the <code>PatientLevelPrediction</code> package.</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw"><a href="https://www.rdocumentation.org/packages/utils/topics/citation">citation</a></span>(<span class="st">"PatientLevelPrediction"</span>)</code></pre></div>
-<pre><code>## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018).
-## "Design and implementation of a standardized framework to generate
-## and evaluate patient-level prediction models using observational
-## healthcare data." _Journal of the American Medical Informatics
-## Association_, *25*(8), 969-975. &lt;URL:
-## https://doi.org/10.1093/jamia/ocy032&gt;.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }</code></pre>
-<p><strong>Please reference this paper if you use the PLP Package in your work:</strong></p>
-<p><a href="http://dx.doi.org/10.1093/jamia/ocy032">Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.</a></p>
-<p>This work is supported in part through the National Science Foundation grant IIS 1251151.</p>
-</div>
-  </div>
-
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-        <div id="tocnav">
-      <h2 class="hasAnchor">
-<a href="#tocnav" class="anchor"></a>Contents</h2>
-      <ul class="nav nav-pills nav-stacked">
-<li><a href="#introduction">Introduction</a></li>
-      <li>
-<a href="#model-implementation">Model implementation</a><ul class="nav nav-pills nav-stacked">
-<li><a href="#define-the-model">Define the model</a></li>
-      <li><a href="#create-the-model">Create the model</a></li>
-      <li><a href="#run-the-model">Run the model</a></li>
-      </ul>
-</li>
-      <li><a href="#acknowledgments">Acknowledgments</a></li>
-      </ul>
-</div>
-      </div>
-
-</div>
-
-
-      <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-</div>
-
-  
-
-  </body>
-</html>
diff --git a/docs/articles/InstallationGuide.html b/docs/articles/InstallationGuide.html
index 29f442285..054ce33de 100644
--- a/docs/articles/InstallationGuide.html
+++ b/docs/articles/InstallationGuide.html
@@ -19,6 +19,8 @@
 <![endif]-->
 </head>
 <body data-spy="scroll" data-target="#toc">
+    
+
     <div class="container template-article">
       <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
@@ -31,7 +33,7 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
@@ -53,7 +55,7 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
@@ -72,10 +74,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -84,13 +86,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -101,21 +106,30 @@
   </ul>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
       </ul>
 <ul class="nav navbar-nav navbar-right">
 <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
@@ -130,15 +144,16 @@
 
       
 
-      </header><script src="InstallationGuide_files/header-attrs-2.11/header-attrs.js"></script><div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header toc-ignore">
       <h1 data-toc-skip>Patient-Level Prediction Installation Guide</h1>
-                        <h4 class="author">Jenna Reps, Peter R. Rijnbeek</h4>
+                        <h4 data-toc-skip class="author">Jenna Reps,
+Peter R. Rijnbeek, Egill Fridgeirsson</h4>
             
-            <h4 class="date">2022-03-09</h4>
+            <h4 data-toc-skip class="date">2024-09-09</h4>
       
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/InstallationGuide.Rmd"><code>vignettes/InstallationGuide.Rmd</code></a></small>
+      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/vignettes/InstallationGuide.Rmd" class="external-link"><code>vignettes/InstallationGuide.Rmd</code></a></small>
       <div class="hidden name"><code>InstallationGuide.Rmd</code></div>
 
     </div>
@@ -146,142 +161,160 @@ <h4 class="date">2022-03-09</h4>
     
     
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Installation guide}
 -->
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
-<p>This vignette describes how you need to install the Observational Health Data Sciencs and Informatics (OHDSI) <a href="http://github.com/OHDSI/PatientLevelPrediction"><code>PatientLevelPrediction</code></a> package under Windows, Mac, and Linux.</p>
+<div class="section level2">
+<h2 id="introduction">Introduction<a class="anchor" aria-label="anchor" href="#introduction"></a>
+</h2>
+<p>This vignette describes how you need to install the Observational
+Health Data Science and Informatics (OHDSI) <a href="http://github.com/OHDSI/PatientLevelPrediction" class="external-link"><code>PatientLevelPrediction</code></a>
+package under Windows, Mac, and Linux.</p>
 </div>
-<div id="software-prerequisites" class="section level1">
-<h1 class="hasAnchor">
-<a href="#software-prerequisites" class="anchor"></a>Software Prerequisites</h1>
-<div id="windows-users" class="section level2">
-<h2 class="hasAnchor">
-<a href="#windows-users" class="anchor"></a>Windows Users</h2>
-<p>Under Windows the OHDSI Patient Level Prediction (PLP) package requires installing:</p>
+<div class="section level2">
+<h2 id="software-prerequisites">Software Prerequisites<a class="anchor" aria-label="anchor" href="#software-prerequisites"></a>
+</h2>
+<div class="section level3">
+<h3 id="windows-users">Windows Users<a class="anchor" aria-label="anchor" href="#windows-users"></a>
+</h3>
+<p>Under Windows the OHDSI Patient Level Prediction (PLP) package
+requires installing:</p>
 <ul>
-<li>R (<a href="https://cran.cnr.berkeley.edu/" class="uri">https://cran.cnr.berkeley.edu/</a> ) - (R &gt;= 3.3.0, but latest is recommended)</li>
-<li>Rstudio (<a href="https://www.rstudio.com/" class="uri">https://www.rstudio.com/</a> )</li>
-<li>Java (<a href="http://www.java.com" class="uri">http://www.java.com</a> )</li>
-<li>RTools (<a href="https://cran.r-project.org/bin/windows/Rtools/" class="uri">https://cran.r-project.org/bin/windows/Rtools/</a>)</li>
+<li>R (<a href="https://cran.cnr.berkeley.edu/" class="external-link uri">https://cran.cnr.berkeley.edu/</a> ) - (R &gt;= 4.0.0, but
+latest is recommended)</li>
+<li>Rstudio (<a href="https://www.rstudio.com/" class="external-link uri">https://www.rstudio.com/</a> )</li>
+<li>Java (<a href="http://www.java.com" class="external-link uri">http://www.java.com</a> )</li>
+<li>RTools (<a href="https://cran.r-project.org/bin/windows/Rtools/" class="external-link uri">https://cran.r-project.org/bin/windows/Rtools/</a>)</li>
 </ul>
 </div>
-<div id="maclinux-users" class="section level2">
-<h2 class="hasAnchor">
-<a href="#maclinux-users" class="anchor"></a>Mac/Linux Users</h2>
-<p>Under Mac and Linux the OHDSI Patient Level Prediction (PLP) package requires installing:</p>
+<div class="section level3">
+<h3 id="maclinux-users">Mac/Linux Users<a class="anchor" aria-label="anchor" href="#maclinux-users"></a>
+</h3>
+<p>Under Mac and Linux the OHDSI Patient Level Prediction (PLP) package
+requires installing:</p>
 <ul>
-<li>R (<a href="https://cran.cnr.berkeley.edu/" class="uri">https://cran.cnr.berkeley.edu/</a> ) - (R &gt;= 3.3.0, but latest is recommended)</li>
-<li>Rstudio (<a href="https://www.rstudio.com/" class="uri">https://www.rstudio.com/</a> )</li>
-<li>Java (<a href="http://www.java.com" class="uri">http://www.java.com</a> )</li>
-<li>Xcode command line tools(run in terminal: xcode-select –install) [MAC USERS ONLY]</li>
+<li>R (<a href="https://cran.cnr.berkeley.edu/" class="external-link uri">https://cran.cnr.berkeley.edu/</a> ) - (R &gt;= 4.0.0, but
+latest is recommended)</li>
+<li>Rstudio (<a href="https://www.rstudio.com/" class="external-link uri">https://www.rstudio.com/</a> )</li>
+<li>Java (<a href="http://www.java.com" class="external-link uri">http://www.java.com</a> )</li>
+<li>Xcode command line tools(run in terminal: xcode-select –install)
+[MAC USERS ONLY]</li>
 </ul>
 </div>
 </div>
-<div id="installing-the-package" class="section level1">
-<h1 class="hasAnchor">
-<a href="#installing-the-package" class="anchor"></a>Installing the Package</h1>
-<p>The preferred way to install the package is by using <code>remotes</code>, which will automatically install the latest release and all the latest dependencies.</p>
-<p>If you do not want the official release you could install the bleading edge version of the package (latest develop branch).</p>
-<p>Note that the latest develop branch could contain bugs, please report them to us if you experience problems.</p>
-<div id="installing-patientlevelprediction-using-remotes" class="section level2">
-<h2 class="hasAnchor">
-<a href="#installing-patientlevelprediction-using-remotes" class="anchor"></a>Installing PatientLevelPrediction using remotes</h2>
+<div class="section level2">
+<h2 id="installing-the-package">Installing the Package<a class="anchor" aria-label="anchor" href="#installing-the-package"></a>
+</h2>
+<p>The preferred way to install the package is by using
+<code>remotes</code>, which will automatically install the latest
+release and all the latest dependencies.</p>
+<p>If you do not want the official release you could install the
+bleeding edge version of the package (latest develop branch).</p>
+<p>Note that the latest develop branch could contain bugs, please report
+them to us if you experience problems.</p>
+<div class="section level3">
+<h3 id="installing-patientlevelprediction-using-remotes">Installing PatientLevelPrediction using remotes<a class="anchor" aria-label="anchor" href="#installing-patientlevelprediction-using-remotes"></a>
+</h3>
 <p>To install using <code>remotes</code> run:</p>
 <div class="sourceCode" id="cb1"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/utils/install.packages.html">install.packages</a></span><span class="op">(</span><span class="st">"remotes"</span><span class="op">)</span>
-<span class="fu">remotes</span><span class="fu">::</span><span class="fu"><a href="https://remotes.r-lib.org/reference/install_github.html">install_github</a></span><span class="op">(</span><span class="st">"OHDSI/FeatureExtraction"</span><span class="op">)</span>
-<span class="fu">remotes</span><span class="fu">::</span><span class="fu"><a href="https://remotes.r-lib.org/reference/install_github.html">install_github</a></span><span class="op">(</span><span class="st">"OHDSI/PatientLevelPrediction"</span><span class="op">)</span></code></pre></div>
-<p>When installing make sure to close any other Rstudio sessions that are using <code>PatientLevelPrediction</code> or any dependency. Keeping Rstudio sessions open can cause locks that prevent the package installing.</p>
+<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/utils/install.packages.html" class="external-link">install.packages</a></span><span class="op">(</span><span class="st">"remotes"</span><span class="op">)</span></span>
+<span><span class="fu">remotes</span><span class="fu">::</span><span class="fu"><a href="https://remotes.r-lib.org/reference/install_github.html" class="external-link">install_github</a></span><span class="op">(</span><span class="st">"OHDSI/PatientLevelPrediction"</span><span class="op">)</span></span></code></pre></div>
+<p>When installing make sure to close any other Rstudio sessions that
+are using <code>PatientLevelPrediction</code> or any dependency. Keeping
+Rstudio sessions open can cause locks that prevent the package
+installing.</p>
 </div>
 </div>
-<div id="creating-python-reticulate-environment" class="section level1">
-<h1 class="hasAnchor">
-<a href="#creating-python-reticulate-environment" class="anchor"></a>Creating Python Reticulate Environment</h1>
-<p>Many of the classifiers in the <code>PatientLevelPrediction</code> use a Python back end. To set up a python environment run:</p>
+<div class="section level2">
+<h2 id="creating-python-reticulate-environment">Creating Python Reticulate Environment<a class="anchor" aria-label="anchor" href="#creating-python-reticulate-environment"></a>
+</h2>
+<p>Many of the classifiers in the <code>PatientLevelPrediction</code>
+use a Python backend. To set up a python environment run:</p>
 <div class="sourceCode" id="cb2"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="kw"><a href="https://rdrr.io/r/base/library.html">library</a></span><span class="op">(</span><span class="va"><a href="https://ohdsi.github.io/PatientLevelPrediction">PatientLevelPrediction</a></span><span class="op">)</span>
-<span class="fu">reticulate</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/reticulate/man/install_miniconda.html">install_miniconda</a></span><span class="op">(</span><span class="op">)</span>
-<span class="fu"><a href="../reference/configurePython.html">configurePython</a></span><span class="op">(</span>envname<span class="op">=</span><span class="st">'r-reticulate'</span>, envtype<span class="op">=</span><span class="st">'conda'</span><span class="op">)</span></code></pre></div>
-<p>Some of the less frequently used classifiers are not installed during this set-up to add them run:</p>
-<p>For GBM survival:</p>
-<div class="sourceCode" id="cb3"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu">reticulate</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/reticulate/man/conda-tools.html">conda_install</a></span><span class="op">(</span>envname<span class="op">=</span><span class="st">'r-reticulate'</span>, packages <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="st">'scikit-survival'</span><span class="op">)</span>, forge <span class="op">=</span> <span class="cn">TRUE</span>, pip <span class="op">=</span> <span class="cn">FALSE</span>, pip_ignore_installed <span class="op">=</span> <span class="cn">TRUE</span>, conda <span class="op">=</span> <span class="st">"auto"</span>, channel <span class="op">=</span> <span class="st">'sebp'</span><span class="op">)</span></code></pre></div>
-</div>
-<div id="testing-installation" class="section level1">
-<h1 class="hasAnchor">
-<a href="#testing-installation" class="anchor"></a>Testing installation</h1>
-<p>To test whether the package is installed correctly, using the test script in ‘/extras’, run:</p>
-<div class="sourceCode" id="cb4"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="co"># load the checkPlpInstallation function</span>
-<span class="kw"><a href="https://rdrr.io/r/base/library.html">library</a></span><span class="op">(</span><span class="va"><a href="https://devtools.r-lib.org/">devtools</a></span><span class="op">)</span>
-<span class="fu"><a href="https://devtools.r-lib.org/reference/source_url.html">source_url</a></span><span class="op">(</span><span class="st">'https://raw.github.com/OHDSI/PatientLevelPrediction/issue242/extras/checkPlpInstallation.R'</span><span class="op">)</span>
-
-<span class="co"># set up the database connection details</span>
-<span class="kw"><a href="https://rdrr.io/r/base/library.html">library</a></span><span class="op">(</span><span class="va"><a href="https://ohdsi.github.io/DatabaseConnector/">DatabaseConnector</a></span><span class="op">)</span>
-<span class="va">connectionDetails</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/DatabaseConnector/reference/createConnectionDetails.html">createConnectionDetails</a></span><span class="op">(</span>
-  dbms <span class="op">=</span> <span class="st">'sql_server'</span>, 
-  user <span class="op">=</span> <span class="st">'username'</span>, 
-  password <span class="op">=</span> <span class="st">'hidden'</span>, 
-  server <span class="op">=</span> <span class="st">'your server'</span>, 
-  port <span class="op">=</span> <span class="st">'your port'</span>
-  <span class="op">)</span>
-
-<span class="co"># run the test</span>
-<span class="fu">checkPlpInstallation</span><span class="op">(</span>
-  connectionDetails <span class="op">=</span> <span class="va">connectionDetails</span>, 
-  python <span class="op">=</span> <span class="cn">T</span>
-  <span class="op">)</span></code></pre></div>
-<p>To test the installation (excluding python) run:</p>
-<div class="sourceCode" id="cb5"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu">checkPlpInstallation</span><span class="op">(</span>
-  connectionDetails <span class="op">=</span> <span class="va">connectionDetails</span>, 
-  python <span class="op">=</span> <span class="cn">F</span>
-  <span class="op">)</span></code></pre></div>
-<p>The check can take a while to run since it will build the following models in sequence on simulated data: Logistic Regression, RandomForest, MLP, AdaBoost, Decision Tree, Naive Bayes, KNN, Gradient Boosting. Moreover, it will test the database connection.</p>
+<code class="sourceCode R"><span><span class="kw"><a href="https://rdrr.io/r/base/library.html" class="external-link">library</a></span><span class="op">(</span><span class="va"><a href="https://ohdsi.github.io/PatientLevelPrediction" class="external-link">PatientLevelPrediction</a></span><span class="op">)</span></span>
+<span><span class="fu">reticulate</span><span class="fu">::</span><span class="fu"><a href="https://rstudio.github.io/reticulate/reference/install_miniconda.html" class="external-link">install_miniconda</a></span><span class="op">(</span><span class="op">)</span></span>
+<span><span class="fu"><a href="../reference/configurePython.html">configurePython</a></span><span class="op">(</span>envname<span class="op">=</span><span class="st">'r-reticulate'</span>, envtype<span class="op">=</span><span class="st">'conda'</span><span class="op">)</span></span></code></pre></div>
 </div>
-<div id="installation-issues" class="section level1">
-<h1 class="hasAnchor">
-<a href="#installation-issues" class="anchor"></a>Installation issues</h1>
-<p>Installation issues need to be posted in our issue tracker: <a href="http://github.com/OHDSI/PatientLevelPrediction/issues" class="uri">http://github.com/OHDSI/PatientLevelPrediction/issues</a></p>
+<div class="section level2">
+<h2 id="installation-issues">Installation issues<a class="anchor" aria-label="anchor" href="#installation-issues"></a>
+</h2>
+<p>Installation issues need to be posted in our issue tracker: <a href="http://github.com/OHDSI/PatientLevelPrediction/issues" class="external-link uri">http://github.com/OHDSI/PatientLevelPrediction/issues</a></p>
 <p>The list below provides solutions for some common issues:</p>
 <ol style="list-style-type: decimal">
-<li><p>If you have an error when trying to install a package in R saying <strong>‘Dependancy X not available …’</strong> then this can sometimes be fixed by running <code><a href="https://rdrr.io/r/utils/install.packages.html">install.packages('X')</a></code> and then once that completes trying to reinstall the package that had the error.</p></li>
-<li><p>I have found that using the github `remotes`` to install packages can be impacted if you have <strong>multiple R sessions</strong> open as one session with a library open can cause the library to be locked and this can prevent an install of a package that depends on that library.</p></li>
+<li><p>If you have an error when trying to install a package in R saying
+<strong>‘Dependancy X not available …’</strong> then this can sometimes
+be fixed by running <code>install.packages('X')</code> and then once
+that completes trying to reinstall the package that had the
+error.</p></li>
+<li><p>I have found that using the github <code>remotes</code> to
+install packages can be impacted if you have <strong>multiple R
+sessions</strong> open as one session with a library open can cause the
+library to be locked and this can prevent an install of a package that
+depends on that library.</p></li>
 </ol>
+<div class="section level3">
+<h3 id="common-issues">Common issues<a class="anchor" aria-label="anchor" href="#common-issues"></a>
+</h3>
+<div class="section level4">
+<h4 id="python-environment-maclinux-users">python environment Mac/linux users:<a class="anchor" aria-label="anchor" href="#python-environment-maclinux-users"></a>
+</h4>
+<p>to make sure R uses the r-reticulate python environment you may need
+to edit your .Rprofile with the location of the python binary for the
+PLP environment. Edit the .Rprofile by running:</p>
+<div class="sourceCode" id="cb3"><pre class="downlit sourceCode r">
+<code class="sourceCode R"><span><span class="fu">usethis</span><span class="fu">::</span><span class="fu"><a href="https://usethis.r-lib.org/reference/edit.html" class="external-link">edit_r_profile</a></span><span class="op">(</span><span class="op">)</span></span></code></pre></div>
+<p>and add</p>
+<div class="sourceCode" id="cb4"><pre class="downlit sourceCode r">
+<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/base/Sys.setenv.html" class="external-link">Sys.setenv</a></span><span class="op">(</span>PATH <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html" class="external-link">paste</a></span><span class="op">(</span><span class="st">"your python bin location"</span>, <span class="fu"><a href="https://rdrr.io/r/base/Sys.getenv.html" class="external-link">Sys.getenv</a></span><span class="op">(</span><span class="st">"PATH"</span><span class="op">)</span>, sep<span class="op">=</span><span class="st">":"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
+<p>to the file then save. Where your python bin location is the location
+returned by</p>
+<div class="sourceCode" id="cb5"><pre class="downlit sourceCode r">
+<code class="sourceCode R"><span><span class="fu">reticulate</span><span class="fu">::</span><span class="fu"><a href="https://rstudio.github.io/reticulate/reference/conda-tools.html" class="external-link">conda_list</a></span><span class="op">(</span><span class="op">)</span> </span></code></pre></div>
+<p>e.g., My PLP virtual environment location was
+/anaconda3/envs/PLP/bin/python so I added:<br>
+Sys.setenv(PATH = paste(“/anaconda3/envs/PLP/bin”, Sys.getenv(“PATH”),
+sep=“:”))</p>
+</div>
 </div>
-<div id="acknowledgments" class="section level1">
-<h1 class="hasAnchor">
-<a href="#acknowledgments" class="anchor"></a>Acknowledgments</h1>
-<p>Considerable work has been dedicated to provide the <code>PatientLevelPrediction</code> package.</p>
+</div>
+<div class="section level2">
+<h2 id="acknowledgments">Acknowledgments<a class="anchor" aria-label="anchor" href="#acknowledgments"></a>
+</h2>
+<p>Considerable work has been dedicated to provide the
+<code>PatientLevelPrediction</code> package.</p>
 <div class="sourceCode" id="cb6"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/utils/citation.html">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></code></pre></div>
-<pre><code>## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design
-## and implementation of a standardized framework to generate and evaluate
-## patient-level prediction models using observational healthcare data."
-## _Journal of the American Medical Informatics Association_, *25*(8),
-## 969-975. &lt;URL: https://doi.org/10.1093/jamia/ocy032&gt;.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }</code></pre>
-<p><strong>Please reference this paper if you use the PLP Package in your work:</strong></p>
-<p><a href="http://dx.doi.org/10.1093/jamia/ocy032">Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.</a></p>
-<p>This work is supported in part through the National Science Foundation grant IIS 1251151.</p>
+<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/utils/citation.html" class="external-link">citation</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></span></code></pre></div>
+<pre><code><span><span class="co">## </span></span>
+<span><span class="co">## To cite PatientLevelPrediction in publications use:</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">##   Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design</span></span>
+<span><span class="co">##   and implementation of a standardized framework to generate and</span></span>
+<span><span class="co">##   evaluate patient-level prediction models using observational</span></span>
+<span><span class="co">##   healthcare data." _Journal of the American Medical Informatics</span></span>
+<span><span class="co">##   Association_, *25*(8), 969-975.</span></span>
+<span><span class="co">##   &lt;https://doi.org/10.1093/jamia/ocy032&gt;.</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">## A BibTeX entry for LaTeX users is</span></span>
+<span><span class="co">## </span></span>
+<span><span class="co">##   @Article{,</span></span>
+<span><span class="co">##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},</span></span>
+<span><span class="co">##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},</span></span>
+<span><span class="co">##     journal = {Journal of the American Medical Informatics Association},</span></span>
+<span><span class="co">##     volume = {25},</span></span>
+<span><span class="co">##     number = {8},</span></span>
+<span><span class="co">##     pages = {969-975},</span></span>
+<span><span class="co">##     year = {2018},</span></span>
+<span><span class="co">##     url = {https://doi.org/10.1093/jamia/ocy032},</span></span>
+<span><span class="co">##   }</span></span></code></pre>
+<p><strong>Please reference this paper if you use the PLP Package in
+your work:</strong></p>
+<p><a href="http://dx.doi.org/10.1093/jamia/ocy032" class="external-link">Reps JM, Schuemie
+MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a
+standardized framework to generate and evaluate patient-level prediction
+models using observational healthcare data. J Am Med Inform Assoc.
+2018;25(8):969-975.</a></p>
+<p>This work is supported in part through the National Science
+Foundation grant IIS 1251151.</p>
 </div>
   </div>
 
@@ -296,11 +329,13 @@ <h1 class="hasAnchor">
 
 
       <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+  <p></p>
+<p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p>
+<p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
       </footer>
@@ -309,5 +344,7 @@ <h1 class="hasAnchor">
   
 
 
+  
+
   </body>
 </html>
diff --git a/docs/articles/InstallationGuide_files/header-attrs-2.11/header-attrs.js b/docs/articles/InstallationGuide_files/header-attrs-2.11/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/InstallationGuide_files/header-attrs-2.11/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/InstallationGuide_files/header-attrs-2.7/header-attrs.js b/docs/articles/InstallationGuide_files/header-attrs-2.7/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/InstallationGuide_files/header-attrs-2.7/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/PatientLevelPrediction.html b/docs/articles/PatientLevelPrediction.html
deleted file mode 100644
index 1b8d82727..000000000
--- a/docs/articles/PatientLevelPrediction.html
+++ /dev/null
@@ -1,301 +0,0 @@
-<!DOCTYPE html>
-<!-- Generated by pkgdown: do not edit by hand --><html lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-<title>Quick Install Guide • PatientLevelPrediction</title>
-<!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous">
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous">
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous">
-<!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script><meta property="og:title" content="Quick Install Guide">
-<meta property="og:description" content="PatientLevelPrediction">
-<!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-</head>
-<body data-spy="scroll" data-target="#toc">
-    <div class="container template-article">
-      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-<li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/InstallationGuide.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-<li>
-      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
-    </li>
-    <li>
-      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
-    </li>
-    <li>
-      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-<ul class="nav navbar-nav navbar-right">
-<li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-</div>
-<!--/.nav-collapse -->
-  </div>
-<!--/.container -->
-</div>
-<!--/.navbar -->
-
-      
-
-      </header><script src="PatientLevelPrediction_files/header-attrs-2.11/header-attrs.js"></script><div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header toc-ignore">
-      <h1 data-toc-skip>Quick Install Guide</h1>
-                        <h4 class="author">Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek</h4>
-            
-            <h4 class="date">2022-03-09</h4>
-      
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/PatientLevelPrediction.rmd"><code>vignettes/PatientLevelPrediction.rmd</code></a></small>
-      <div class="hidden name"><code>PatientLevelPrediction.rmd</code></div>
-
-    </div>
-
-    
-    
-<!--
-%\VignetteEngine{knitr}
-%\VignetteIndexEntry{Quick Install Guide}
--->
-<div id="quick-install-guide" class="section level1">
-<h1 class="hasAnchor">
-<a href="#quick-install-guide" class="anchor"></a>Quick Install Guide</h1>
-<hr>
-<div id="intalling-the-r-package" class="section level2">
-<h2 class="hasAnchor">
-<a href="#intalling-the-r-package" class="anchor"></a>Intalling the R package</h2>
-<p>The preferred way to install the package is by using drat, which will automatically install the latest release and all the latest dependencies. If the drat code fails or you do not want the official release you could use devtools to install the bleading edge version of the package (latest master). Note that the latest master could contain bugs, please report them to us if you experience problems.</p>
-<p>To install using drat run:</p>
-<div class="sourceCode" id="cb1"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/utils/install.packages.html">install.packages</a></span><span class="op">(</span><span class="st">"drat"</span><span class="op">)</span>
-<span class="fu">drat</span><span class="fu">::</span><span class="fu">addRepo</span><span class="op">(</span><span class="st">"OHDSI"</span><span class="op">)</span>
-<span class="fu"><a href="https://rdrr.io/r/utils/install.packages.html">install.packages</a></span><span class="op">(</span><span class="st">"PatientLevelPrediction"</span><span class="op">)</span></code></pre></div>
-<p>To install using devtools run:</p>
-<div class="sourceCode" id="cb2"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/utils/install.packages.html">install.packages</a></span><span class="op">(</span><span class="st">'devtools'</span><span class="op">)</span>
-<span class="fu">devtools</span><span class="fu">::</span><span class="fu"><a href="https://devtools.r-lib.org/reference/remote-reexports.html">install_github</a></span><span class="op">(</span><span class="st">"OHDSI/FeatureExtraction"</span><span class="op">)</span>
-<span class="fu">devtools</span><span class="fu">::</span><span class="fu"><a href="https://devtools.r-lib.org/reference/remote-reexports.html">install_github</a></span><span class="op">(</span><span class="st">'ohdsi/PatientLevelPrediction'</span><span class="op">)</span></code></pre></div>
-<p>When installing using devtools make sure to close any other Rstudio sessions that are using PatientLevelPrediction or any dependency. Keeping Rstudio sessions open can cause locks that prevent the package installing.</p>
-</div>
-<div id="setting-up-python" class="section level2">
-<h2 class="hasAnchor">
-<a href="#setting-up-python" class="anchor"></a>Setting up Python</h2>
-<p>Many of the classifiers in PatientLevelPrediction use python. To use the python classifiers you need to install and set up the a python environment in R. We used the reticulate package:</p>
-<div class="sourceCode" id="cb3"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="kw"><a href="https://rdrr.io/r/base/library.html">library</a></span><span class="op">(</span><span class="va"><a href="https://ohdsi.github.io/PatientLevelPrediction">PatientLevelPrediction</a></span><span class="op">)</span>
-<span class="fu">reticulate</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/reticulate/man/install_miniconda.html">install_miniconda</a></span><span class="op">(</span><span class="op">)</span>
-<span class="fu"><a href="../reference/configurePython.html">configurePython</a></span><span class="op">(</span>envname<span class="op">=</span><span class="st">'r-reticulate'</span>, envtype<span class="op">=</span><span class="st">'conda'</span><span class="op">)</span></code></pre></div>
-<p>To add the R keras interface, in Rstudio run:</p>
-<div class="sourceCode" id="cb4"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu">devtools</span><span class="fu">::</span><span class="fu"><a href="https://devtools.r-lib.org/reference/remote-reexports.html">install_github</a></span><span class="op">(</span><span class="st">"rstudio/keras"</span><span class="op">)</span>
-<span class="kw"><a href="https://rdrr.io/r/base/library.html">library</a></span><span class="op">(</span><span class="va"><a href="https://keras.rstudio.com">keras</a></span><span class="op">)</span>
-<span class="fu"><a href="https://keras.rstudio.com/reference/install_keras.html">install_keras</a></span><span class="op">(</span><span class="op">)</span></code></pre></div>
-<p>Some of the less frequently used classifiers are considered optional and are not installed by default. To install then, run:</p>
-<p>For GBM survival:</p>
-<div class="sourceCode" id="cb5"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu">reticulate</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/reticulate/man/conda-tools.html">conda_install</a></span><span class="op">(</span>envname<span class="op">=</span><span class="st">'r-reticulate'</span>, packages <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="st">'scikit-survival'</span><span class="op">)</span>, forge <span class="op">=</span> <span class="cn">TRUE</span>, pip <span class="op">=</span> <span class="cn">FALSE</span>, pip_ignore_installed <span class="op">=</span> <span class="cn">TRUE</span>, conda <span class="op">=</span> <span class="st">"auto"</span>, channel <span class="op">=</span> <span class="st">'sebp'</span><span class="op">)</span></code></pre></div>
-<p>For any of the torch models:</p>
-<div class="sourceCode" id="cb6"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu">reticulate</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/reticulate/man/conda-tools.html">conda_install</a></span><span class="op">(</span>envname<span class="op">=</span><span class="st">'r-reticulate'</span>, packages <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html">c</a></span><span class="op">(</span><span class="st">'pytorch'</span>, <span class="st">'torchvision'</span>, <span class="st">'cpuonly'</span><span class="op">)</span>, forge <span class="op">=</span> <span class="cn">TRUE</span>, pip <span class="op">=</span> <span class="cn">FALSE</span>, channel <span class="op">=</span> <span class="st">'pytorch'</span>, pip_ignore_installed <span class="op">=</span> <span class="cn">TRUE</span>, conda <span class="op">=</span> <span class="st">'auto'</span><span class="op">)</span></code></pre></div>
-</div>
-<div id="testing-the-patientlevelprediction-installation" class="section level2">
-<h2 class="hasAnchor">
-<a href="#testing-the-patientlevelprediction-installation" class="anchor"></a>Testing the PatientLevelPrediction Installation</h2>
-<p>To test whether the package is installed correctly run:</p>
-<div class="sourceCode" id="cb7"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="kw"><a href="https://rdrr.io/r/base/library.html">library</a></span><span class="op">(</span><span class="va"><a href="https://ohdsi.github.io/PatientLevelPrediction">PatientLevelPrediction</a></span><span class="op">)</span>
-<span class="kw"><a href="https://rdrr.io/r/base/library.html">library</a></span><span class="op">(</span><span class="va"><a href="https://ohdsi.github.io/DatabaseConnector/">DatabaseConnector</a></span><span class="op">)</span>
-<span class="va">connectionDetails</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/DatabaseConnector/reference/createConnectionDetails.html">createConnectionDetails</a></span><span class="op">(</span>dbms <span class="op">=</span> <span class="st">'sql_server'</span>, 
-                                             user <span class="op">=</span> <span class="st">'username'</span>, 
-                                             password <span class="op">=</span> <span class="st">'hidden'</span>, 
-                                             server <span class="op">=</span> <span class="st">'your server'</span>, 
-                                             port <span class="op">=</span> <span class="st">'your port'</span><span class="op">)</span>
-<span class="fu">PatientLevelPrediction</span><span class="fu">::</span><span class="fu">checkPlpInstallation</span><span class="op">(</span>connectionDetails <span class="op">=</span> <span class="va">connectionDetails</span>, 
-                                             python <span class="op">=</span> <span class="cn">T</span><span class="op">)</span></code></pre></div>
-<p>To test the installation (excluding python) run:</p>
-<div class="sourceCode" id="cb8"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="kw"><a href="https://rdrr.io/r/base/library.html">library</a></span><span class="op">(</span><span class="va"><a href="https://ohdsi.github.io/PatientLevelPrediction">PatientLevelPrediction</a></span><span class="op">)</span>
-<span class="kw"><a href="https://rdrr.io/r/base/library.html">library</a></span><span class="op">(</span><span class="va"><a href="https://ohdsi.github.io/DatabaseConnector/">DatabaseConnector</a></span><span class="op">)</span>
-<span class="va">connectionDetails</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://ohdsi.github.io/DatabaseConnector/reference/createConnectionDetails.html">createConnectionDetails</a></span><span class="op">(</span>dbms <span class="op">=</span> <span class="st">'sql_server'</span>, 
-                                           user <span class="op">=</span> <span class="st">'username'</span>, 
-                                           password <span class="op">=</span> <span class="st">'hidden'</span>, 
-                                           server <span class="op">=</span> <span class="st">'your server'</span>, 
-                                           port <span class="op">=</span> <span class="st">'your port'</span><span class="op">)</span>
-<span class="fu">PatientLevelPrediction</span><span class="fu">::</span><span class="fu">checkPlpInstallation</span><span class="op">(</span>connectionDetails <span class="op">=</span> <span class="va">connectionDetails</span>, 
-                                             python <span class="op">=</span> <span class="cn">F</span><span class="op">)</span></code></pre></div>
-<p>The check can take a while to run since it will build the following models in sequence on simulated data: Logistic Regression, RandomForest, MLP, AdaBoost, Decision Tree, Naive Bayes, KNN, Gradient Boosting. Moreover, it will test the database connection.</p>
-</div>
-</div>
-<div id="common-issues" class="section level1">
-<h1 class="hasAnchor">
-<a href="#common-issues" class="anchor"></a>Common issues</h1>
-<div id="python-environment-maclinux-users" class="section level2">
-<h2 class="hasAnchor">
-<a href="#python-environment-maclinux-users" class="anchor"></a>python environment Mac/linux users:</h2>
-<p>to make sure R uses the r-reticulate python environment you may need to edit your .Rprofile with the location of the python binary for the PLP environment. Edit the .Rprofile by running:</p>
-<div class="sourceCode" id="cb9"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu">usethis</span><span class="fu">::</span><span class="fu"><a href="https://usethis.r-lib.org/reference/edit.html">edit_r_profile</a></span><span class="op">(</span><span class="op">)</span></code></pre></div>
-<p>and add</p>
-<div class="sourceCode" id="cb10"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/base/Sys.setenv.html">Sys.setenv</a></span><span class="op">(</span>PATH <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste</a></span><span class="op">(</span><span class="st">"your python bin location"</span>, <span class="fu"><a href="https://rdrr.io/r/base/Sys.getenv.html">Sys.getenv</a></span><span class="op">(</span><span class="st">"PATH"</span><span class="op">)</span>, sep<span class="op">=</span><span class="st">":"</span><span class="op">)</span><span class="op">)</span></code></pre></div>
-<p>to the file then save. Where your python bin location is the location returned by</p>
-<div class="sourceCode" id="cb11"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu">reticulate</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/reticulate/man/conda-tools.html">conda_list</a></span><span class="op">(</span><span class="op">)</span> </code></pre></div>
-<p>e.g., My PLP virtual environment location was /anaconda3/envs/PLP/bin/python so I added:<br>
-Sys.setenv(PATH = paste(“/anaconda3/envs/PLP/bin”, Sys.getenv(“PATH”), sep=“:”))</p>
-</div>
-</div>
-<div id="old-instructions" class="section level1">
-<h1 class="hasAnchor">
-<a href="#old-instructions" class="anchor"></a>Old Instructions</h1>
-<div id="to-configure-python-via-anaconda" class="section level2">
-<h2 class="hasAnchor">
-<a href="#to-configure-python-via-anaconda" class="anchor"></a>To configure python via anaconda</h2>
-<ul>
-<li>Close your RStudio</li>
-<li>Install python 3.7 using anaconda (<a href="https://www.anaconda.com/download" class="uri">https://www.anaconda.com/download</a>) [make sure you pick the correct operating system] and note the installation location. Anaconda should update you path variable with the python binary.</li>
-<li>Open a new Rstudio and check whether your python is configured correctly by running:</li>
-</ul>
-<div class="sourceCode" id="cb12"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/base/system.html">system</a></span><span class="op">(</span><span class="st">"python --version"</span><span class="op">)</span> </code></pre></div>
-<p>If set up correctly you should see “Python 3.x.x :: Anaconda, Inc.” returned.</p>
-<ul>
-<li>If not set up correctly then:
-<ul>
-<li>Windows users: make sure your anaconda python binary is in the System PATH environmental variable: go to my computer -&gt; system properties -&gt; advanced system settings Then at the bottom right you’ll see a button: Environmental Variables, clicking on that will enable you to edit the PATH variable. Add the following Anaconda locations to your path: <code>D:\Anaconda3;D:\Anaconda3\Scripts;D:\Anaconda3\Library\bin</code> (this assumes your installation was done to <code>D:\Anaconda3</code>).</li>
-<li>Mac/Linux users: edit the bash profile to add python in the Path by running in the terminal: touch ~/.bash_profile; open ~/.bash_profile; and adding in the location of python in the PATH variable. Unfortunately, you also need to make an edit to the .Rprofile for R to get the correct PATH. To do this open the .Rprofile by running:</li>
-</ul>
-</li>
-</ul>
-<div class="sourceCode" id="cb13"><pre class="downlit sourceCode r">
-<code class="sourceCode R">  <span class="fu">usethis</span><span class="fu">::</span><span class="fu"><a href="https://usethis.r-lib.org/reference/edit.html">edit_r_profile</a></span><span class="op">(</span><span class="op">)</span></code></pre></div>
-<p>and in this file add</p>
-<div class="sourceCode" id="cb14"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/base/Sys.setenv.html">Sys.setenv</a></span><span class="op">(</span>PATH <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html">paste</a></span><span class="op">(</span><span class="st">"your python bin location"</span>, <span class="fu"><a href="https://rdrr.io/r/base/Sys.getenv.html">Sys.getenv</a></span><span class="op">(</span><span class="st">"PATH"</span><span class="op">)</span>, sep<span class="op">=</span><span class="st">":"</span><span class="op">)</span><span class="op">)</span></code></pre></div>
-<ul>
-<li>After editing your Path or .Rprofile open a new Rstudio session and test that python is correctly set up via</li>
-</ul>
-<div class="sourceCode" id="cb15"><pre class="downlit sourceCode r">
-<code class="sourceCode R"><span class="fu"><a href="https://rdrr.io/r/base/system.html">system</a></span><span class="op">(</span><span class="st">"python --version"</span><span class="op">)</span></code></pre></div>
-</div>
-</div>
-  </div>
-
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-
-        <nav id="toc" data-toggle="toc"><h2 data-toc-skip>Contents</h2>
-    </nav>
-</div>
-
-</div>
-
-
-
-      <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-</div>
-
-  
-
-
-  </body>
-</html>
diff --git a/docs/articles/PatientLevelPrediction_files/header-attrs-2.11/header-attrs.js b/docs/articles/PatientLevelPrediction_files/header-attrs-2.11/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/PatientLevelPrediction_files/header-attrs-2.11/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/PatientLevelPrediction_files/header-attrs-2.7/header-attrs.js b/docs/articles/PatientLevelPrediction_files/header-attrs-2.7/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/PatientLevelPrediction_files/header-attrs-2.7/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/PlottingLearningCurves.html b/docs/articles/PlottingLearningCurves.html
deleted file mode 100644
index e6cf4939c..000000000
--- a/docs/articles/PlottingLearningCurves.html
+++ /dev/null
@@ -1,276 +0,0 @@
-<!DOCTYPE html>
-<!-- Generated by pkgdown: do not edit by hand --><html lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-<title>Plotting learning curves • PatientLevelPrediction</title>
-<!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous">
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous">
-<!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script><!-- sticky kit --><script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script><meta property="og:title" content="Plotting learning curves">
-<meta property="og:description" content="">
-<meta name="twitter:card" content="summary">
-<!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-</head>
-<body>
-    <div class="container template-article">
-      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.1</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-<li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-<li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/AddingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction framework</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CustomPredictionAlgorithms.html">Custom patient-level prediction algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/PlottingLearningCurves.html">Plotting learning curves</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-<ul class="nav navbar-nav navbar-right">
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-</div>
-<!--/.nav-collapse -->
-  </div>
-<!--/.container -->
-</div>
-<!--/.navbar -->
-
-      
-      </header><div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header toc-ignore">
-      <h1>Plotting learning curves</h1>
-                        <h4 class="author">Peter R. Rijnbeek, Xiaoyong Pan, Luis H. John, Jenna Reps</h4>
-            
-            <h4 class="date">May 22, 2018</h4>
-      
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/PlottingLearningCurves.Rmd"><code>vignettes/PlottingLearningCurves.Rmd</code></a></small>
-      <div class="hidden name"><code>PlottingLearningCurves.Rmd</code></div>
-
-    </div>
-
-    
-    
-
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
-<p>Prediction models will show overly-optimistic performance when predicting on the same data as used for training. Therefore, we generally partition our data into a training set and testing set. We then train our prediction model on the training set portion and asses its ability to generalize to unseen data by measuring its performance on the testing set.</p>
-<p>Learning curves inform about the effect of training set size on model performance by training a sequence of prediction models on successively larger subsets of the training set. A learning curve plot can also help in diagnosing a bias or variance problem. Learning curves objects can be created and plotted with the <code>PatientLevelPrediction</code> package.</p>
-<div id="background" class="section level2">
-<h2 class="hasAnchor">
-<a href="#background" class="anchor"></a>Background</h2>
-<p>Figure 1 shows a commonly observed learning curve plot, where model performance is mapped to the vertical axis and training set size is mapped to the horizontal axis. If training set size is small, the performance on the training set is high, because a model can generally be fitted well to a limited number of training examples. At the same time, the performance on the testing set will be poor, because the model trained on such a limited number of training examples will not generalize well to unseen data in the testing set. As the training set size increases, the performance of the model on the training set will decrease. It becomes more difficult for the model to find a good fit through all the training examples. Also, the model will be trained on a more representative portion of training examples, making it generalize better to unseen data. This can be observed by the testing set performance increasing.</p>
-<div class="figure">
-<img src="learningCurve.png" alt="Figure 1. Learning curve plot with model performance mapped to the vertical axis and training set size mapped to the horizontal axis."><p class="caption">Figure 1. Learning curve plot with model performance mapped to the vertical axis and training set size mapped to the horizontal axis.</p>
-</div>
-</div>
-<div id="bias-and-variance" class="section level2">
-<h2 class="hasAnchor">
-<a href="#bias-and-variance" class="anchor"></a>Bias and variance</h2>
-<p>We can observe high variance (overfitting) in a prediction model if it performs well on the training set, but poorly on the testing set (Figure 2). Adding additional data is a common approach to counteract high variance. From the learning curve it becomes apparent, that adding additional data may improve performance on the testing set a little further, as the learning curve has not yet plateaued and, thus, the model is not saturated yet. Therefore, adding more data will decrease the gap between training set and testing set, which is the main indicator for a high variance problem.</p>
-<div class="figure">
-<img src="learningCurveVariance.png" alt="Figure 2. Prediction model suffering from high variance."><p class="caption">Figure 2. Prediction model suffering from high variance.</p>
-</div>
-<p>Furthermore, we can observe high bias (underfitting) if a prediction model performs poorly on the training set as well as on the testing set (Figure 3). The learning curves of training set and testing set have flattened on a low performance with only a small gap in between them. Adding additional data will in this case have little to no impact on the model performance. Choosing another prediction algorithm that can find more complex (potentiallly non-linear) relationships in the data may be an alternative approach to consider.</p>
-<div class="figure">
-<img src="learningCurveBias.png" alt="Figure 3. Prediction model suffering from high bias."><p class="caption">Figure 3. Prediction model suffering from high bias.</p>
-</div>
-</div>
-</div>
-<div id="usage" class="section level1">
-<h1 class="hasAnchor">
-<a href="#usage" class="anchor"></a>Usage</h1>
-<p>Use the <a href="https://github.com/OHDSI">OHDSI tool ecosystem</a> to generate a <code>population</code> and <code>plpData</code> object. Alternatively, you can make use of the data simulator. The following code snippet creates a population of 12000 patients.</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/Random">set.seed</a></span>(<span class="dv">1234</span>)
-<span class="kw"><a href="https://www.rdocumentation.org/packages/utils/topics/data">data</a></span>(plpDataSimulationProfile)
-sampleSize &lt;-<span class="st"> </span><span class="dv">12000</span>
-plpData &lt;-<span class="st"> </span><span class="kw"><a href="../reference/simulatePlpData.html">simulatePlpData</a></span>(
-  plpDataSimulationProfile,
-  <span class="dt">n =</span> sampleSize
-)
-
-population &lt;-<span class="st"> </span><span class="kw"><a href="../reference/createStudyPopulation.html">createStudyPopulation</a></span>(
-  plpData,
-  <span class="dt">outcomeId =</span> <span class="dv">2</span>,
-  <span class="dt">binary =</span> <span class="ot">TRUE</span>,
-  <span class="dt">firstExposureOnly =</span> <span class="ot">FALSE</span>,
-  <span class="dt">washoutPeriod =</span> <span class="dv">0</span>,
-  <span class="dt">removeSubjectsWithPriorOutcome =</span> <span class="ot">FALSE</span>,
-  <span class="dt">priorOutcomeLookback =</span> <span class="dv">99999</span>,
-  <span class="dt">requireTimeAtRisk =</span> <span class="ot">FALSE</span>,
-  <span class="dt">minTimeAtRisk =</span> <span class="dv">0</span>,
-  <span class="dt">riskWindowStart =</span> <span class="dv">0</span>,
-  <span class="dt">addExposureDaysToStart =</span> <span class="ot">FALSE</span>,
-  <span class="dt">riskWindowEnd =</span> <span class="dv">365</span>,
-  <span class="dt">addExposureDaysToEnd =</span> <span class="ot">FALSE</span>,
-  <span class="dt">verbosity =</span> futile.logger<span class="op">::</span>INFO
-)</code></pre></div>
-<p>Specify the prediction algorithm to be used.</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="co"># Use LASSO logistic regression</span>
-modelSettings &lt;-<span class="st"> </span><span class="kw"><a href="../reference/setLassoLogisticRegression.html">setLassoLogisticRegression</a></span>()</code></pre></div>
-<p>Specify a test fraction and a sequence of training set fractions.</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">testFraction &lt;-<span class="st"> </span><span class="fl">0.2</span>
-trainFractions &lt;-<span class="st"> </span><span class="kw"><a href="https://www.rdocumentation.org/packages/base/topics/seq">seq</a></span>(<span class="fl">0.1</span>, <span class="fl">0.8</span>, <span class="fl">0.1</span>)</code></pre></div>
-<p>Specify the test split to be used.</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="co"># Use a split by person, alterantively a time split is possible</span>
-testSplit &lt;-<span class="st"> 'person'</span></code></pre></div>
-<p>Create the learning curve object.</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">learningCurve &lt;-<span class="st"> </span><span class="kw"><a href="../reference/createLearningCurve.html">createLearningCurve</a></span>(
-  population,
-  <span class="dt">plpData =</span> plpData,
-  <span class="dt">modelSettings =</span> modelSettings,
-  <span class="dt">testFraction =</span> testFraction,
-  <span class="dt">trainFractions =</span> trainFractions,
-  <span class="dt">splitSeed =</span> <span class="dv">1000</span>,
-  <span class="dt">saveModel =</span> <span class="ot">FALSE</span>,
-  <span class="dt">timeStamp =</span> <span class="ot">FALSE</span>
-)</code></pre></div>
-<p>Plot the learning curve object (Figure 4). Specify one of the available metrics: <code>AUROC</code>, <code>AUPRC</code>, <code>sBrier</code>.</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw"><a href="../reference/plotLearningCurve.html">plotLearningCurve</a></span>(
-  learningCurve,
-  <span class="dt">metric=</span><span class="st">'AUROC'</span>,
-  <span class="dt">plotTitle =</span> <span class="st">'Learning Curve'</span>,
-  <span class="dt">plotSubtitle =</span> <span class="st">'AUROC performance'</span>
-)</code></pre></div>
-<div class="figure">
-<img src="learningCurvePlot.png" alt="Figure 4. Learning curve plot."><p class="caption">Figure 4. Learning curve plot.</p>
-</div>
-<div id="parallel-processing" class="section level2">
-<h2 class="hasAnchor">
-<a href="#parallel-processing" class="anchor"></a>Parallel processing</h2>
-<p>The learning curve object can be created in parallel, which can reduce computation time significantly. Currently this functionality is only available for LASSO logistic regression. Depending on the number of parallel workers it may require a significant amount of memory. We advise to use the parallelized learning curve function for parameter search and exploratory data analysis. Logging and saving functionality is unavailable.</p>
-<p>Use the parallelized version of the learning curve function to create the learning curve object in parallel. R will find the number of available processing cores automatically and register the required parallel backend.</p>
-<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">learningCurvePar &lt;-<span class="st"> </span><span class="kw"><a href="../reference/createLearningCurvePar.html">createLearningCurvePar</a></span>(
-  population,
-  <span class="dt">plpData =</span>  plpData,
-  <span class="dt">modelSettings =</span> modelSettings,
-  <span class="dt">testSplit =</span> testSplit,
-  <span class="dt">testFraction =</span> testFraction,
-  <span class="dt">trainFractions =</span> trainFractions,
-  <span class="dt">splitSeed =</span> <span class="dv">1000</span>
-)</code></pre></div>
-</div>
-</div>
-  </div>
-
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-        <div id="tocnav">
-      <h2 class="hasAnchor">
-<a href="#tocnav" class="anchor"></a>Contents</h2>
-      <ul class="nav nav-pills nav-stacked">
-<li>
-<a href="#introduction">Introduction</a><ul class="nav nav-pills nav-stacked">
-<li><a href="#background">Background</a></li>
-      <li><a href="#bias-and-variance">Bias and variance</a></li>
-      </ul>
-</li>
-      <li>
-<a href="#usage">Usage</a><ul class="nav nav-pills nav-stacked">
-<li><a href="#parallel-processing">Parallel processing</a></li>
-      </ul>
-</li>
-      </ul>
-</div>
-      </div>
-
-</div>
-
-
-      <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-</div>
-
-  
-
-  </body>
-</html>
diff --git a/docs/articles/Videos.html b/docs/articles/Videos.html
index 8f0d7b5f9..7277e6d0f 100644
--- a/docs/articles/Videos.html
+++ b/docs/articles/Videos.html
@@ -19,6 +19,8 @@
 <![endif]-->
 </head>
 <body data-spy="scroll" data-target="#toc">
+    
+
     <div class="container template-article">
       <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
@@ -31,7 +33,7 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
@@ -53,7 +55,7 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
@@ -72,10 +74,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -84,13 +86,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -101,21 +106,30 @@
   </ul>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
       </ul>
 <ul class="nav navbar-nav navbar-right">
 <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
@@ -130,15 +144,16 @@
 
       
 
-      </header><script src="Videos_files/header-attrs-2.11/header-attrs.js"></script><div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header toc-ignore">
       <h1 data-toc-skip>Demo Videos</h1>
-                        <h4 class="author">Jenna Reps, Peter R. Rijnbeek</h4>
+                        <h4 data-toc-skip class="author">Jenna Reps,
+Peter R. Rijnbeek</h4>
             
-            <h4 class="date">2022-03-09</h4>
+            <h4 data-toc-skip class="date">2024-09-09</h4>
       
-      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/vignettes/Videos.rmd"><code>vignettes/Videos.rmd</code></a></small>
+      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/vignettes/Videos.rmd" class="external-link"><code>vignettes/Videos.rmd</code></a></small>
       <div class="hidden name"><code>Videos.rmd</code></div>
 
     </div>
@@ -146,183 +161,123 @@ <h4 class="date">2022-03-09</h4>
     
     
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Demo Videos}
 -->
-<div id="what-is-a-cohort-table" class="section level2">
-<h2 class="hasAnchor">
-<a href="#what-is-a-cohort-table" class="anchor"></a>What is a cohort table?</h2>
-<table class="table">
-<tr>
-<th>
-Click To Launch
-</th>
-<th>
-Description of Demo
-</th>
-</tr>
-<tr>
-<td width="40%">
-<a href="https://youtu.be/GY2ZTcizY90" target="_blank"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" alt="Video Vignette PLP Package" width="240" height="180" border="10"></a>
-</td>
-<td width="60%">
-Learn what a cohort table looks like and what columns are required.
-</td>
-</tr>
-</table>
-</div>
-<div id="setting-up-a-connection-between-your-database-and-r" class="section level2">
-<h2 class="hasAnchor">
-<a href="#setting-up-a-connection-between-your-database-and-r" class="anchor"></a>Setting up a connection between your database and R</h2>
-<table class="table">
-<tr>
-<th>
-Click To Launch
-</th>
-<th>
-Description of Demo
-</th>
-</tr>
-<tr>
-<td width="40%">
-<a href="https://youtu.be/8F2X5SKN64w" target="_blank"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" alt="Video Vignette PLP Package" width="240" height="180" border="10"></a>
-</td>
-<td width="60%">
-Learn how to configure the connection to your OMOP CDM data from R using the OHDSI DatabaseConnector package.
-</td>
-</tr>
-</table>
-</div>
-<div id="running-a-single-patientlevelprediction-model" class="section level2">
-<h2 class="hasAnchor">
-<a href="#running-a-single-patientlevelprediction-model" class="anchor"></a>Running a single PatientLevelPrediction model</h2>
+<div class="section level2">
+<h2 id="what-is-a-cohort-table">What is a cohort table?<a class="anchor" aria-label="anchor" href="#what-is-a-cohort-table"></a>
+</h2>
 <table class="table">
-<tr>
-<th>
-Click To Launch
-</th>
-<th>
-Description of Demo
-</th>
-</tr>
-<tr>
-<td width="40%">
-<a href="https://youtu.be/W9uf1IIhDQs" target="_blank"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" alt="Video Vignette PLP Package" width="240" height="180" border="10"></a>
-</td>
-<td width="60%">
-Learn how to develop and validate a single PatientLevelPrediction model.
-</td>
-</tr>
+<colgroup>
+<col width="66%">
+<col width="33%">
+</colgroup>
+<thead><tr class="header">
+<th>Click To Launch</th>
+<th>Description of Demo</th>
+</tr></thead>
+<tbody><tr class="odd">
+<td><a href="https://youtu.be/GY2ZTcizY90" class="external-link"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" alt="Video Vignette PLP Package"></a></td>
+<td>Learn what a cohort table looks like and what columns are
+required.</td>
+</tr></tbody>
 </table>
 </div>
-<div id="running-multiple-patientlevelprediction-models-study" class="section level2">
-<h2 class="hasAnchor">
-<a href="#running-multiple-patientlevelprediction-models-study" class="anchor"></a>Running multiple PatientLevelPrediction models study</h2>
+<div class="section level2">
+<h2 id="setting-up-a-connection-between-your-database-and-r">Setting up a connection between your database and R<a class="anchor" aria-label="anchor" href="#setting-up-a-connection-between-your-database-and-r"></a>
+</h2>
 <table class="table">
-<tr>
-<th>
-Click To Launch
-</th>
-<th>
-Description of Demo
-</th>
-</tr>
-<tr>
-<td width="40%">
-<a href="https://youtu.be/Ppsn_08z5FE" target="_blank"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" alt="Video Vignette PLP Package" width="240" height="180" border="10"></a>
-</td>
-<td width="60%">
-Learn how to develop and validate multiple PatientLevelPrediction models.
-</td>
-</tr>
+<colgroup>
+<col width="55%">
+<col width="44%">
+</colgroup>
+<thead><tr class="header">
+<th>Click To Launch</th>
+<th>Description of Demo</th>
+</tr></thead>
+<tbody><tr class="odd">
+<td><a href="https://youtu.be/8F2X5SKN64w" class="external-link"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" alt="Video Vignette PLP Package"></a></td>
+<td>Learn how to configure the connection to your OMOP CDM data from R
+using the OHDSI DatabaseConnector package.</td>
+</tr></tbody>
 </table>
 </div>
-<div id="designing-a-study-in-atlas" class="section level2">
-<h2 class="hasAnchor">
-<a href="#designing-a-study-in-atlas" class="anchor"></a>Designing a study in Atlas</h2>
+<div class="section level2">
+<h2 id="running-a-single-patientlevelprediction-model">Running a single PatientLevelPrediction model<a class="anchor" aria-label="anchor" href="#running-a-single-patientlevelprediction-model"></a>
+</h2>
 <table class="table">
-<tr>
-<th>
-Click To Launch
-</th>
-<th>
-Description of Demo
-</th>
-</tr>
-<tr>
-<td width="40%">
-<a href="https://youtu.be/HqUdO3zx3DI" target="_blank"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" alt="Video Vignette PLP Package" width="240" height="180" border="10"></a>
-</td>
-<td width="60%">
-Learn how to design a multiple or single PatientLevelPrediction study using Atlas. Atlas creates an R package that just needs to be built and then you’re on your way to developing multiple models!
-</td>
-</tr>
+<colgroup>
+<col width="65%">
+<col width="34%">
+</colgroup>
+<thead><tr class="header">
+<th>Click To Launch</th>
+<th>Description of Demo</th>
+</tr></thead>
+<tbody><tr class="odd">
+<td><a href="https://youtu.be/7AraOsTynD4" class="external-link"><img src="http://img.youtube.com/vi/7AraOsTynD4/0.jpg" alt="Video Vignette PLP Package"></a></td>
+<td>Learn how to develop and validate a single PatientLevelPrediction
+model.</td>
+</tr></tbody>
 </table>
 </div>
-<div id="building-and-running-the-atlas-study" class="section level2">
-<h2 class="hasAnchor">
-<a href="#building-and-running-the-atlas-study" class="anchor"></a>Building and running the Atlas study</h2>
+<div class="section level2">
+<h2 id="running-multiple-patientlevelprediction-models-study">Running multiple PatientLevelPrediction models study<a class="anchor" aria-label="anchor" href="#running-multiple-patientlevelprediction-models-study"></a>
+</h2>
 <table class="table">
-<tr>
-<th>
-Click To Launch
-</th>
-<th>
-Description of Demo
-</th>
-</tr>
-<tr>
-<td width="40%">
-<a href="https://youtu.be/qESMHZnpHJQ" target="_blank"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" alt="Video Vignette PLP Package" width="240" height="180" border="10"></a>
-</td>
-<td width="60%">
-Learn how to build the R package generated by Atlas and how to then run the study.
-</td>
-</tr>
+<colgroup>
+<col width="65%">
+<col width="34%">
+</colgroup>
+<thead><tr class="header">
+<th>Click To Launch</th>
+<th>Description of Demo</th>
+</tr></thead>
+<tbody><tr class="odd">
+<td><a href="https://youtu.be/7wUilx580PE" class="external-link"><img src="http://img.youtube.com/vi/7wUilx580PE/0.jpg" alt="Video Vignette PLP Package"></a></td>
+<td>Learn how to develop and validate multiple PatientLevelPrediction
+models.</td>
+</tr></tbody>
 </table>
 </div>
-<div id="exploring-the-results-in-the-shiny-app" class="section level2">
-<h2 class="hasAnchor">
-<a href="#exploring-the-results-in-the-shiny-app" class="anchor"></a>Exploring the results in the shiny app</h2>
+<div class="section level2">
+<h2 id="exploring-the-results-in-the-shiny-app">Exploring the results in the shiny app<a class="anchor" aria-label="anchor" href="#exploring-the-results-in-the-shiny-app"></a>
+</h2>
 <table class="table">
-<tr>
-<th>
-Click To Launch
-</th>
-<th>
-Description of Demo
-</th>
-</tr>
-<tr>
-<td width="40%">
-<a href="https://youtu.be/StpV40yl1UE" target="_blank"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" alt="Video Vignette PLP Package" width="240" height="180" border="10"></a>
-</td>
-<td width="60%">
-Learn how to interactively explore the model performance and model via the shiny apps viewPlp() and viewMultiplePlp()
-</td>
-</tr>
+<colgroup>
+<col width="54%">
+<col width="45%">
+</colgroup>
+<thead><tr class="header">
+<th>Click To Launch</th>
+<th>Description of Demo</th>
+</tr></thead>
+<tbody><tr class="odd">
+<td><a href="https://youtu.be/BulmuH32y_Y" class="external-link"><img src="http://img.youtube.com/vi/BulmuH32y_Y/0.jpg" alt="Video Vignette PLP Package"></a></td>
+<td>Learn how to interactively explore the model performance and model
+via the shiny apps viewPlp() and viewMultiplePlp()</td>
+</tr></tbody>
 </table>
 </div>
-<div id="validating-existing-models-on-omop-cdm-data" class="section level2">
-<h2 class="hasAnchor">
-<a href="#validating-existing-models-on-omop-cdm-data" class="anchor"></a>Validating existing models on OMOP CDM data</h2>
+<div class="section level2">
+<h2 id="validating-existing-models-on-omop-cdm-data">Validating existing models on OMOP CDM data<a class="anchor" aria-label="anchor" href="#validating-existing-models-on-omop-cdm-data"></a>
+</h2>
 <table class="table">
-<tr>
-<th>
-Click To Launch
-</th>
-<th>
-Description of Demo
-</th>
-</tr>
-<tr>
-<td width="40%">
-<a href="https://youtu.be/oBsfg9hfrpI" target="_blank"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" alt="Video Vignette PLP Package" width="240" height="180" border="10"></a>
-</td>
-<td width="60%">
-This demo shows how you can add any existing score or logistic model and valdiate the model on new OMOP CDM data. This is useful for benchmarking when developing new models or to perform extensive external validation of a model across the OHDSI network.
-</td>
-</tr>
+<colgroup>
+<col width="36%">
+<col width="63%">
+</colgroup>
+<thead><tr class="header">
+<th>Click To Launch</th>
+<th>Description of Demo</th>
+</tr></thead>
+<tbody><tr class="odd">
+<td><a href="https://youtu.be/oBsfg9hfrpI" class="external-link"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" alt="Video Vignette PLP Package"></a></td>
+<td>This demo shows how you can add any existing score or logistic model
+and validate the model on new OMOP CDM data. This is useful for
+benchmarking when developing new models or to perform extensive external
+validation of a model across the OHDSI network.</td>
+</tr></tbody>
 </table>
 </div>
   </div>
@@ -338,11 +293,13 @@ <h2 class="hasAnchor">
 
 
       <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+  <p></p>
+<p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p>
+<p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
       </footer>
@@ -351,5 +308,7 @@ <h2 class="hasAnchor">
   
 
 
+  
+
   </body>
 </html>
diff --git a/docs/articles/Videos_files/header-attrs-2.11/header-attrs.js b/docs/articles/Videos_files/header-attrs-2.11/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/Videos_files/header-attrs-2.11/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/Videos_files/header-attrs-2.7/header-attrs.js b/docs/articles/Videos_files/header-attrs-2.7/header-attrs.js
deleted file mode 100644
index dd57d92e0..000000000
--- a/docs/articles/Videos_files/header-attrs-2.7/header-attrs.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
-// be compatible with the behavior of Pandoc < 2.8).
-document.addEventListener('DOMContentLoaded', function(e) {
-  var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
-  var i, h, a;
-  for (i = 0; i < hs.length; i++) {
-    h = hs[i];
-    if (!/^h[1-6]$/i.test(h.tagName)) continue;  // it should be a header h1-h6
-    a = h.attributes;
-    while (a.length > 0) h.removeAttribute(a[0].name);
-  }
-});
diff --git a/docs/articles/arch1.png b/docs/articles/arch1.png
deleted file mode 100644
index e4846e56f..000000000
Binary files a/docs/articles/arch1.png and /dev/null differ
diff --git a/docs/articles/atlasdownload1.png b/docs/articles/atlasdownload1.png
deleted file mode 100644
index ef6559fa9..000000000
Binary files a/docs/articles/atlasdownload1.png and /dev/null differ
diff --git a/docs/articles/atlasdownload1.webp b/docs/articles/atlasdownload1.webp
new file mode 100644
index 000000000..6cac340ed
Binary files /dev/null and b/docs/articles/atlasdownload1.webp differ
diff --git a/docs/articles/atlasdownload2.png b/docs/articles/atlasdownload2.png
deleted file mode 100644
index 619f8c799..000000000
Binary files a/docs/articles/atlasdownload2.png and /dev/null differ
diff --git a/docs/articles/atlasdownload2.webp b/docs/articles/atlasdownload2.webp
new file mode 100644
index 000000000..452c5ca21
Binary files /dev/null and b/docs/articles/atlasdownload2.webp differ
diff --git a/docs/articles/atlasplp1.png b/docs/articles/atlasplp1.png
deleted file mode 100644
index 4b21b2143..000000000
Binary files a/docs/articles/atlasplp1.png and /dev/null differ
diff --git a/docs/articles/atlasplp1.webp b/docs/articles/atlasplp1.webp
new file mode 100644
index 000000000..71a3c1ce9
Binary files /dev/null and b/docs/articles/atlasplp1.webp differ
diff --git a/docs/articles/atlasplp2.png b/docs/articles/atlasplp2.png
deleted file mode 100644
index 6bc7b93ad..000000000
Binary files a/docs/articles/atlasplp2.png and /dev/null differ
diff --git a/docs/articles/atlasplp3.png b/docs/articles/atlasplp3.png
deleted file mode 100644
index 0911b31ea..000000000
Binary files a/docs/articles/atlasplp3.png and /dev/null differ
diff --git a/docs/articles/atlasplp3.webp b/docs/articles/atlasplp3.webp
new file mode 100644
index 000000000..523d0143c
Binary files /dev/null and b/docs/articles/atlasplp3.webp differ
diff --git a/docs/articles/atlasplp4.png b/docs/articles/atlasplp4.png
deleted file mode 100644
index b5db1b153..000000000
Binary files a/docs/articles/atlasplp4.png and /dev/null differ
diff --git a/docs/articles/cirenn.png b/docs/articles/cirenn.png
deleted file mode 100644
index f4e8ed054..000000000
Binary files a/docs/articles/cirenn.png and /dev/null differ
diff --git a/docs/articles/cnn_lstm.png b/docs/articles/cnn_lstm.png
deleted file mode 100644
index a16e1417d..000000000
Binary files a/docs/articles/cnn_lstm.png and /dev/null differ
diff --git a/docs/articles/cnn_mlf2.png b/docs/articles/cnn_mlf2.png
deleted file mode 100644
index 2b69c159b..000000000
Binary files a/docs/articles/cnn_mlf2.png and /dev/null differ
diff --git a/docs/articles/conv_arch1.png b/docs/articles/conv_arch1.png
deleted file mode 100644
index 5970b3f1c..000000000
Binary files a/docs/articles/conv_arch1.png and /dev/null differ
diff --git a/docs/articles/conv_arch2.png b/docs/articles/conv_arch2.png
deleted file mode 100644
index a51ccf08e..000000000
Binary files a/docs/articles/conv_arch2.png and /dev/null differ
diff --git a/docs/articles/covcnn.png b/docs/articles/covcnn.png
deleted file mode 100644
index 82dd2832f..000000000
Binary files a/docs/articles/covcnn.png and /dev/null differ
diff --git a/docs/articles/covcnn2.png b/docs/articles/covcnn2.png
deleted file mode 100644
index 0734a49eb..000000000
Binary files a/docs/articles/covcnn2.png and /dev/null differ
diff --git a/docs/articles/demographicSummary.png b/docs/articles/demographicSummary.png
deleted file mode 100644
index 8ceafbee8..000000000
Binary files a/docs/articles/demographicSummary.png and /dev/null differ
diff --git a/docs/articles/demographicSummary.webp b/docs/articles/demographicSummary.webp
new file mode 100644
index 000000000..7d0437deb
Binary files /dev/null and b/docs/articles/demographicSummary.webp differ
diff --git a/docs/articles/ensemble.png b/docs/articles/ensemble.png
deleted file mode 100644
index 6e2173a48..000000000
Binary files a/docs/articles/ensemble.png and /dev/null differ
diff --git a/docs/articles/example1/ATLAS_O.png b/docs/articles/example1/ATLAS_O.png
deleted file mode 100644
index 3cda2abf7..000000000
Binary files a/docs/articles/example1/ATLAS_O.png and /dev/null differ
diff --git a/docs/articles/example1/ATLAS_O.webp b/docs/articles/example1/ATLAS_O.webp
new file mode 100644
index 000000000..85e63dc9e
Binary files /dev/null and b/docs/articles/example1/ATLAS_O.webp differ
diff --git a/docs/articles/example1/ATLAS_T.png b/docs/articles/example1/ATLAS_T.png
deleted file mode 100644
index 8be57dc9e..000000000
Binary files a/docs/articles/example1/ATLAS_T.png and /dev/null differ
diff --git a/docs/articles/example1/ATLAS_T.webp b/docs/articles/example1/ATLAS_T.webp
new file mode 100644
index 000000000..df3a8245f
Binary files /dev/null and b/docs/articles/example1/ATLAS_T.webp differ
diff --git a/docs/articles/example2/aceinhibitors.png b/docs/articles/example2/aceinhibitors.png
deleted file mode 100644
index ce5148f1d..000000000
Binary files a/docs/articles/example2/aceinhibitors.png and /dev/null differ
diff --git a/docs/articles/example2/aceinhibitors.webp b/docs/articles/example2/aceinhibitors.webp
new file mode 100644
index 000000000..564f8af77
Binary files /dev/null and b/docs/articles/example2/aceinhibitors.webp differ
diff --git a/docs/articles/example2/angioedema.png b/docs/articles/example2/angioedema.png
deleted file mode 100644
index 3adc8dcc9..000000000
Binary files a/docs/articles/example2/angioedema.png and /dev/null differ
diff --git a/docs/articles/example2/angioedema.webp b/docs/articles/example2/angioedema.webp
new file mode 100644
index 000000000..8c728ce50
Binary files /dev/null and b/docs/articles/example2/angioedema.webp differ
diff --git a/docs/articles/generalizability.png b/docs/articles/generalizability.png
deleted file mode 100644
index b476ea71f..000000000
Binary files a/docs/articles/generalizability.png and /dev/null differ
diff --git a/docs/articles/generalizability.webp b/docs/articles/generalizability.webp
new file mode 100644
index 000000000..ba6d14de4
Binary files /dev/null and b/docs/articles/generalizability.webp differ
diff --git a/docs/articles/index.html b/docs/articles/index.html
index f48ef229e..631931e62 100644
--- a/docs/articles/index.html
+++ b/docs/articles/index.html
@@ -1,66 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Articles • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Articles" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Articles • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Articles"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-article-index">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -71,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -93,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -112,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -124,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -138,39 +85,41 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
       <h1>Articles</h1>
@@ -180,57 +129,53 @@ <h1>Articles</h1>
       <h3>All vignettes</h3>
       <p class="section-desc"></p>
 
-      <dl>
-        <dt><a href="AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a></dt>
-        <dd></dt>
-        <dt><a href="AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a></dt>
-        <dd></dt>
-        <dt><a href="AddingCustomSamples.html">Adding Custom Sampling Functions</a></dt>
-        <dd></dt>
-        <dt><a href="AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a></dt>
-        <dd></dt>
-        <dt><a href="BestPractices.html">Best Practice Research</a></dt>
-        <dd></dt>
-        <dt><a href="BuildingEnsembleModels.html">Building Ensemble Models</a></dt>
-        <dd></dt>
-        <dt><a href="BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a></dt>
-        <dd></dt>
-        <dt><a href="BuildingPredictiveModels.html">Building patient-level predictive models</a></dt>
-        <dd></dt>
-        <dt><a href="CreatingLearningCurves.html">Creating Learning Curves</a></dt>
-        <dd></dt>
-        <dt><a href="CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a></dt>
-        <dd></dt>
-        <dt><a href="CreatingShinyApp.html">Creating Shiny App</a></dt>
-        <dd></dt>
-        <dt><a href="InstallationGuide.html">Patient-Level Prediction Installation Guide</a></dt>
-        <dd></dt>
-        <dt><a href="PatientLevelPrediction.html">Quick Install Guide</a></dt>
-        <dd></dt>
-        <dt><a href="Videos.html">Demo Videos</a></dt>
-        <dd></dt>
-      </dl>
-    </div>
+      <dl><dt><a href="AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a></dt>
+        <dd>
+        </dd><dt><a href="AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a></dt>
+        <dd>
+        </dd><dt><a href="AddingCustomSamples.html">Adding Custom Sampling Functions</a></dt>
+        <dd>
+        </dd><dt><a href="AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a></dt>
+        <dd>
+        </dd><dt><a href="BenchmarkTasks.html">Benchmark Tasks</a></dt>
+        <dd>
+        </dd><dt><a href="BestPractices.html">Best Practice Research</a></dt>
+        <dd>
+        </dd><dt><a href="BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a></dt>
+        <dd>
+        </dd><dt><a href="BuildingPredictiveModels.html">Building patient-level predictive models</a></dt>
+        <dd>
+        </dd><dt><a href="ClinicalModels.html">Clinical Models</a></dt>
+        <dd>
+        </dd><dt><a href="ConstrainedPredictors.html">Constrained predictors</a></dt>
+        <dd>
+        </dd><dt><a href="CreatingLearningCurves.html">Creating Learning Curves</a></dt>
+        <dd>
+        </dd><dt><a href="CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a></dt>
+        <dd>
+        </dd><dt><a href="InstallationGuide.html">Patient-Level Prediction Installation Guide</a></dt>
+        <dd>
+        </dd><dt><a href="Videos.html">Demo Videos</a></dt>
+        <dd>
+      </dd></dl></div>
   </div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/articles/lstm_last.png b/docs/articles/lstm_last.png
deleted file mode 100644
index 3e6fc16e5..000000000
Binary files a/docs/articles/lstm_last.png and /dev/null differ
diff --git a/docs/articles/popdef1.png b/docs/articles/popdef1.png
deleted file mode 100644
index 3d654fe7d..000000000
Binary files a/docs/articles/popdef1.png and /dev/null differ
diff --git a/docs/articles/popdef1.webp b/docs/articles/popdef1.webp
new file mode 100644
index 000000000..83ef7afd6
Binary files /dev/null and b/docs/articles/popdef1.webp differ
diff --git a/docs/articles/popdef2.png b/docs/articles/popdef2.png
deleted file mode 100644
index a596e188d..000000000
Binary files a/docs/articles/popdef2.png and /dev/null differ
diff --git a/docs/articles/popdef2.webp b/docs/articles/popdef2.webp
new file mode 100644
index 000000000..31887dd1b
Binary files /dev/null and b/docs/articles/popdef2.webp differ
diff --git a/docs/articles/popdef3.png b/docs/articles/popdef3.png
deleted file mode 100644
index 34527ef9f..000000000
Binary files a/docs/articles/popdef3.png and /dev/null differ
diff --git a/docs/articles/popdef3.webp b/docs/articles/popdef3.webp
new file mode 100644
index 000000000..8b409ed49
Binary files /dev/null and b/docs/articles/popdef3.webp differ
diff --git a/docs/articles/popdef4.png b/docs/articles/popdef4.png
deleted file mode 100644
index 35d4949a5..000000000
Binary files a/docs/articles/popdef4.png and /dev/null differ
diff --git a/docs/articles/popdef4.webp b/docs/articles/popdef4.webp
new file mode 100644
index 000000000..2709497e7
Binary files /dev/null and b/docs/articles/popdef4.webp differ
diff --git a/docs/articles/popdef5.png b/docs/articles/popdef5.png
deleted file mode 100644
index f6315b8a8..000000000
Binary files a/docs/articles/popdef5.png and /dev/null differ
diff --git a/docs/articles/popdef5.webp b/docs/articles/popdef5.webp
new file mode 100644
index 000000000..748b8901b
Binary files /dev/null and b/docs/articles/popdef5.webp differ
diff --git a/docs/articles/popdef6.png b/docs/articles/popdef6.png
deleted file mode 100644
index 96a8abd1f..000000000
Binary files a/docs/articles/popdef6.png and /dev/null differ
diff --git a/docs/articles/popdef6.webp b/docs/articles/popdef6.webp
new file mode 100644
index 000000000..583dc9fba
Binary files /dev/null and b/docs/articles/popdef6.webp differ
diff --git a/docs/articles/precisionRecall.png b/docs/articles/precisionRecall.png
deleted file mode 100644
index 1f1d0f154..000000000
Binary files a/docs/articles/precisionRecall.png and /dev/null differ
diff --git a/docs/articles/precisionRecall.webp b/docs/articles/precisionRecall.webp
new file mode 100644
index 000000000..af6b0cfe5
Binary files /dev/null and b/docs/articles/precisionRecall.webp differ
diff --git a/docs/articles/predictionDistribution.png b/docs/articles/predictionDistribution.png
deleted file mode 100644
index 87bafc361..000000000
Binary files a/docs/articles/predictionDistribution.png and /dev/null differ
diff --git a/docs/articles/preferencePDF.png b/docs/articles/preferencePDF.png
deleted file mode 100644
index 3b3528452..000000000
Binary files a/docs/articles/preferencePDF.png and /dev/null differ
diff --git a/docs/articles/preferencePDF.webp b/docs/articles/preferencePDF.webp
new file mode 100644
index 000000000..189a356be
Binary files /dev/null and b/docs/articles/preferencePDF.webp differ
diff --git a/docs/articles/problems.png b/docs/articles/problems.png
deleted file mode 100644
index 931efa6d6..000000000
Binary files a/docs/articles/problems.png and /dev/null differ
diff --git a/docs/articles/problems.webp b/docs/articles/problems.webp
new file mode 100644
index 000000000..5c1c27bb4
Binary files /dev/null and b/docs/articles/problems.webp differ
diff --git a/docs/articles/shinyroc.png b/docs/articles/shinyroc.png
deleted file mode 100644
index 579fab31f..000000000
Binary files a/docs/articles/shinyroc.png and /dev/null differ
diff --git a/docs/articles/shinyroc.webp b/docs/articles/shinyroc.webp
new file mode 100644
index 000000000..a11724623
Binary files /dev/null and b/docs/articles/shinyroc.webp differ
diff --git a/docs/articles/shinysummary.png b/docs/articles/shinysummary.png
deleted file mode 100644
index 75cec2430..000000000
Binary files a/docs/articles/shinysummary.png and /dev/null differ
diff --git a/docs/articles/shinysummary.webp b/docs/articles/shinysummary.webp
new file mode 100644
index 000000000..0d256ade1
Binary files /dev/null and b/docs/articles/shinysummary.webp differ
diff --git a/docs/articles/sparseCalibration.png b/docs/articles/sparseCalibration.png
deleted file mode 100644
index d6e34c0cf..000000000
Binary files a/docs/articles/sparseCalibration.png and /dev/null differ
diff --git a/docs/articles/sparseCalibration.webp b/docs/articles/sparseCalibration.webp
new file mode 100644
index 000000000..043019e5b
Binary files /dev/null and b/docs/articles/sparseCalibration.webp differ
diff --git a/docs/articles/sparseRoc.png b/docs/articles/sparseRoc.png
deleted file mode 100644
index 8a4b13cec..000000000
Binary files a/docs/articles/sparseRoc.png and /dev/null differ
diff --git a/docs/articles/sparseRoc.webp b/docs/articles/sparseRoc.webp
new file mode 100644
index 000000000..2ea3ea56f
Binary files /dev/null and b/docs/articles/sparseRoc.webp differ
diff --git a/docs/articles/studydesign.png b/docs/articles/studydesign.png
deleted file mode 100644
index 453f4aadd..000000000
Binary files a/docs/articles/studydesign.png and /dev/null differ
diff --git a/docs/articles/studydesign.webp b/docs/articles/studydesign.webp
new file mode 100644
index 000000000..28717c7d2
Binary files /dev/null and b/docs/articles/studydesign.webp differ
diff --git a/docs/articles/variableScatterplot.png b/docs/articles/variableScatterplot.png
deleted file mode 100644
index bdcf0df4a..000000000
Binary files a/docs/articles/variableScatterplot.png and /dev/null differ
diff --git a/docs/articles/variableScatterplot.webp b/docs/articles/variableScatterplot.webp
new file mode 100644
index 000000000..de6f8999d
Binary files /dev/null and b/docs/articles/variableScatterplot.webp differ
diff --git a/docs/authors.html b/docs/authors.html
index 31e2e4ba2..caa14ab0f 100644
--- a/docs/authors.html
+++ b/docs/authors.html
@@ -1,66 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Citation and Authors • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="bootstrap-toc.css">
-<script src="bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="pkgdown.css" rel="stylesheet">
-<script src="pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Citation and Authors" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Authors and Citation • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="bootstrap-toc.css"><script src="bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="pkgdown.css" rel="stylesheet"><script src="pkgdown.js"></script><meta property="og:title" content="Authors and Citation"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-citation-authors">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -71,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -93,13 +38,12 @@
   <a href="reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -112,10 +56,10 @@
       <a href="articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="articles/BestPractices.html">Best Practice Research</a>
+      <a href="articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -124,13 +68,16 @@
       <a href="articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -138,49 +85,85 @@
     <li>
       <a href="articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="contents col-md-9">
-    <div class="page-header">
-      <h1>Citation</h1>
-      <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/CITATION'><code>inst/CITATION</code></a></small>
+    <div class="section level2 authors-section">
+      <div class="page-header">
+        <h1>Authors</h1>
+      </div>
+
+      
+      <ul class="list-unstyled"><li>
+          <p><strong>Jenna Reps</strong>. Author, maintainer. 
+          </p>
+        </li>
+        <li>
+          <p><strong>Martijn Schuemie</strong>. Author. 
+          </p>
+        </li>
+        <li>
+          <p><strong>Marc Suchard</strong>. Author. 
+          </p>
+        </li>
+        <li>
+          <p><strong>Patrick Ryan</strong>. Author. 
+          </p>
+        </li>
+        <li>
+          <p><strong>Peter Rijnbeek</strong>. Author. 
+          </p>
+        </li>
+        <li>
+          <p><strong>Egill Fridgeirsson</strong>. Author. 
+          </p>
+        </li>
+      </ul></div>
+    <div class="section level2 citation-section">
+    <div>
+      <h1 id="citation">Citation</h1>
+      <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/inst/CITATION" class="external-link"><code>inst/CITATION</code></a></small>
     </div>
+    </div>
+
 
     <p>Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018).
-&ldquo;Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data.&rdquo;
+“Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data.”
 <em>Journal of the American Medical Informatics Association</em>, <b>25</b>(8), 969-975.
-<a href="https://doi.org/10.1093/jamia/ocy032">https://doi.org/10.1093/jamia/ocy032</a>. 
+<a href="https://doi.org/10.1093/jamia/ocy032" class="external-link">https://doi.org/10.1093/jamia/ocy032</a>. 
 </p>
     <pre>@Article{,
   author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
@@ -193,55 +176,26 @@ <h1>Citation</h1>
   url = {https://doi.org/10.1093/jamia/ocy032},
 }</pre>
 
-    <div class="page-header">
-      <h1>Authors</h1>
-    </div>
-
-    <ul class="list-unstyled">
-      <li>
-        <p><strong>Jenna Reps</strong>. Author, maintainer. 
-        </p>
-      </li>
-      <li>
-        <p><strong>Martijn Schuemie</strong>. Author. 
-        </p>
-      </li>
-      <li>
-        <p><strong>Marc Suchard</strong>. Author. 
-        </p>
-      </li>
-      <li>
-        <p><strong>Patrick Ryan</strong>. Author. 
-        </p>
-      </li>
-      <li>
-        <p><strong>Peter Rijnbeek</strong>. Author. 
-        </p>
-      </li>
-    </ul>
-
   </div>
 
 </div>
 
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/index.html b/docs/index.html
index d889ddc26..e9e2def0d 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -5,16 +5,16 @@
 <meta charset="utf-8">
 <meta http-equiv="X-UA-Compatible" content="IE=edge">
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
-<title>Package for patient level prediction using data in the OMOP Common Data
+<title>Developing patient level prediction using data in the OMOP Common Data
     Model • PatientLevelPrediction</title>
 <!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous">
 <script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="bootstrap-toc.css">
 <script src="bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous">
 <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous">
 <!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="pkgdown.css" rel="stylesheet">
-<script src="pkgdown.js"></script><meta property="og:title" content="Package for patient level prediction using data in the OMOP Common Data
+<script src="pkgdown.js"></script><meta property="og:title" content="Developing patient level prediction using data in the OMOP Common Data
     Model">
-<meta property="og:description" content="A package for creating patient level prediction models. Given a
+<meta property="og:description" content="A user friendly way to create patient level prediction models using the OMOP common data model. Given a
     cohort of interest and an outcome of interest, the package can use data in the
     OMOP Common Data Model to build a large set of features. These features can then
     be assessed to fit a predictive model using a number of machine learning algorithms.
@@ -25,6 +25,8 @@
 <![endif]-->
 </head>
 <body data-spy="scroll" data-target="#toc">
+    
+
     <div class="container template-home">
       <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
@@ -37,7 +39,7 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
@@ -59,7 +61,7 @@
   <a href="reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
@@ -78,10 +80,10 @@
       <a href="articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="articles/BestPractices.html">Best Practice Research</a>
+      <a href="articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -90,13 +92,16 @@
       <a href="articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -107,21 +112,30 @@
   </ul>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="news/index.html">Changelog</a>
 </li>
       </ul>
 <ul class="nav navbar-nav navbar-right">
 <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
@@ -139,52 +153,55 @@
       </header><div class="row">
   <div class="contents col-md-9">
 
-<div id="patientlevelprediction" class="section level1">
-<div class="page-header"><h1 class="hasAnchor">
-<a href="#patientlevelprediction" class="anchor"></a>PatientLevelPrediction</h1></div>
-
-<p>PatientLevelPrediction is part of <a href="https://ohdsi.github.io/Hades">HADES</a>.</p>
+<div class="section level1">
+<div class="page-header"><h1 id="patientlevelprediction">PatientLevelPrediction<a class="anchor" aria-label="anchor" href="#patientlevelprediction"></a>
+</h1></div>
+<p><a href="https://github.com/OHDSI/PatientLevelPrediction/actions?query=workflow%3AR-CMD-check" class="external-link"><img src="https://github.com/OHDSI/PatientLevelPrediction/workflows/R-CMD-check/badge.svg" alt="Build Status"></a></p>
+<p><a href="https://codecov.io/github/OHDSI/PatientLevelPrediction?branch=main" class="external-link"><img src="https://codecov.io/github/OHDSI/PatientLevelPrediction/coverage.svg?branch=main" alt="codecov.io"></a></p>
+<p>PatientLevelPrediction is part of <a href="https://ohdsi.github.io/Hades" class="external-link">HADES</a>.</p>
 </div>
-<div id="introduction" class="section level1">
-<h1 class="hasAnchor">
-<a href="#introduction" class="anchor"></a>Introduction</h1>
+<div class="section level1">
+<h1 id="introduction">Introduction<a class="anchor" aria-label="anchor" href="#introduction"></a>
+</h1>
 <p>PatientLevelPrediction is an R package for building and validating patient-level predictive models using data in the OMOP Common Data Model format.</p>
-<p>Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. <a href="https://academic.oup.com/jamia/article/25/8/969/4989437">Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data.</a> J Am Med Inform Assoc. 2018;25(8):969-975.</p>
+<p>Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. <a href="https://academic.oup.com/jamia/article/25/8/969/4989437" class="external-link">Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data.</a> J Am Med Inform Assoc. 2018;25(8):969-975.</p>
 <p>The figure below illustrates the prediction problem we address. Among a population at risk, we aim to predict which patients at a defined moment in time (t = 0) will experience some outcome during a time-at-risk. Prediction is done using only information about the patients in an observation window prior to that moment in time.</p>
-<p><img src="articles/Figure1.png"></p>
+<p><img src="articles/Figure1.webp"></p>
 <p>To define a prediction problem we have to define t=0 by a Target Cohort (T), the outcome we like to predict by an outcome cohort (O), and the time-at-risk (TAR). Furthermore, we have to make design choices for the model we like to develop, and determine the observational datasets to perform internal and external validation. This conceptual framework works for all type of prediction problems, for example those presented below (T=green, O=red).</p>
-<p><img src="articles/problems.png"></p>
+<p><img src="articles/problems.webp"></p>
 </div>
-<div id="features" class="section level1">
-<h1 class="hasAnchor">
-<a href="#features" class="anchor"></a>Features</h1>
+<div class="section level1">
+<h1 id="features">Features<a class="anchor" aria-label="anchor" href="#features"></a>
+</h1>
 <ul>
 <li>Takes one or more target cohorts (Ts) and one or more outcome cohorts (Os) and develops and validates models for all T and O combinations.</li>
 <li>Allows for multiple prediction design options.</li>
 <li>Extracts the necessary data from a database in OMOP Common Data Model format for multiple covariate settings.</li>
 <li>Uses a large set of covariates including for example all drugs, diagnoses, procedures, as well as age, comorbidity indexes, and custom covariates.</li>
-<li>Includes a large number of state-of-the-art machine learning algorithms that can be used to develop predictive models, including Regularized logistic regression, Random forest, Gradient boosting machines, Decision tree, Naive Bayes, K-nearest neighbours, Neural network and Deep learning (Convolutional neural networks, Recurrent neural network and Deep nets).</li>
+<li>Allows you to add custom covariates or cohort covariates.</li>
+<li>Includes a large number of state-of-the-art machine learning algorithms that can be used to develop predictive models, including Regularized logistic regression, Random forest, Gradient boosting machines, Decision tree, Naive Bayes, K-nearest neighbours, Neural network, AdaBoost and Support vector machines.</li>
 <li>Allows you to add custom algorithms.</li>
+<li>Allows you to add custom feature engineering</li>
+<li>Allows you to add custom under/over sampling (or any other sampling) [note: based on existing research this is not recommended]</li>
 <li>Contains functionality to externally validate models.</li>
 <li>Includes functions to plot and explore model performance (ROC + Calibration).</li>
-<li>Includes a shiny app to interactively view and explore results.</li>
-<li>Implements existing models.</li>
-<li>Builds ensemble models.</li>
-<li>Builds Deep Learning models.</li>
+<li>Build ensemble models using EnsemblePatientLevelPrediction.</li>
+<li>Build Deep Learning models using DeepPatientLevelPrediction.</li>
 <li>Generates learning curves.</li>
-<li>Automatically creates a word document containing all the study results.</li>
+<li>Includes a shiny app to interactively view and explore results.</li>
+<li>In the shiny app you can create a html file document (report or protocol) containing all the study results.</li>
 </ul>
 </div>
-<div id="screenshots" class="section level1">
-<h1 class="hasAnchor">
-<a href="#screenshots" class="anchor"></a>Screenshots</h1>
+<div class="section level1">
+<h1 id="screenshots">Screenshots<a class="anchor" aria-label="anchor" href="#screenshots"></a>
+</h1>
 <table class="table">
 <tr valign="bottom">
 <td width="50%">
-<p><img src="https://github.com/OHDSI/PatientLevelPrediction/raw/master/vignettes/sparseCalibration.png" alt="Calibration plot" title="Calibration plot"></p>
+<p><img src="https://github.com/OHDSI/PatientLevelPrediction/raw/main/vignettes/sparseCalibration.webp" alt="Calibration plot" title="Calibration plot"></p>
 </td>
 <td width="50%">
-<p><img src="https://github.com/OHDSI/PatientLevelPrediction/raw/master/vignettes/sparseROC.png" alt="ROC plot" title="ROC plot"></p>
+<p><img src="https://github.com/OHDSI/PatientLevelPrediction/raw/main/vignettes/sparseROC.webp" alt="ROC plot" title="ROC plot"></p>
 </td>
 </tr>
 <tr>
@@ -198,81 +215,79 @@ <h1 class="hasAnchor">
 </table>
 <p>Demo of the Shiny Apps can be found here:</p>
 <ul>
-<li><a href="http://data.ohdsi.org/PredictionViewer/">Single Prediction Viewer Shiny App</a></li>
-<li><a href="http://data.ohdsi.org/smokingPhenotypeExplorer/">Multiple Prediction Viewer Shiny App</a></li>
+<li><a href="http://data.ohdsi.org/smokingPhenotypeExplorer/" class="external-link">Multiple Prediction Viewer Shiny App</a></li>
 </ul>
 </div>
-<div id="technology" class="section level1">
-<h1 class="hasAnchor">
-<a href="#technology" class="anchor"></a>Technology</h1>
-<p>PatientLevelPrediction is an R package, with some functions implemented in C++ and python.</p>
+<div class="section level1">
+<h1 id="technology">Technology<a class="anchor" aria-label="anchor" href="#technology"></a>
+</h1>
+<p>PatientLevelPrediction is an R package, with some functions using python through reticulate.</p>
 </div>
-<div id="system-requirements" class="section level1">
-<h1 class="hasAnchor">
-<a href="#system-requirements" class="anchor"></a>System Requirements</h1>
-<p>Requires R (version 3.3.0 or higher). Installation on Windows requires <a href="http://cran.r-project.org/bin/windows/Rtools/">RTools</a>. Libraries used in PatientLevelPrediction require Java and Python.</p>
-<p>The python installation is required for some of the machine learning algorithms. We advise to install Python 3.7 using Anaconda (<a href="https://www.continuum.io/downloads" class="uri">https://www.continuum.io/downloads</a>).</p>
+<div class="section level1">
+<h1 id="system-requirements">System Requirements<a class="anchor" aria-label="anchor" href="#system-requirements"></a>
+</h1>
+<p>Requires R (version 4.0 or higher). Installation on Windows requires <a href="http://cran.r-project.org/bin/windows/Rtools/" class="external-link">RTools</a>. Libraries used in PatientLevelPrediction require Java and Python.</p>
+<p>The python installation is required for some of the machine learning algorithms. We advise to install Python 3.8 or higher using Anaconda (<a href="https://www.continuum.io/downloads" class="external-link uri">https://www.continuum.io/downloads</a>).</p>
 </div>
-<div id="getting-started" class="section level1">
-<h1 class="hasAnchor">
-<a href="#getting-started" class="anchor"></a>Getting Started</h1>
+<div class="section level1">
+<h1 id="getting-started">Getting Started<a class="anchor" aria-label="anchor" href="#getting-started"></a>
+</h1>
 <ul>
-<li><p>To install the package please read the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/InstallationGuide.pdf">Package Installation guide</a></p></li>
+<li><p>To install the package please read the <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/InstallationGuide.pdf" class="external-link">Package Installation guide</a></p></li>
 <li><p>Have a look at the video below for an extensive demo of the package.</p></li>
 </ul>
-<p><a href="http://www.youtube.com/watch?feature=player_embedded&amp;v=BEukCbT8UoA%0D%0A" target="_blank"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" alt="Video Vignette PLP Package" width="240" height="180" border="10"></a></p>
+<p><a href="http://www.youtube.com/watch?feature=player_embedded&amp;v=BEukCbT8UoA%0A" target="_blank" class="external-link"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" alt="Video Vignette PLP Package" width="240" height="180" border="10"></a></p>
 <p>Please read the main vignette for the package:</p>
 <ul>
-<li><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf">Building Single Patient-Level Predictive Models</a></li>
+<li><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingPredictiveModels.pdf" class="external-link">Building Single Patient-Level Predictive Models</a></li>
 </ul>
 <p>In addition we have created vignettes that describe advanced functionality in more detail:</p>
 <ul>
-<li><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingMultiplePredictiveModels.pdf">Building Multiple Patient-Level Predictive Models</a></li>
-<li><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/ImplementingExistingModels.pdf">Implementing Existing Patient-Level Predictive Models</a></li>
-<li><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomAlgorithms.pdf">Adding Custom Machine Learning Algorithms</a></li>
-<li><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingDeepLearningModels.pdf">Building Deep Learning Models</a></li>
-<li><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingEnsembleModels.pdf">Building Ensemble Models</a></li>
-<li><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/CreatingLearningCurves.pdf">Creating Learning Curves</a></li>
+<li><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingMultiplePredictiveModels.pdf" class="external-link">Building Multiple Patient-Level Predictive Models</a></li>
+<li><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/ImplementingExistingModels.pdf" class="external-link">Implementing Existing Patient-Level Predictive Models</a></li>
+<li><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/AddingCustomAlgorithms.pdf" class="external-link">Adding Custom Machine Learning Algorithms</a></li>
+<li><a href="https://github.com/OHDSI/DeepPatientLevelPrediction" class="external-link">Building Deep Learning Models</a></li>
+<li><a href="https://github.com/OHDSI/EnsemblePatientLevelPrediction" class="external-link">Building Ensemble Models</a></li>
+<li><a href="https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/CreatingLearningCurves.pdf" class="external-link">Creating Learning Curves</a></li>
 </ul>
-<p>Package manual: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/master/extras/PatientLevelPrediction.pdf">PatientLevelPrediction.pdf</a></p>
+<p>Package manual: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/main/extras/PatientLevelPrediction.pdf" class="external-link">PatientLevelPrediction.pdf</a></p>
 </div>
-<div id="user-documentation" class="section level1">
-<h1 class="hasAnchor">
-<a href="#user-documentation" class="anchor"></a>User Documentation</h1>
-<p>Documentation can be found on the <a href="https://ohdsi.github.io/PatientLevelPrediction">package website</a>.</p>
+<div class="section level1">
+<h1 id="user-documentation">User Documentation<a class="anchor" aria-label="anchor" href="#user-documentation"></a>
+</h1>
+<p>Documentation can be found on the <a href="https://ohdsi.github.io/PatientLevelPrediction" class="external-link">package website</a>.</p>
 <p>PDF versions of the documentation are also available, as mentioned above.</p>
 </div>
-<div id="support" class="section level1">
-<h1 class="hasAnchor">
-<a href="#support" class="anchor"></a>Support</h1>
+<div class="section level1">
+<h1 id="support">Support<a class="anchor" aria-label="anchor" href="#support"></a>
+</h1>
 <ul>
-<li>Developer questions/comments/feedback: <a href="http://forums.ohdsi.org/c/developers">OHDSI Forum</a>
+<li>Developer questions/comments/feedback: <a href="http://forums.ohdsi.org/c/developers" class="external-link">OHDSI Forum</a>
 </li>
-<li>We use the <a href="https://github.com/OHDSI/PatientLevelPrediction/issues">GitHub issue tracker</a> for all bugs/issues/enhancements</li>
+<li>We use the <a href="https://github.com/OHDSI/PatientLevelPrediction/issues" class="external-link">GitHub issue tracker</a> for all bugs/issues/enhancements</li>
 </ul>
 </div>
-<div id="contributing" class="section level1">
-<h1 class="hasAnchor">
-<a href="#contributing" class="anchor"></a>Contributing</h1>
-<p>Read <a href="https://ohdsi.github.io/Hades/contribute.html">here</a> how you can contribute to this package.</p>
+<div class="section level1">
+<h1 id="contributing">Contributing<a class="anchor" aria-label="anchor" href="#contributing"></a>
+</h1>
+<p>Read <a href="https://ohdsi.github.io/Hades/contribute.html" class="external-link">here</a> how you can contribute to this package.</p>
 </div>
-<div id="license" class="section level1">
-<h1 class="hasAnchor">
-<a href="#license" class="anchor"></a>License</h1>
+<div class="section level1">
+<h1 id="license">License<a class="anchor" aria-label="anchor" href="#license"></a>
+</h1>
 <p>PatientLevelPrediction is licensed under Apache License 2.0</p>
 </div>
-<div id="development" class="section level1">
-<h1 class="hasAnchor">
-<a href="#development" class="anchor"></a>Development</h1>
+<div class="section level1">
+<h1 id="development">Development<a class="anchor" aria-label="anchor" href="#development"></a>
+</h1>
 <p>PatientLevelPrediction is being developed in R Studio.</p>
-<p>Beta</p>
 </div>
-<div id="acknowledgements" class="section level1">
-<h1 class="hasAnchor">
-<a href="#acknowledgements" class="anchor"></a>Acknowledgements</h1>
+<div class="section level1">
+<h1 id="acknowledgements">Acknowledgements<a class="anchor" aria-label="anchor" href="#acknowledgements"></a>
+</h1>
 <ul>
 <li>The package is maintained by Jenna Reps and Peter Rijnbeek and has been developed with major contributions from Martijn Schuemie, Patrick Ryan, and Marc Suchard.</li>
-<li>We like to thank the following persons for their contributions to the package: Seng Chan You, Ross Williams, Henrik John, Xiaoyong Pan, James Wiggins.</li>
+<li>We like to thank the following persons for their contributions to the package: Seng Chan You, Ross Williams, Henrik John, Xiaoyong Pan, James Wiggins, Egill Fridgeirsson, Alex Rekkas</li>
 <li>This project is supported in part through the National Science Foundation grant IIS 1251151.</li>
 </ul>
 </div>
@@ -281,56 +296,55 @@ <h1 class="hasAnchor">
 
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
     <div class="links">
-<h2>Links</h2>
+<h2 data-toc-skip>Links</h2>
 <ul class="list-unstyled">
-<li>Browse source code at <br><a href="https://github.com/OHDSI/PatientLevelPrediction/">https://​github.com/​OHDSI/​PatientLevelPrediction/​</a>
-</li>
-<li>Report a bug at <br><a href="https://github.com/OHDSI/PatientLevelPrediction/issues">https://​github.com/​OHDSI/​PatientLevelPrediction/​issues</a>
-</li>
-<li>Ask a question at <br><a href="http://forums.ohdsi.org">http://​forums.ohdsi.org</a>
-</li>
+<li><a href="https://github.com/OHDSI/PatientLevelPrediction/" class="external-link">Browse source code</a></li>
+<li><a href="https://github.com/OHDSI/PatientLevelPrediction/issues" class="external-link">Report a bug</a></li>
+<li><a href="http://forums.ohdsi.org" class="external-link">Ask a question</a></li>
 </ul>
 </div>
+
 <div class="license">
-<h2>License</h2>
+<h2 data-toc-skip>License</h2>
 <ul class="list-unstyled">
 <li>Apache License 2.0</li>
 </ul>
 </div>
+
+
 <div class="citation">
-<h2>Citation</h2>
+<h2 data-toc-skip>Citation</h2>
 <ul class="list-unstyled">
-<li><a href="authors.html">Citing PatientLevelPrediction</a></li>
+<li><a href="authors.html#citation">Citing PatientLevelPrediction</a></li>
 </ul>
 </div>
+
 <div class="developers">
-<h2>Developers</h2>
+<h2 data-toc-skip>Developers</h2>
 <ul class="list-unstyled">
 <li>Jenna Reps <br><small class="roles"> Author, maintainer </small>  </li>
 <li>Martijn Schuemie <br><small class="roles"> Author </small>  </li>
 <li>Marc Suchard <br><small class="roles"> Author </small>  </li>
 <li>Patrick Ryan <br><small class="roles"> Author </small>  </li>
 <li>Peter Rijnbeek <br><small class="roles"> Author </small>  </li>
+<li>Egill Fridgeirsson <br><small class="roles"> Author </small>  </li>
 </ul>
 </div>
 
-  <div class="dev-status">
-<h2>Dev status</h2>
-<ul class="list-unstyled">
-<li><a href="https://github.com/OHDSI/PatientLevelPrediction/actions?query=workflow%3AR-CMD-check"><img src="https://github.com/OHDSI/PatientLevelPrediction/workflows/R-CMD-check/badge.svg" alt="Build Status"></a></li>
-<li><a href="https://codecov.io/github/OHDSI/PatientLevelPrediction?branch=issue242"><img src="https://codecov.io/github/OHDSI/PatientLevelPrediction/coverage.svg?branch=develop" alt="codecov.io"></a></li>
-</ul>
-</div>
-</div>
+
+
+  </div>
 </div>
 
 
       <footer><div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+  <p></p>
+<p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p>
+<p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
       </footer>
@@ -339,5 +353,7 @@ <h2>Dev status</h2>
   
 
 
+  
+
   </body>
 </html>
diff --git a/docs/news/index.html b/docs/news/index.html
index 366774228..10eee02ad 100644
--- a/docs/news/index.html
+++ b/docs/news/index.html
@@ -1,66 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Changelog • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Changelog" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Changelog • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Changelog"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-news">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -71,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -93,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -112,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -124,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -138,69 +85,260 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
       <h1 data-toc-skip>Changelog <small></small></h1>
-      <small>Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/NEWS.md'><code>NEWS.md</code></a></small>
+      <small>Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/NEWS.md" class="external-link"><code>NEWS.md</code></a></small>
     </div>
 
-    <div id="patientlevelprediction-503" class="section level1">
-<h1 class="page-header" data-toc-text="5.0.3">
-<a href="#patientlevelprediction-503" class="anchor"></a>PatientLevelPrediction 5.0.3</h1>
-<ul>
-<li>updated result schema (added model_design table with settings and added attrition table)</li>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.3.9" id="patientlevelprediction-639">PatientLevelPrediction 6.3.9<a class="anchor" aria-label="anchor" href="#patientlevelprediction-639"></a></h2>
+<ul><li>Hotfix adding schema to DatabaseConnector::getTableNames when creating results tables</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.3.8" id="patientlevelprediction-638">PatientLevelPrediction 6.3.8<a class="anchor" aria-label="anchor" href="#patientlevelprediction-638"></a></h2>
+<ul><li>Add support for R4.4</li>
+<li>Fix notes around documentation (vignette engine and brackets in itemize)</li>
+<li>Use webp image format where possible (not in pdfs) for smaller size</li>
+<li>Make sure random table names are unique in tests</li>
+<li>Remove remote info for Eunomia since it’s in CRAN</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.3.7" id="patientlevelprediction-637">PatientLevelPrediction 6.3.7<a class="anchor" aria-label="anchor" href="#patientlevelprediction-637"></a></h2>
+<ul><li>Clean up dependencies, tibble removed and IHT and ParallelLogger from CRAN</li>
+<li>Use cohortIds for cohortCovariates to comply with FeatureExtraction</li>
+<li>Add cdmDatabaseName from DatabaseDetails to model output</li>
+<li>Fix bug when attributes weren’t preserved on trainData$covariateData after split</li>
+<li>Fix warnings in tests and speed them up</li>
+<li>Fix bug in assignment operator in configurePython</li>
+<li>Delay evaluation of plpData when using do.call like in learningCurves and runMultiplePlp</li>
+<li>Speed up population generation when subjectId’s are distinct</li>
+<li>Fix bug when population was still generated when provided to runPlp</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.3.6" id="patientlevelprediction-636">PatientLevelPrediction 6.3.6<a class="anchor" aria-label="anchor" href="#patientlevelprediction-636"></a></h2>
+<ul><li>fix bug with ohdsi shiny modules version check (issue 415)</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.3.5" id="patientlevelprediction-635">PatientLevelPrediction 6.3.5<a class="anchor" aria-label="anchor" href="#patientlevelprediction-635"></a></h2>
+<ul><li>Fix sklearnToJson to be compatible with scikit-learn&gt;=1.3</li>
+<li>Fix github actions so it’s not hardcoded to use python 3.7</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.3.4" id="patientlevelprediction-634">PatientLevelPrediction 6.3.4<a class="anchor" aria-label="anchor" href="#patientlevelprediction-634"></a></h2>
+<ul><li>added spline feature engineering</li>
+<li>added age/sex stratified imputation feature engineering</li>
+<li>changed result table execution date types to varchar</li>
+<li>updated covariateSummary to use feature engineering</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.3.3" id="patientlevelprediction-633">PatientLevelPrediction 6.3.3<a class="anchor" aria-label="anchor" href="#patientlevelprediction-633"></a></h2>
+<ul><li>fixed bug introduced with new reticulate update in model saving to json tests</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.3.2" id="patientlevelprediction-632">PatientLevelPrediction 6.3.2<a class="anchor" aria-label="anchor" href="#patientlevelprediction-632"></a></h2>
+<ul><li>fixed bug with database insert if result is incomplete</li>
+<li>updated/fixed documentation (Egill)</li>
+<li>added model path to models (Henrik)</li>
+<li>updated hyper-parameter saving to data.frame and made consistent</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.3.1" id="patientlevelprediction-631">PatientLevelPrediction 6.3.1<a class="anchor" aria-label="anchor" href="#patientlevelprediction-631"></a></h2>
+<ul><li>fixed bug with multiple covariate settings in diagnose plp</li>
+<li>added min cell count when exporting database results to csv files</li>
+<li>light GBM added (thanks Jin Choi and Chungsoo Kim)</li>
+<li>fixed minor bugs when uploading results to database</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.2.1" id="patientlevelprediction-621">PatientLevelPrediction 6.2.1<a class="anchor" aria-label="anchor" href="#patientlevelprediction-621"></a></h2>
+<ul><li>added ensure_installed(“ResultModelManager”) to getDataMigrator()</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.1.0" id="patientlevelprediction-610">PatientLevelPrediction 6.1.0<a class="anchor" aria-label="anchor" href="#patientlevelprediction-610"></a></h2>
+<ul><li>shiny app is now using ShinyAppBuilder with a config saved in the /inst folder</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.0.11" id="patientlevelprediction-6011">PatientLevelPrediction 6.0.11<a class="anchor" aria-label="anchor" href="#patientlevelprediction-6011"></a></h2>
+<ul><li>fixed bugs introduced when sklearn inputs changed</li>
+<li>added sklearn model being saved as jsons</li>
+<li>made changes around the DatabaseConnection get table names function to make it work for the updated DatabaseConnection</li>
+<li>removed check RAM stop (now it just warns)</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.0.10" id="patientlevelprediction-6010">PatientLevelPrediction 6.0.10<a class="anchor" aria-label="anchor" href="#patientlevelprediction-6010"></a></h2>
+<ul><li>Updated test to skip test for FE setting if the model does not fit (this was causing occasional test fail)</li>
+<li>replaced .data$ with “” for all dplyr::select to remove warnings</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.0.9" id="patientlevelprediction-609">PatientLevelPrediction 6.0.9<a class="anchor" aria-label="anchor" href="#patientlevelprediction-609"></a></h2>
+<ul><li>Fix bug with python type being required to be int</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.0.8" id="patientlevelprediction-608">PatientLevelPrediction 6.0.8<a class="anchor" aria-label="anchor" href="#patientlevelprediction-608"></a></h2>
+<ul><li>Allow priorType to be passed down to getCV function in case prior is not ‘laplace’</li>
+<li>Seed specified in Cyclops model wasn’t passed to Cyclops</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.0.7" id="patientlevelprediction-607">PatientLevelPrediction 6.0.7<a class="anchor" aria-label="anchor" href="#patientlevelprediction-607"></a></h2>
+<ul><li>fixed issue with shiny viewer converting connection details to large json</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.0.6" id="patientlevelprediction-606">PatientLevelPrediction 6.0.6<a class="anchor" aria-label="anchor" href="#patientlevelprediction-606"></a></h2>
+<ul><li>added check for cdmDatabaseId into createDatabaseDetails</li>
+<li>added test for check for cdmDatabaseId into createDatabaseDetails to error when NULL</li>
+<li>removed session$onSessionEnded(shiny::stopApp) from shiny server</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.0.5" id="patientlevelprediction-605">PatientLevelPrediction 6.0.5<a class="anchor" aria-label="anchor" href="#patientlevelprediction-605"></a></h2>
+<ul><li>fixing cox predictions</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.0.4" id="patientlevelprediction-604">PatientLevelPrediction 6.0.4<a class="anchor" aria-label="anchor" href="#patientlevelprediction-604"></a></h2>
+<ul><li>forcing cdmDatabaseId to be a string if integer is input</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.0.3" id="patientlevelprediction-603">PatientLevelPrediction 6.0.3<a class="anchor" aria-label="anchor" href="#patientlevelprediction-603"></a></h2>
+<ul><li>replaced utils::read.csv with readr::read_csv when inserting results from csv</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.0.2" id="patientlevelprediction-602">PatientLevelPrediction 6.0.2<a class="anchor" aria-label="anchor" href="#patientlevelprediction-602"></a></h2>
+<ul><li>replaced gsub with sub when inserting csvs to database</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.0.1" id="patientlevelprediction-601">PatientLevelPrediction 6.0.1<a class="anchor" aria-label="anchor" href="#patientlevelprediction-601"></a></h2>
+<ul><li>saved result specification csv in windows to fix odd formating issue</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="6.0.0" id="patientlevelprediction-600">PatientLevelPrediction 6.0.0<a class="anchor" aria-label="anchor" href="#patientlevelprediction-600"></a></h2>
+<ul><li>fixed sample data bugs</li>
+<li>updated to use v1.0.0 of OhdsiShinyModules</li>
+<li>updated plp database result tables to use the same structure for cohort and database as other HADES packages</li>
+<li>added function to insert csv results into plp database result tables</li>
+<li>added input for databaseId (database and version) when extracting data to be consistent with other HADES packages. This is saved in plp objects.</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="5.4.4" id="patientlevelprediction-544">PatientLevelPrediction 5.4.4<a class="anchor" aria-label="anchor" href="#patientlevelprediction-544"></a></h2>
+<ul><li>fixed issue with ‘preprocess’ vs ‘preprocessing’ inconsistently used across models</li>
+<li>added metaData tracking for feature engineering or preprocessing when predicting</li>
+<li>fixed issue with FE using trainData$covariateData metaData rather than trainData</li>
+<li>fixed bug when using sameData for FE</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="5.4.3" id="patientlevelprediction-543">PatientLevelPrediction 5.4.3<a class="anchor" aria-label="anchor" href="#patientlevelprediction-543"></a></h2>
+<ul><li>pulled in multiple bug fixes and test improvements from Egill</li>
+<li>pulled in fix for learning curves from Henrik</li>
+<li>Pulled in fix for feature engineering from Solomon</li>
+<li>Cleaned check messages about comparing class(x) with a string by changing to inherits()</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="5.4.2" id="patientlevelprediction-542">PatientLevelPrediction 5.4.2<a class="anchor" aria-label="anchor" href="#patientlevelprediction-542"></a></h2>
+<ul><li>removed json saving for sklearn models since sklearn-json is no longer working for the latest sklearn</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="5.4.1" id="patientlevelprediction-541">PatientLevelPrediction 5.4.1<a class="anchor" aria-label="anchor" href="#patientlevelprediction-541"></a></h2>
+<ul><li>renamed the input corresponding to the string that gets appended to the results table names to tablePrefix</li>
+<li>fixed issues with system.file() from SqlRender code breaking the tests</li>
+<li>added an input fileAppend to the function that exports the database tables to csv files</li>
+<li>moved the plp model (including preprocessing details) outside of the result database (into a specified folder) due to the size of the objects (too large to insert into the database).</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="5.4.0" id="patientlevelprediction-540">PatientLevelPrediction 5.4.0<a class="anchor" aria-label="anchor" href="#patientlevelprediction-540"></a></h2>
+<ul><li>added saving of plp models into the result database</li>
+<li>added default cohortDefinitions in runMultiplePlp</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="5.3.3" id="patientlevelprediction-533">PatientLevelPrediction 5.3.3<a class="anchor" aria-label="anchor" href="#patientlevelprediction-533"></a></h2>
+<ul><li>added modelType to all models for database upload</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="5.3.2" id="patientlevelprediction-532">PatientLevelPrediction 5.3.2<a class="anchor" aria-label="anchor" href="#patientlevelprediction-532"></a></h2>
+<ul><li>moved FeatureExtraction to depends</li>
+<li>fixed using inherits()</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="5.3.1" id="patientlevelprediction-531">PatientLevelPrediction 5.3.1<a class="anchor" aria-label="anchor" href="#patientlevelprediction-531"></a></h2>
+<ul><li>moved most of the shiny app code into OhdsiShinyModules</li>
+<li>removed shiny dependencies and added OhdsiShinyModules to suggests</li>
+<li>fixed bug with linux sklearn saving</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="5.1.1" id="patientlevelprediction-511">PatientLevelPrediction 5.1.1<a class="anchor" aria-label="anchor" href="#patientlevelprediction-511"></a></h2>
+<ul><li>replaced cohortId to targetId for consistency throughout code</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="5.1.0" id="patientlevelprediction-510">PatientLevelPrediction 5.1.0<a class="anchor" aria-label="anchor" href="#patientlevelprediction-510"></a></h2>
+<ul><li>replaced targetId in model design to cohortId for consistency throughout code</li>
+<li>replaced plpDataSettings to restrictPlpDataSettings to improve naming consistency</li>
+<li>added ability to use initial population in runPlp by adding the population to plpData$population</li>
+<li>added splitSettings into modelDesign</li>
+<li>replaced saving json settings with ParallelLogger function</li>
+<li>updated database result schema (removed researcher_id from tables - if desired a new table with the setting_ids and researcher_id could be added, removed study tables and revised results table to performances table with a reference to model_design_id and development_database_id to enable validation results without a model to be inserted)</li>
+<li>added diagnostic code based on PROBAST</li>
+<li>added diagnostic shiny module</li>
+<li>added code to create sqlite database and populate in uploadToDatabase</li>
+<li>add code to convert runPlp+val to sqlite database when viewing shiny</li>
+<li>added code to extract database results into csv files: extractDatabaseToCsv()</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="5.0.5" id="patientlevelprediction-505">PatientLevelPrediction 5.0.5<a class="anchor" aria-label="anchor" href="#patientlevelprediction-505"></a></h2>
+<ul><li>pulled in GBM update (default hyper-parameters and variable importance fix) work done by Egill (egillax)</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="5.0.4" id="patientlevelprediction-504">PatientLevelPrediction 5.0.4<a class="anchor" aria-label="anchor" href="#patientlevelprediction-504"></a></h2>
+<ul><li>updated installation documents</li>
+<li>added tryCatch around plots to prevent code stopping</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="5.0.3" id="patientlevelprediction-503">PatientLevelPrediction 5.0.3<a class="anchor" aria-label="anchor" href="#patientlevelprediction-503"></a></h2>
+<ul><li>updated result schema (added model_design table with settings and added attrition table)</li>
 <li>updated shiny app for new database result schema</li>
 <li>removed C++ code for AUC and Rcpp dependency, now using pROC instead as faster</li>
 <li>made covariate summary optional when externally validating</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-502" class="section level1">
-<h1 class="page-header" data-toc-text="5.0.2">
-<a href="#patientlevelprediction-502" class="anchor"></a>PatientLevelPrediction 5.0.2</h1>
-<ul>
-<li>updated json structure for specifying study design (made it friendlier to read)</li>
-<li>includes smooth calibration plot fix (done by Alex)</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="5.0.2" id="patientlevelprediction-502">PatientLevelPrediction 5.0.2<a class="anchor" aria-label="anchor" href="#patientlevelprediction-502"></a></h2>
+<ul><li>updated json structure for specifying study design (made it friendlier to read)</li>
+<li>includes smooth calibration plot fix - work done by Alex (rekkasa)</li>
 <li>fixed bug with multiple sample methods or feature engineering settings causing invalid error</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-500" class="section level1">
-<h1 class="page-header" data-toc-text="5.0.0">
-<a href="#patientlevelprediction-500" class="anchor"></a>PatientLevelPrediction 5.0.0</h1>
-<ul>
-<li>plpModel now saved as json files when possible</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="5.0.0" id="patientlevelprediction-500">PatientLevelPrediction 5.0.0<a class="anchor" aria-label="anchor" href="#patientlevelprediction-500"></a></h2>
+<ul><li>plpModel now saved as json files when possible</li>
 <li>Updated runPlp to make more modular</li>
 <li>now possible to customise data splitting, feature engineering, sampling (over/under) and learning algorithm</li>
 <li>added function for extracting cohort covariates</li>
@@ -213,413 +351,256 @@ <h1 class="page-header" data-toc-text="5.0.0">
 <li>interface to cyclops code revised</li>
 <li>ensemble learning removed (will be in separate package)</li>
 <li>deep learning removed (will be in DeepPatientLevelPrediction package)</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-442" class="section level1">
-<h1 class="page-header" data-toc-text="4.4.2">
-<a href="#patientlevelprediction-442" class="anchor"></a>PatientLevelPrediction 4.4.2</h1>
-<ul>
-<li>revised toSparseM() to do conversion in one go but check RAM availablility beforehand.</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.4.2" id="patientlevelprediction-442">PatientLevelPrediction 4.4.2<a class="anchor" aria-label="anchor" href="#patientlevelprediction-442"></a></h2>
+<ul><li>revised toSparseM() to do conversion in one go but check RAM availablility beforehand.</li>
 <li>removed temporal plpData conversion in toSparseM (this will be done in DeepPatientLevelPrediction)</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-441" class="section level1">
-<h1 class="page-header" data-toc-text="4.4.1">
-<a href="#patientlevelprediction-441" class="anchor"></a>PatientLevelPrediction 4.4.1</h1>
-<ul>
-<li>shiny can now read csv results</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.4.1" id="patientlevelprediction-441">PatientLevelPrediction 4.4.1<a class="anchor" aria-label="anchor" href="#patientlevelprediction-441"></a></h2>
+<ul><li>shiny can now read csv results</li>
 <li>objects loaded via loadPlpFromCsv() can be saved using savePlpResult()</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-440" class="section level1">
-<h1 class="page-header" data-toc-text="4.4.0">
-<a href="#patientlevelprediction-440" class="anchor"></a>PatientLevelPrediction 4.4.0</h1>
-<ul>
-<li>added database result storage</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.4.0" id="patientlevelprediction-440">PatientLevelPrediction 4.4.0<a class="anchor" aria-label="anchor" href="#patientlevelprediction-440"></a></h2>
+<ul><li>added database result storage</li>
 <li>added interface to database results in shiny</li>
 <li>merged in shinyRepo that changed the shiny app to make it modular and added new features</li>
 <li>removed deep learning as this is being added into new OHDSI package DeepPatientLevelPrediction</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-4310" class="section level1">
-<h1 class="page-header" data-toc-text="4.3.10">
-<a href="#patientlevelprediction-4310" class="anchor"></a>PatientLevelPrediction 4.3.10</h1>
-<ul>
-<li>save xgboost model as json file for transparency</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.3.10" id="patientlevelprediction-4310">PatientLevelPrediction 4.3.10<a class="anchor" aria-label="anchor" href="#patientlevelprediction-4310"></a></h2>
+<ul><li>save xgboost model as json file for transparency</li>
 <li>set connectionDetails to NULL in getPlpData</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-439" class="section level1">
-<h1 class="page-header" data-toc-text="4.3.9">
-<a href="#patientlevelprediction-439" class="anchor"></a>PatientLevelPrediction 4.3.9</h1>
-<ul>
-<li>updated andromeda functions - restrict to pop and tidy covs for speed</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.3.9" id="patientlevelprediction-439">PatientLevelPrediction 4.3.9<a class="anchor" aria-label="anchor" href="#patientlevelprediction-439"></a></h2>
+<ul><li>updated andromeda functions - restrict to pop and tidy covs for speed</li>
 <li>quick fix for GBM survival predicting negative values</li>
 <li>fixed occasional demoSum error for survival models</li>
 <li>updated index creation to use Andromeda function</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-438" class="section level1">
-<h1 class="page-header" data-toc-text="4.3.8">
-<a href="#patientlevelprediction-438" class="anchor"></a>PatientLevelPrediction 4.3.8</h1>
-<ul>
-<li>fixed bug when normalize data is false</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.3.8" id="patientlevelprediction-438">PatientLevelPrediction 4.3.8<a class="anchor" aria-label="anchor" href="#patientlevelprediction-438"></a></h2>
+<ul><li>fixed bug when normalize data is false</li>
 <li>fixed bugs when single feature (gbm + python)</li>
 <li>updated GBM</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-437" class="section level1">
-<h1 class="page-header" data-toc-text="4.3.7">
-<a href="#patientlevelprediction-437" class="anchor"></a>PatientLevelPrediction 4.3.7</h1>
-<ul>
-<li>updated calibration slope</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.3.7" id="patientlevelprediction-437">PatientLevelPrediction 4.3.7<a class="anchor" aria-label="anchor" href="#patientlevelprediction-437"></a></h2>
+<ul><li>updated calibration slope</li>
 <li>fixed missing age/gender in prediction</li>
 <li>fixed shiny intercept bug</li>
 <li>fixed diagnostic</li>
 <li>fixed missing covariateSettings in load cvs plp</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-436" class="section level1">
-<h1 class="page-header" data-toc-text="4.3.6">
-<a href="#patientlevelprediction-436" class="anchor"></a>PatientLevelPrediction 4.3.6</h1>
-<ul>
-<li>Removed plpData from evaluation</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.3.6" id="patientlevelprediction-436">PatientLevelPrediction 4.3.6<a class="anchor" aria-label="anchor" href="#patientlevelprediction-436"></a></h2>
+<ul><li>Removed plpData from evaluation</li>
 <li>Added recalibration into externalVal</li>
 <li>Updated shiny app for recalibration</li>
 <li>Added population creation setting to use cohortEndDate as timeAtRisk end</li>
 <li>fixed tests</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-433" class="section level1">
-<h1 class="page-header" data-toc-text="4.3.3">
-<a href="#patientlevelprediction-433" class="anchor"></a>PatientLevelPrediction 4.3.3</h1>
-<ul>
-<li>Reduced imports by adding code to install some dependencies when used</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-432" class="section level1">
-<h1 class="page-header" data-toc-text="4.3.2">
-<a href="#patientlevelprediction-432" class="anchor"></a>PatientLevelPrediction 4.3.2</h1>
-<ul>
-<li>fixed csv result saving bug when no model param</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-431" class="section level1">
-<h1 class="page-header" data-toc-text="4.3.1">
-<a href="#patientlevelprediction-431" class="anchor"></a>PatientLevelPrediction 4.3.1</h1>
-<ul>
-<li>fixed r check vignette issues</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.3.3" id="patientlevelprediction-433">PatientLevelPrediction 4.3.3<a class="anchor" aria-label="anchor" href="#patientlevelprediction-433"></a></h2>
+<ul><li>Reduced imports by adding code to install some dependencies when used</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.3.2" id="patientlevelprediction-432">PatientLevelPrediction 4.3.2<a class="anchor" aria-label="anchor" href="#patientlevelprediction-432"></a></h2>
+<ul><li>fixed csv result saving bug when no model param</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.3.1" id="patientlevelprediction-431">PatientLevelPrediction 4.3.1<a class="anchor" aria-label="anchor" href="#patientlevelprediction-431"></a></h2>
+<ul><li>fixed r check vignette issues</li>
 <li>added conda install to test</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-430" class="section level1">
-<h1 class="page-header" data-toc-text="4.3.0">
-<a href="#patientlevelprediction-430" class="anchor"></a>PatientLevelPrediction 4.3.0</h1>
-<ul>
-<li>finalised permutation feature importance</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-4210" class="section level1">
-<h1 class="page-header" data-toc-text="4.2.10">
-<a href="#patientlevelprediction-4210" class="anchor"></a>PatientLevelPrediction 4.2.10</h1>
-<ul>
-<li>fixed deepNN index issue (reported on github - thanks dapritchard)</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.3.0" id="patientlevelprediction-430">PatientLevelPrediction 4.3.0<a class="anchor" aria-label="anchor" href="#patientlevelprediction-430"></a></h2>
+<ul><li>finalised permutation feature importance</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.2.10" id="patientlevelprediction-4210">PatientLevelPrediction 4.2.10<a class="anchor" aria-label="anchor" href="#patientlevelprediction-4210"></a></h2>
+<ul><li>fixed deepNN index issue (reported on github - thanks dapritchard)</li>
 <li>add compression to python pickles</li>
 <li>removed requirement to have outcomeCount for prediction with python models</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-429" class="section level1">
-<h1 class="page-header" data-toc-text="4.2.9">
-<a href="#patientlevelprediction-429" class="anchor"></a>PatientLevelPrediction 4.2.9</h1>
-<ul>
-<li>cleaned all checks</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.2.9" id="patientlevelprediction-429">PatientLevelPrediction 4.2.9<a class="anchor" aria-label="anchor" href="#patientlevelprediction-429"></a></h2>
+<ul><li>cleaned all checks</li>
 <li>fixed bug in python toSparseMatrix</li>
 <li>fixed warning in studyPop</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-428" class="section level1">
-<h1 class="page-header" data-toc-text="4.2.8">
-<a href="#patientlevelprediction-428" class="anchor"></a>PatientLevelPrediction 4.2.8</h1>
-<ul>
-<li>fixed bug (identified by Chungsoo) in covariateSummary</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.2.8" id="patientlevelprediction-428">PatientLevelPrediction 4.2.8<a class="anchor" aria-label="anchor" href="#patientlevelprediction-428"></a></h2>
+<ul><li>fixed bug (identified by Chungsoo) in covariateSummary</li>
 <li>fixed bug with thresholdSummary</li>
 <li>edited threshold summary function to make it cleaner</li>
 <li>added to ensemble where you can combine multiple models into an ensemble</li>
 <li>cleaned up the notes and tests</li>
 <li>updated simulated data covariateId in tests to use integer64</li>
 <li>fixed description imports (and sorted them)</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-427" class="section level1">
-<h1 class="page-header" data-toc-text="4.2.7">
-<a href="#patientlevelprediction-427" class="anchor"></a>PatientLevelPrediction 4.2.7</h1>
-<ul>
-<li>fixed Cox model calibration plots</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.2.7" id="patientlevelprediction-427">PatientLevelPrediction 4.2.7<a class="anchor" aria-label="anchor" href="#patientlevelprediction-427"></a></h2>
+<ul><li>fixed Cox model calibration plots</li>
 <li>fixed int64 conversion bug</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-426" class="section level1">
-<h1 class="page-header" data-toc-text="4.2.6">
-<a href="#patientlevelprediction-426" class="anchor"></a>PatientLevelPrediction 4.2.6</h1>
-<ul>
-<li>added baseline risk to Cox model</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-423" class="section level1">
-<h1 class="page-header" data-toc-text="4.2.3">
-<a href="#patientlevelprediction-423" class="anchor"></a>PatientLevelPrediction 4.2.3</h1>
-<ul>
-<li>updated shiny: added attrition and hyper-parameter grid search into settings</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-422" class="section level1">
-<h1 class="page-header" data-toc-text="4.2.2">
-<a href="#patientlevelprediction-422" class="anchor"></a>PatientLevelPrediction 4.2.2</h1>
-<ul>
-<li>updated shiny app added 95% CI to AUC in summary, size is now complete data size and there is a column valPercent that tells what percentage of the data were used for validation</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-421" class="section level1">
-<h1 class="page-header" data-toc-text="4.2.1">
-<a href="#patientlevelprediction-421" class="anchor"></a>PatientLevelPrediction 4.2.1</h1>
-<ul>
-<li>updated GBMsurvival to use survival metrics and c-stat</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-420" class="section level1">
-<h1 class="page-header" data-toc-text="4.2.0">
-<a href="#patientlevelprediction-420" class="anchor"></a>PatientLevelPrediction 4.2.0</h1>
-<ul>
-<li>added survival metrics</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-410" class="section level1">
-<h1 class="page-header" data-toc-text="4.1.0">
-<a href="#patientlevelprediction-410" class="anchor"></a>PatientLevelPrediction 4.1.0</h1>
-<ul>
-<li>added updates and fixes into master from development branch</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-406" class="section level1">
-<h1 class="page-header" data-toc-text="4.0.6">
-<a href="#patientlevelprediction-406" class="anchor"></a>PatientLevelPrediction 4.0.6</h1>
-<ul>
-<li>fixed bug with pdw data extraction due to multiple person_id columns</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.2.6" id="patientlevelprediction-426">PatientLevelPrediction 4.2.6<a class="anchor" aria-label="anchor" href="#patientlevelprediction-426"></a></h2>
+<ul><li>added baseline risk to Cox model</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.2.3" id="patientlevelprediction-423">PatientLevelPrediction 4.2.3<a class="anchor" aria-label="anchor" href="#patientlevelprediction-423"></a></h2>
+<ul><li>updated shiny: added attrition and hyper-parameter grid search into settings</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.2.2" id="patientlevelprediction-422">PatientLevelPrediction 4.2.2<a class="anchor" aria-label="anchor" href="#patientlevelprediction-422"></a></h2>
+<ul><li>updated shiny app added 95% CI to AUC in summary, size is now complete data size and there is a column valPercent that tells what percentage of the data were used for validation</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.2.1" id="patientlevelprediction-421">PatientLevelPrediction 4.2.1<a class="anchor" aria-label="anchor" href="#patientlevelprediction-421"></a></h2>
+<ul><li>updated GBMsurvival to use survival metrics and c-stat</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.2.0" id="patientlevelprediction-420">PatientLevelPrediction 4.2.0<a class="anchor" aria-label="anchor" href="#patientlevelprediction-420"></a></h2>
+<ul><li>added survival metrics</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.1.0" id="patientlevelprediction-410">PatientLevelPrediction 4.1.0<a class="anchor" aria-label="anchor" href="#patientlevelprediction-410"></a></h2>
+<ul><li>added updates and fixes into master from development branch</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.0.6" id="patientlevelprediction-406">PatientLevelPrediction 4.0.6<a class="anchor" aria-label="anchor" href="#patientlevelprediction-406"></a></h2>
+<ul><li>fixed bug with pdw data extraction due to multiple person_id columns</li>
 <li>fixed bug in shiny app converting covariate values due to tibble</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-405" class="section level1">
-<h1 class="page-header" data-toc-text="4.0.5">
-<a href="#patientlevelprediction-405" class="anchor"></a>PatientLevelPrediction 4.0.5</h1>
-<ul>
-<li>added calibration updates: cal-in-large, weak cal</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.0.5" id="patientlevelprediction-405">PatientLevelPrediction 4.0.5<a class="anchor" aria-label="anchor" href="#patientlevelprediction-405"></a></h2>
+<ul><li>added calibration updates: cal-in-large, weak cal</li>
 <li>updated smooth cal plot (sample for speed in big data)</li>
 <li>defaulted to 100 values in calibrationSummary + updated cal plot</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-404" class="section level1">
-<h1 class="page-header" data-toc-text="4.0.4">
-<a href="#patientlevelprediction-404" class="anchor"></a>PatientLevelPrediction 4.0.4</h1>
-<ul>
-<li>fixed backwards compat with normalization</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.0.4" id="patientlevelprediction-404">PatientLevelPrediction 4.0.4<a class="anchor" aria-label="anchor" href="#patientlevelprediction-404"></a></h2>
+<ul><li>fixed backwards compat with normalization</li>
 <li>fixed python joblib dependancy</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-402" class="section level1">
-<h1 class="page-header" data-toc-text="4.0.2">
-<a href="#patientlevelprediction-402" class="anchor"></a>PatientLevelPrediction 4.0.2</h1>
-<ul>
-<li>fixed bug in preprocessing</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.0.2" id="patientlevelprediction-402">PatientLevelPrediction 4.0.2<a class="anchor" aria-label="anchor" href="#patientlevelprediction-402"></a></h2>
+<ul><li>fixed bug in preprocessing</li>
 <li>added cross validation aucs to LR, GBM, RF and MLP</li>
 <li>added more settings into MLP</li>
 <li>added threads option in LR</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-401" class="section level1">
-<h1 class="page-header" data-toc-text="4.0.1">
-<a href="#patientlevelprediction-401" class="anchor"></a>PatientLevelPrediction 4.0.1</h1>
-<ul>
-<li>fixed minor bug with shiny dependency</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.0.1" id="patientlevelprediction-401">PatientLevelPrediction 4.0.1<a class="anchor" aria-label="anchor" href="#patientlevelprediction-401"></a></h2>
+<ul><li>fixed minor bug with shiny dependency</li>
 <li>fixed some tests</li>
 <li>added standardizedMeanDiff to covariatesummary</li>
 <li>updated createStudyPopulation to make it cleaner to read and count outcome per TAR</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-400" class="section level1">
-<h1 class="page-header" data-toc-text="4.0.0">
-<a href="#patientlevelprediction-400" class="anchor"></a>PatientLevelPrediction 4.0.0</h1>
-<ul>
-<li>Andromeda replaced ff data objects</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="4.0.0" id="patientlevelprediction-400">PatientLevelPrediction 4.0.0<a class="anchor" aria-label="anchor" href="#patientlevelprediction-400"></a></h2>
+<ul><li>Andromeda replaced ff data objects</li>
 <li>added age/gender into cohort</li>
 <li>fixed python warnings</li>
 <li>updated shiny plp viewer</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-3016" class="section level1">
-<h1 class="page-header" data-toc-text="3.0.16">
-<a href="#patientlevelprediction-3016" class="anchor"></a>PatientLevelPrediction 3.0.16</h1>
-<ul>
-<li>Fixed bug when running multiple analyses using a data extraction sample with multiple covariate settings</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-3015" class="section level1">
-<h1 class="page-header" data-toc-text="3.0.15">
-<a href="#patientlevelprediction-3015" class="anchor"></a>PatientLevelPrediction 3.0.15</h1>
-<ul>
-<li>improved shiny PLP viewer</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="3.0.16" id="patientlevelprediction-3016">PatientLevelPrediction 3.0.16<a class="anchor" aria-label="anchor" href="#patientlevelprediction-3016"></a></h2>
+<ul><li>Fixed bug when running multiple analyses using a data extraction sample with multiple covariate settings</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="3.0.15" id="patientlevelprediction-3015">PatientLevelPrediction 3.0.15<a class="anchor" aria-label="anchor" href="#patientlevelprediction-3015"></a></h2>
+<ul><li>improved shiny PLP viewer</li>
 <li>added diagnostic shiny viewer</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-3014" class="section level1">
-<h1 class="page-header" data-toc-text="3.0.14">
-<a href="#patientlevelprediction-3014" class="anchor"></a>PatientLevelPrediction 3.0.14</h1>
-<ul>
-<li>updated external validate code to enable custom covariates using ATLAS cohorts</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="3.0.14" id="patientlevelprediction-3014">PatientLevelPrediction 3.0.14<a class="anchor" aria-label="anchor" href="#patientlevelprediction-3014"></a></h2>
+<ul><li>updated external validate code to enable custom covariates using ATLAS cohorts</li>
 <li>fixed issues with startAnchor and endAnchor</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-3013" class="section level1">
-<h1 class="page-header" data-toc-text="3.0.13">
-<a href="#patientlevelprediction-3013" class="anchor"></a>PatientLevelPrediction 3.0.13</h1>
-<ul>
-<li>Deprecating addExposureDaysToStart and addExposureDaysToEnd arguments in createStudyPopulation, adding new arguments called startAnchor and endAnchor. The hope is this is less confusing.</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="3.0.13" id="patientlevelprediction-3013">PatientLevelPrediction 3.0.13<a class="anchor" aria-label="anchor" href="#patientlevelprediction-3013"></a></h2>
+<ul><li>Deprecating addExposureDaysToStart and addExposureDaysToEnd arguments in createStudyPopulation, adding new arguments called startAnchor and endAnchor. The hope is this is less confusing.</li>
 <li>fixed transfer learning code (can now transfer or fine-tune model)</li>
 <li>made view plp shiny apps work when some results are missing</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-3012" class="section level1">
-<h1 class="page-header" data-toc-text="3.0.12">
-<a href="#patientlevelprediction-3012" class="anchor"></a>PatientLevelPrediction 3.0.12</h1>
-<ul>
-<li>set up testing</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="3.0.12" id="patientlevelprediction-3012">PatientLevelPrediction 3.0.12<a class="anchor" aria-label="anchor" href="#patientlevelprediction-3012"></a></h2>
+<ul><li>set up testing</li>
 <li>fixed build warnings</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-3011" class="section level1">
-<h1 class="page-header" data-toc-text="3.0.11">
-<a href="#patientlevelprediction-3011" class="anchor"></a>PatientLevelPrediction 3.0.11</h1>
-<ul>
-<li>added tests to get &gt;70% coverage (keras tests too slow for travis)</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="3.0.11" id="patientlevelprediction-3011">PatientLevelPrediction 3.0.11<a class="anchor" aria-label="anchor" href="#patientlevelprediction-3011"></a></h2>
+<ul><li>added tests to get &gt;70% coverage (keras tests too slow for travis)</li>
 <li>Fixed minor bugs</li>
 <li>Fixed deep learning code and removed pythonInR dependancy</li>
 <li>combined shiny into one file with one interface</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-3010" class="section level1">
-<h1 class="page-header" data-toc-text="3.0.10">
-<a href="#patientlevelprediction-3010" class="anchor"></a>PatientLevelPrediction 3.0.10</h1>
-<ul>
-<li>added recalibration using 25% sample in existing models</li>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="3.0.10" id="patientlevelprediction-3010">PatientLevelPrediction 3.0.10<a class="anchor" aria-label="anchor" href="#patientlevelprediction-3010"></a></h2>
+<ul><li>added recalibration using 25% sample in existing models</li>
 <li>added option to provide score to probabilities for existing models</li>
 <li>fixed warnings with some plots</li>
-</ul>
-</div>
-    <div id="patientlevelprediction-309" class="section level1">
-<h1 class="page-header" data-toc-text="3.0.9">
-<a href="#patientlevelprediction-309" class="anchor"></a>PatientLevelPrediction 3.0.9</h1>
-<p>Small bug fixes:</p>
-<ul>
-<li>added analysisId into model saving/loading</li>
-<li>made external validation saving recursive</li>
-<li>added removal of patients with negative TAR when creating population</li>
-<li>added option to apply model without preprocessing settings (make them NULL)</li>
-<li>updated create study population to remove patients with negative time-at-risk</li>
-</ul>
+</ul></div>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="3.0.9" id="patientlevelprediction-309">PatientLevelPrediction 3.0.9<a class="anchor" aria-label="anchor" href="#patientlevelprediction-309"></a></h2>
+<p>Small bug fixes: - added analysisId into model saving/loading - made external validation saving recursive - added removal of patients with negative TAR when creating population - added option to apply model without preprocessing settings (make them NULL) - updated create study population to remove patients with negative time-at-risk</p>
 </div>
-    <div id="patientlevelprediction-308" class="section level1">
-<h1 class="page-header" data-toc-text="3.0.8">
-<a href="#patientlevelprediction-308" class="anchor"></a>PatientLevelPrediction 3.0.8</h1>
-<p>Changes:</p>
-<ul>
-<li>merged in bug fix from Martijn - fixed AUC bug causing crash with big data</li>
-<li>update SQL code to be compatible with v6.0 OMOP CDM</li>
-<li>added save option to external validate PLP</li>
-</ul>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="3.0.8" id="patientlevelprediction-308">PatientLevelPrediction 3.0.8<a class="anchor" aria-label="anchor" href="#patientlevelprediction-308"></a></h2>
+<p>Changes: - merged in bug fix from Martijn - fixed AUC bug causing crash with big data - update SQL code to be compatible with v6.0 OMOP CDM - added save option to external validate PLP</p>
 </div>
-    <div id="patientlevelprediction-307" class="section level1">
-<h1 class="page-header" data-toc-text="3.0.7">
-<a href="#patientlevelprediction-307" class="anchor"></a>PatientLevelPrediction 3.0.7</h1>
-<p>Changes:</p>
-<ul>
-<li>Updated splitting functions to include a splitby subject and renamed personSplitter to randomSplitter</li>
-<li>Cast indices to integer in python functions to fix bug with non integer sparse matrix indices</li>
-</ul>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="3.0.7" id="patientlevelprediction-307">PatientLevelPrediction 3.0.7<a class="anchor" aria-label="anchor" href="#patientlevelprediction-307"></a></h2>
+<p>Changes: - Updated splitting functions to include a splitby subject and renamed personSplitter to randomSplitter - Cast indices to integer in python functions to fix bug with non integer sparse matrix indices</p>
 </div>
-    <div id="patientlevelprediction-305" class="section level1">
-<h1 class="page-header" data-toc-text="3.0.5">
-<a href="#patientlevelprediction-305" class="anchor"></a>PatientLevelPrediction 3.0.5</h1>
-<p>Changes:</p>
-<ul>
-<li>Added GLM status to log (will now inform about any fitting issue in log)</li>
-<li>Added GBM survival model (still under development)</li>
-<li>Added RF quantile regression (still under development)</li>
-<li>Updated viewMultiplePlp() to match PLP skeleton package app</li>
-<li>Updated single plp vignette with additional example</li>
-<li>Merge in deep learning updates from Chan</li>
-</ul>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="3.0.5" id="patientlevelprediction-305">PatientLevelPrediction 3.0.5<a class="anchor" aria-label="anchor" href="#patientlevelprediction-305"></a></h2>
+<p>Changes: - Added GLM status to log (will now inform about any fitting issue in log) - Added GBM survival model (still under development) - Added RF quantile regression (still under development) - Updated viewMultiplePlp() to match PLP skeleton package app - Updated single plp vignette with additional example - Merge in deep learning updates from Chan</p>
 </div>
-    <div id="patientlevelprediction-304" class="section level1">
-<h1 class="page-header" data-toc-text="3.0.4">
-<a href="#patientlevelprediction-304" class="anchor"></a>PatientLevelPrediction 3.0.4</h1>
-<p>Changes:</p>
-<ul>
-<li>Updated website</li>
-</ul>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="3.0.4" id="patientlevelprediction-304">PatientLevelPrediction 3.0.4<a class="anchor" aria-label="anchor" href="#patientlevelprediction-304"></a></h2>
+<p>Changes: - Updated website</p>
 </div>
-    <div id="patientlevelprediction-303" class="section level1">
-<h1 class="page-header" data-toc-text="3.0.3">
-<a href="#patientlevelprediction-303" class="anchor"></a>PatientLevelPrediction 3.0.3</h1>
-<p>Changes:</p>
-<ul>
-<li>Added more tests</li>
-<li>test files now match R files</li>
-</ul>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="3.0.3" id="patientlevelprediction-303">PatientLevelPrediction 3.0.3<a class="anchor" aria-label="anchor" href="#patientlevelprediction-303"></a></h2>
+<p>Changes: - Added more tests - test files now match R files</p>
 </div>
-    <div id="patientlevelprediction-302" class="section level1">
-<h1 class="page-header" data-toc-text="3.0.2">
-<a href="#patientlevelprediction-302" class="anchor"></a>PatientLevelPrediction 3.0.2</h1>
-<p>Changes:</p>
-<ul>
-<li>Fixed ensemble stacker</li>
-</ul>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="3.0.2" id="patientlevelprediction-302">PatientLevelPrediction 3.0.2<a class="anchor" aria-label="anchor" href="#patientlevelprediction-302"></a></h2>
+<p>Changes: - Fixed ensemble stacker</p>
 </div>
-    <div id="patientlevelprediction-301" class="section level1">
-<h1 class="page-header" data-toc-text="3.0.1">
-<a href="#patientlevelprediction-301" class="anchor"></a>PatientLevelPrediction 3.0.1</h1>
-<p>Changes:</p>
-<ul>
-<li>Using reticulate for python interface</li>
-<li>Speed improvements</li>
-<li>Bug fixes</li>
-</ul>
+    <div class="section level2">
+<h2 class="page-header" data-toc-text="3.0.1" id="patientlevelprediction-301">PatientLevelPrediction 3.0.1<a class="anchor" aria-label="anchor" href="#patientlevelprediction-301"></a></h2>
+<p>Changes: - Using reticulate for python interface - Speed improvements - Bug fixes</p>
 </div>
   </div>
 
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/pkgdown.css b/docs/pkgdown.css
index 1273238dd..80ea5b838 100644
--- a/docs/pkgdown.css
+++ b/docs/pkgdown.css
@@ -56,8 +56,10 @@ img.icon {
   float: right;
 }
 
-img {
+/* Ensure in-page images don't run outside their container */
+.contents img {
   max-width: 100%;
+  height: auto;
 }
 
 /* Fix bug in bootstrap (only seen in firefox) */
@@ -78,11 +80,10 @@ dd {
 /* Section anchors ---------------------------------*/
 
 a.anchor {
-  margin-left: -30px;
-  display:inline-block;
-  width: 30px;
-  height: 30px;
-  visibility: hidden;
+  display: none;
+  margin-left: 5px;
+  width: 20px;
+  height: 20px;
 
   background-image: url(./link.svg);
   background-repeat: no-repeat;
@@ -90,17 +91,15 @@ a.anchor {
   background-position: center center;
 }
 
-.hasAnchor:hover a.anchor {
-  visibility: visible;
-}
-
-@media (max-width: 767px) {
-  .hasAnchor:hover a.anchor {
-    visibility: hidden;
-  }
+h1:hover .anchor,
+h2:hover .anchor,
+h3:hover .anchor,
+h4:hover .anchor,
+h5:hover .anchor,
+h6:hover .anchor {
+  display: inline-block;
 }
 
-
 /* Fixes for fixed navbar --------------------------*/
 
 .contents h1, .contents h2, .contents h3, .contents h4 {
@@ -264,31 +263,26 @@ table {
 
 /* Syntax highlighting ---------------------------------------------------- */
 
-pre {
-  word-wrap: normal;
-  word-break: normal;
-  border: 1px solid #eee;
-}
-
-pre, code {
+pre, code, pre code {
   background-color: #f8f8f8;
   color: #333;
 }
+pre, pre code {
+  white-space: pre-wrap;
+  word-break: break-all;
+  overflow-wrap: break-word;
+}
 
-pre code {
-  overflow: auto;
-  word-wrap: normal;
-  white-space: pre;
+pre {
+  border: 1px solid #eee;
 }
 
-pre .img {
+pre .img, pre .r-plt {
   margin: 5px 0;
 }
 
-pre .img img {
+pre .img img, pre .r-plt img {
   background-color: #fff;
-  display: block;
-  height: auto;
 }
 
 code a, pre a {
@@ -305,9 +299,8 @@ a.sourceLine:hover {
 .kw      {color: #264D66;} /* keyword */
 .co      {color: #888888;} /* comment */
 
-.message { color: black;   font-weight: bolder;}
-.error   { color: orange;  font-weight: bolder;}
-.warning { color: #6A0366; font-weight: bolder;}
+.error   {font-weight: bolder;}
+.warning {font-weight: bolder;}
 
 /* Clipboard --------------------------*/
 
@@ -365,3 +358,27 @@ mark {
     content: "";
   }
 }
+
+/* Section anchors ---------------------------------
+   Added in pandoc 2.11: https://github.com/jgm/pandoc-templates/commit/9904bf71
+*/
+
+div.csl-bib-body { }
+div.csl-entry {
+  clear: both;
+}
+.hanging-indent div.csl-entry {
+  margin-left:2em;
+  text-indent:-2em;
+}
+div.csl-left-margin {
+  min-width:2em;
+  float:left;
+}
+div.csl-right-inline {
+  margin-left:2em;
+  padding-left:1em;
+}
+div.csl-indent {
+  margin-left: 2em;
+}
diff --git a/docs/pkgdown.js b/docs/pkgdown.js
index 7e7048fae..6f0eee40b 100644
--- a/docs/pkgdown.js
+++ b/docs/pkgdown.js
@@ -80,7 +80,7 @@
     $(document).ready(function() {
       var copyButton = "<button type='button' class='btn btn-primary btn-copy-ex' type = 'submit' title='Copy to clipboard' aria-label='Copy to clipboard' data-toggle='tooltip' data-placement='left auto' data-trigger='hover' data-clipboard-copy><i class='fa fa-copy'></i></button>";
 
-      $(".examples, div.sourceCode").addClass("hasCopyButton");
+      $("div.sourceCode").addClass("hasCopyButton");
 
       // Insert copy buttons:
       $(copyButton).prependTo(".hasCopyButton");
@@ -91,7 +91,7 @@
       // Initialize clipboard:
       var clipboardBtnCopies = new ClipboardJS('[data-clipboard-copy]', {
         text: function(trigger) {
-          return trigger.parentNode.textContent;
+          return trigger.parentNode.textContent.replace(/\n#>[^\n]*/g, "");
         }
       });
 
diff --git a/docs/pkgdown.yml b/docs/pkgdown.yml
index 7f8ad5eb7..24865cad6 100644
--- a/docs/pkgdown.yml
+++ b/docs/pkgdown.yml
@@ -1,20 +1,20 @@
-pandoc: 2.11.4
-pkgdown: 1.6.1
+pandoc: 3.1.11
+pkgdown: 2.0.7
 pkgdown_sha: ~
 articles:
   AddingCustomFeatureEngineering: AddingCustomFeatureEngineering.html
   AddingCustomModels: AddingCustomModels.html
   AddingCustomSamples: AddingCustomSamples.html
   AddingCustomSplitting: AddingCustomSplitting.html
+  BenchmarkTasks: BenchmarkTasks.html
   BestPractices: BestPractices.html
-  BuildingEnsembleModels: BuildingEnsembleModels.html
   BuildingMultiplePredictiveModels: BuildingMultiplePredictiveModels.html
   BuildingPredictiveModels: BuildingPredictiveModels.html
+  ClinicalModels: ClinicalModels.html
+  ConstrainedPredictors: ConstrainedPredictors.html
   CreatingLearningCurves: CreatingLearningCurves.html
   CreatingNetworkStudies: CreatingNetworkStudies.html
-  CreatingShinyApp: CreatingShinyApp.html
   InstallationGuide: InstallationGuide.html
-  PatientLevelPrediction: PatientLevelPrediction.html
   Videos: Videos.html
-last_built: 2022-03-09T19:04Z
+last_built: 2024-09-09T14:25Z
 
diff --git a/docs/reference/MapIds.html b/docs/reference/MapIds.html
new file mode 100644
index 000000000..5cb0b7906
--- /dev/null
+++ b/docs/reference/MapIds.html
@@ -0,0 +1,178 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Map covariate and row Ids so they start from 1 — MapIds • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Map covariate and row Ids so they start from 1 — MapIds"><meta property="og:description" content="this functions takes covariate data and a cohort/population and remaps 
+the covariate and row ids, restricts to pop and saves/creates mapping"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Map covariate and row Ids so they start from 1</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Formatting.R" class="external-link"><code>R/Formatting.R</code></a></small>
+    <div class="hidden name"><code>MapIds.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>this functions takes covariate data and a cohort/population and remaps 
+the covariate and row ids, restricts to pop and saves/creates mapping</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">MapIds</span><span class="op">(</span><span class="va">covariateData</span>, cohort <span class="op">=</span> <span class="cn">NULL</span>, mapping <span class="op">=</span> <span class="cn">NULL</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>covariateData</dt>
+<dd><p>a covariateData object</p></dd>
+
+
+<dt>cohort</dt>
+<dd><p>if specified rowIds restricted to the ones in cohort</p></dd>
+
+
+<dt>mapping</dt>
+<dd><p>A pre defined mapping to use</p></dd>
+
+</dl></div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/PatientLevelPrediction.html b/docs/reference/PatientLevelPrediction.html
index 635b9807e..77e24885e 100644
--- a/docs/reference/PatientLevelPrediction.html
+++ b/docs/reference/PatientLevelPrediction.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>PatientLevelPrediction — PatientLevelPrediction • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="PatientLevelPrediction — PatientLevelPrediction" />
-<meta property="og:description" content="A package for running predictions using data in the OMOP CDM" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>PatientLevelPrediction — PatientLevelPrediction • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="PatientLevelPrediction — PatientLevelPrediction"><meta property="og:description" content="A package for running predictions using data in the OMOP CDM"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>PatientLevelPrediction</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/PatientLevelPrediction.R'><code>R/PatientLevelPrediction.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/PatientLevelPrediction.R" class="external-link"><code>R/PatientLevelPrediction.R</code></a></small>
     <div class="hidden name"><code>PatientLevelPrediction.Rd</code></div>
     </div>
 
@@ -184,33 +132,44 @@ <h1>PatientLevelPrediction</h1>
     </div>
 
 
-
+    <div id="see-also">
+    <h2>See also</h2>
+    <div class="dont-index"><p>Useful links:</p><ul><li><p><a href="https://ohdsi.github.io/PatientLevelPrediction" class="external-link">https://ohdsi.github.io/PatientLevelPrediction</a></p></li>
+<li><p><a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">https://github.com/OHDSI/PatientLevelPrediction</a></p></li>
+<li><p>Report bugs at <a href="https://github.com/OHDSI/PatientLevelPrediction/issues" class="external-link">https://github.com/OHDSI/PatientLevelPrediction/issues</a></p></li>
+</ul></div>
+    </div>
+    <div id="author">
+    <h2>Author</h2>
+    <p><strong>Maintainer</strong>: Jenna Reps <a href="mailto:jreps@its.jnj.com">jreps@its.jnj.com</a></p>
+<p>Authors:</p><ul><li><p>Martijn Schuemie</p></li>
+<li><p>Marc Suchard</p></li>
+<li><p>Patrick Ryan</p></li>
+<li><p>Peter Rijnbeek</p></li>
+<li><p>Egill Fridgeirsson</p></li>
+</ul></div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/accuracy.html b/docs/reference/accuracy.html
index 92d9a6902..111a58e86 100644
--- a/docs/reference/accuracy.html
+++ b/docs/reference/accuracy.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculate the accuracy — accuracy • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculate the accuracy — accuracy" />
-<meta property="og:description" content="Calculate the accuracy" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculate the accuracy — accuracy • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculate the accuracy — accuracy"><meta property="og:description" content="Calculate the accuracy"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculate the accuracy</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ThresholdSummary.R'><code>R/ThresholdSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ThresholdSummary.R" class="external-link"><code>R/ThresholdSummary.R</code></a></small>
     <div class="hidden name"><code>accuracy.Rd</code></div>
     </div>
 
@@ -183,61 +131,60 @@ <h1>Calculate the accuracy</h1>
     <p>Calculate the accuracy</p>
     </div>
 
-    <pre class="usage"><span class='fu'>accuracy</span><span class='op'>(</span><span class='va'>TP</span>, <span class='va'>TN</span>, <span class='va'>FN</span>, <span class='va'>FP</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>TP</th>
-      <td><p>Number of true positives</p></td>
-    </tr>
-    <tr>
-      <th>TN</th>
-      <td><p>Number of true negatives</p></td>
-    </tr>
-    <tr>
-      <th>FN</th>
-      <td><p>Number of false negatives</p></td>
-    </tr>
-    <tr>
-      <th>FP</th>
-      <td><p>Number of false positives</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>accuracy value</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">accuracy</span><span class="op">(</span><span class="va">TP</span>, <span class="va">TN</span>, <span class="va">FN</span>, <span class="va">FP</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>TP</dt>
+<dd><p>Number of true positives</p></dd>
+
+
+<dt>TN</dt>
+<dd><p>Number of true negatives</p></dd>
+
+
+<dt>FN</dt>
+<dd><p>Number of false negatives</p></dd>
+
+
+<dt>FP</dt>
+<dd><p>Number of false positives</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>accuracy value</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculate the accuracy</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/addDiagnosePlpToDatabase.html b/docs/reference/addDiagnosePlpToDatabase.html
new file mode 100644
index 000000000..31b3a41d3
--- /dev/null
+++ b/docs/reference/addDiagnosePlpToDatabase.html
@@ -0,0 +1,207 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Insert a diagnostic result into a PLP result schema database — addDiagnosePlpToDatabase • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Insert a diagnostic result into a PLP result schema database — addDiagnosePlpToDatabase"><meta property="og:description" content="This function inserts a diagnostic result into the result schema"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Insert a diagnostic result into a PLP result schema database</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/uploadToDatabaseDiagnostics.R" class="external-link"><code>R/uploadToDatabaseDiagnostics.R</code></a></small>
+    <div class="hidden name"><code>addDiagnosePlpToDatabase.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>This function inserts a diagnostic result into the result schema</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">addDiagnosePlpToDatabase</span><span class="op">(</span></span>
+<span>  <span class="va">diagnosePlp</span>,</span>
+<span>  <span class="va">connectionDetails</span>,</span>
+<span>  <span class="va">databaseSchemaSettings</span>,</span>
+<span>  <span class="va">cohortDefinitions</span>,</span>
+<span>  databaseList <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  overWriteIfExists <span class="op">=</span> <span class="cn">T</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>diagnosePlp</dt>
+<dd><p>An object of class <code>diagnosePlp</code></p></dd>
+
+
+<dt>connectionDetails</dt>
+<dd><p>A connection details created by using the
+function <code>createConnectionDetails</code> in the
+<code>DatabaseConnector</code> package.</p></dd>
+
+
+<dt>databaseSchemaSettings</dt>
+<dd><p>A object created by <code>createDatabaseSchemaSettings</code> with all the settings specifying the result tables</p></dd>
+
+
+<dt>cohortDefinitions</dt>
+<dd><p>A set of one or more cohorts extracted using ROhdsiWebApi::exportCohortDefinitionSet()</p></dd>
+
+
+<dt>databaseList</dt>
+<dd><p>(Optional) If you wish to overwrite the settings in the plp object use <code>createdatabaseList</code> to specify the databases</p></dd>
+
+
+<dt>overWriteIfExists</dt>
+<dd><p>(default: T) Whether to delete existing results and overwrite them</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>Returns NULL but uploads the diagnostic into the database schema specified in databaseSchemaSettings</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>This function can be used to upload a diagnostic result into a database</p>
+    </div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/addMultipleDiagnosePlpToDatabase.html b/docs/reference/addMultipleDiagnosePlpToDatabase.html
new file mode 100644
index 000000000..56ee9c4c8
--- /dev/null
+++ b/docs/reference/addMultipleDiagnosePlpToDatabase.html
@@ -0,0 +1,202 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Insert mutliple diagnosePlp results saved to a directory into a PLP result schema database — addMultipleDiagnosePlpToDatabase • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Insert mutliple diagnosePlp results saved to a directory into a PLP result schema database — addMultipleDiagnosePlpToDatabase"><meta property="og:description" content="This function inserts diagnosePlp results into the result schema"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Insert mutliple diagnosePlp results saved to a directory into a PLP result schema database</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/uploadToDatabaseDiagnostics.R" class="external-link"><code>R/uploadToDatabaseDiagnostics.R</code></a></small>
+    <div class="hidden name"><code>addMultipleDiagnosePlpToDatabase.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>This function inserts diagnosePlp results into the result schema</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">addMultipleDiagnosePlpToDatabase</span><span class="op">(</span></span>
+<span>  <span class="va">connectionDetails</span>,</span>
+<span>  <span class="va">databaseSchemaSettings</span>,</span>
+<span>  <span class="va">cohortDefinitions</span>,</span>
+<span>  databaseList <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  <span class="va">resultLocation</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>connectionDetails</dt>
+<dd><p>A connection details created by using the
+function <code>createConnectionDetails</code> in the
+<code>DatabaseConnector</code> package.</p></dd>
+
+
+<dt>databaseSchemaSettings</dt>
+<dd><p>A object created by <code>createDatabaseSchemaSettings</code> with all the settings specifying the result tables</p></dd>
+
+
+<dt>cohortDefinitions</dt>
+<dd><p>(list) A list of cohortDefinitions (each list must contain: name, id)</p></dd>
+
+
+<dt>databaseList</dt>
+<dd><p>(Optional) ...</p></dd>
+
+
+<dt>resultLocation</dt>
+<dd><p>The location of the diagnostic results</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>Returns NULL but uploads multiple diagnosePlp results into the database schema specified in databaseSchemaSettings</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>This function can be used to upload diagnosePlp results into a database</p>
+    </div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/addMultipleRunPlpToDatabase.html b/docs/reference/addMultipleRunPlpToDatabase.html
new file mode 100644
index 000000000..47b53a4c0
--- /dev/null
+++ b/docs/reference/addMultipleRunPlpToDatabase.html
@@ -0,0 +1,212 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Populate the PatientLevelPrediction results tables — addMultipleRunPlpToDatabase • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Populate the PatientLevelPrediction results tables — addMultipleRunPlpToDatabase"><meta property="og:description" content="This function formats and uploads results that have been generated via an ATLAS prediction package into a database"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Populate the PatientLevelPrediction results tables</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/uploadToDatabase.R" class="external-link"><code>R/uploadToDatabase.R</code></a></small>
+    <div class="hidden name"><code>addMultipleRunPlpToDatabase.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>This function formats and uploads results that have been generated via an ATLAS prediction package into a database</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">addMultipleRunPlpToDatabase</span><span class="op">(</span></span>
+<span>  <span class="va">connectionDetails</span>,</span>
+<span>  databaseSchemaSettings <span class="op">=</span> <span class="fu"><a href="createDatabaseSchemaSettings.html">createDatabaseSchemaSettings</a></span><span class="op">(</span>resultSchema <span class="op">=</span> <span class="st">"main"</span><span class="op">)</span>,</span>
+<span>  <span class="va">cohortDefinitions</span>,</span>
+<span>  databaseList <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  resultLocation <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  <span class="va">resultLocationVector</span>,</span>
+<span>  <span class="va">modelSaveLocation</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>connectionDetails</dt>
+<dd><p>A connection details created by using the
+function <code>createConnectionDetails</code> in the
+<code>DatabaseConnector</code> package.</p></dd>
+
+
+<dt>databaseSchemaSettings</dt>
+<dd><p>A object created by <code>createDatabaseSchemaSettings</code> with all the settings specifying the result tables</p></dd>
+
+
+<dt>cohortDefinitions</dt>
+<dd><p>A set of one or more cohorts extracted using ROhdsiWebApi::exportCohortDefinitionSet()</p></dd>
+
+
+<dt>databaseList</dt>
+<dd><p>(Optional) A list created by <code>createDatabaseList</code> to specify the databases</p></dd>
+
+
+<dt>resultLocation</dt>
+<dd><p>(string) location of directory where the main package results were saved</p></dd>
+
+
+<dt>resultLocationVector</dt>
+<dd><p>(only used when resultLocation is missing) a vector of locations with development or validation results</p></dd>
+
+
+<dt>modelSaveLocation</dt>
+<dd><p>The location of the file system for saving the models in a subdirectory</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>Returns NULL but uploads all the results in resultLocation to the PatientLevelPrediction result tables in resultSchema</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>This function can be used upload PatientLevelPrediction results into a database</p>
+    </div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/addRecalibration.html b/docs/reference/addRecalibration.html
deleted file mode 100644
index 2b920805e..000000000
--- a/docs/reference/addRecalibration.html
+++ /dev/null
@@ -1,229 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>addRecalibration — addRecalibration • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="addRecalibration — addRecalibration" />
-<meta property="og:description" content="Adds the recalibration results to the main results" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>addRecalibration</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Recalibration.R'><code>R/Recalibration.R</code></a></small>
-    <div class="hidden name"><code>addRecalibration.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Adds the recalibration results to the main results</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>addRecalibration</span><span class='op'>(</span><span class='va'>performanceEvaluation</span>, <span class='va'>recalibration</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>performanceEvaluation</th>
-      <td><p>The main result performanceEvaluation</p></td>
-    </tr>
-    <tr>
-      <th>recalibration</th>
-      <td><p>The recalibration result</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>An object of class <code>runPlp</code> that is recalibrated on the new data</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Append the recalibration results into the main results</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/addRunPlpToDatabase.html b/docs/reference/addRunPlpToDatabase.html
new file mode 100644
index 000000000..9752d3ec5
--- /dev/null
+++ b/docs/reference/addRunPlpToDatabase.html
@@ -0,0 +1,207 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Function to add the run plp (development or validation) to database — addRunPlpToDatabase • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Function to add the run plp (development or validation) to database — addRunPlpToDatabase"><meta property="og:description" content="This function adds a runPlp or external validation result into a database"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Function to add the run plp (development or validation) to database</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/uploadToDatabase.R" class="external-link"><code>R/uploadToDatabase.R</code></a></small>
+    <div class="hidden name"><code>addRunPlpToDatabase.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>This function adds a runPlp or external validation result into a database</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">addRunPlpToDatabase</span><span class="op">(</span></span>
+<span>  <span class="va">runPlp</span>,</span>
+<span>  <span class="va">connectionDetails</span>,</span>
+<span>  <span class="va">databaseSchemaSettings</span>,</span>
+<span>  <span class="va">cohortDefinitions</span>,</span>
+<span>  <span class="va">modelSaveLocation</span>,</span>
+<span>  databaseList <span class="op">=</span> <span class="cn">NULL</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>runPlp</dt>
+<dd><p>An object of class <code>runPlp</code> or class <code>externalValidatePlp</code></p></dd>
+
+
+<dt>connectionDetails</dt>
+<dd><p>A connection details created by using the
+function <code>createConnectionDetails</code> in the
+<code>DatabaseConnector</code> package.</p></dd>
+
+
+<dt>databaseSchemaSettings</dt>
+<dd><p>A object created by <code>createDatabaseSchemaSettings</code> with all the settings specifying the result tables</p></dd>
+
+
+<dt>cohortDefinitions</dt>
+<dd><p>A set of one or more cohorts extracted using ROhdsiWebApi::exportCohortDefinitionSet()</p></dd>
+
+
+<dt>modelSaveLocation</dt>
+<dd><p>The location of the directory that models will be saved to</p></dd>
+
+
+<dt>databaseList</dt>
+<dd><p>(Optional) If you want to change the database name then used <code>createDatabaseList</code> to specify the database settings but use the same cdmDatabaseId was model development/validation</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>Returns a data.frame with the database details</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>This function is used when inserting results into the PatientLevelPrediction database results schema</p>
+    </div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/applyEnsembleModel.html b/docs/reference/applyEnsembleModel.html
deleted file mode 100644
index cc9346c8e..000000000
--- a/docs/reference/applyEnsembleModel.html
+++ /dev/null
@@ -1,280 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Apply trained ensemble model on new data Apply a Patient Level Prediction model on Patient Level
-Prediction Data and get the predicted risk in [0,1] for each person in the population. If the user
-inputs a population with an outcomeCount column then the function also returns the evaluation of
-the prediction (AUC, brier score, calibration) — applyEnsembleModel • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Apply trained ensemble model on new data Apply a Patient Level Prediction model on Patient Level
-Prediction Data and get the predicted risk in [0,1] for each person in the population. If the user
-inputs a population with an outcomeCount column then the function also returns the evaluation of
-the prediction (AUC, brier score, calibration) — applyEnsembleModel" />
-<meta property="og:description" content="Apply trained ensemble model on new data Apply a Patient Level Prediction model on Patient Level
-Prediction Data and get the predicted risk in [0,1] for each person in the population. If the user
-inputs a population with an outcomeCount column then the function also returns the evaluation of
-the prediction (AUC, brier score, calibration)" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Apply trained ensemble model on new data Apply a Patient Level Prediction model on Patient Level
-Prediction Data and get the predicted risk in [0,1] for each person in the population. If the user
-inputs a population with an outcomeCount column then the function also returns the evaluation of
-the prediction (AUC, brier score, calibration)</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Ensemble.R'><code>R/Ensemble.R</code></a></small>
-    <div class="hidden name"><code>applyEnsembleModel.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Apply trained ensemble model on new data Apply a Patient Level Prediction model on Patient Level
-Prediction Data and get the predicted risk in [0,1] for each person in the population. If the user
-inputs a population with an outcomeCount column then the function also returns the evaluation of
-the prediction (AUC, brier score, calibration)</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>applyEnsembleModel</span><span class='op'>(</span>
-  <span class='va'>population</span>,
-  <span class='va'>dataList</span>,
-  <span class='va'>ensembleModel</span>,
-  analysisId <span class='op'>=</span> <span class='cn'>NULL</span>,
-  calculatePerformance <span class='op'>=</span> <span class='cn'>T</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>population</th>
-      <td><p>The population of people who you want to predict the risk for</p></td>
-    </tr>
-    <tr>
-      <th>dataList</th>
-      <td><p>The plpData list for the population</p></td>
-    </tr>
-    <tr>
-      <th>ensembleModel</th>
-      <td><p>The trained ensemble model returned by running runEnsembleModel</p></td>
-    </tr>
-    <tr>
-      <th>analysisId</th>
-      <td><p>The analysis ID, which is the ID of running ensemble model training.</p></td>
-    </tr>
-    <tr>
-      <th>calculatePerformance</th>
-      <td><p>Whether to also calculate the performance metrics [default TRUE]</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='co'># load the model and data</span>
-<span class='va'>plpData</span> <span class='op'>&lt;-</span> <span class='fu'><a href='loadPlpData.html'>loadPlpData</a></span><span class='op'>(</span><span class='st'>"plpdata/"</span><span class='op'>)</span>
-<span class='va'>results</span> <span class='op'>&lt;-</span> <span class='fu'>PatientLevelPrediction</span><span class='fu'>::</span><span class='fu'><a href='runEnsembleModel.html'>runEnsembleModel</a></span><span class='op'>(</span><span class='va'>population</span>,
-                                                    dataList <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='va'>plpData</span>, <span class='va'>plpData</span><span class='op'>)</span>,
-                                                    modelList <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='va'>model</span>, <span class='va'>model</span><span class='op'>)</span>,
-                                                    testSplit <span class='op'>=</span> <span class='st'>"person"</span>,
-                                                    testFraction <span class='op'>=</span> <span class='fl'>0.2</span>,
-                                                    nfold <span class='op'>=</span> <span class='fl'>3</span>,
-                                                    splitSeed <span class='op'>=</span> <span class='fl'>1000</span>,
-                                                    ensembleStrategy <span class='op'>=</span> <span class='st'>"stacked"</span><span class='op'>)</span>
-<span class='co'># use the same population settings as the model:</span>
-<span class='va'>populationSettings</span> <span class='op'>&lt;-</span> <span class='va'>plpModel</span><span class='op'>$</span><span class='va'>populationSettings</span>
-<span class='va'>populationSettings</span><span class='op'>$</span><span class='va'>plpData</span> <span class='op'>&lt;-</span> <span class='va'>plpData</span>
-<span class='va'>population</span> <span class='op'>&lt;-</span> <span class='fu'><a href='https://rdrr.io/r/base/do.call.html'>do.call</a></span><span class='op'>(</span><span class='va'>createStudyPopulation</span>, <span class='va'>populationSettings</span><span class='op'>)</span>
-
-<span class='co'># get the prediction, please make sure the ensemble strategy for training and apply is the same:</span>
-<span class='va'>prediction</span> <span class='op'>&lt;-</span> <span class='fu'>applyEnsembleModel</span><span class='op'>(</span><span class='va'>population</span>,
-                                 dataList <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='va'>plpData</span>, <span class='va'>plpData</span><span class='op'>)</span>,
-                                 ensembleModel <span class='op'>=</span> <span class='va'>results</span>,
-                                 analysisId <span class='op'>=</span> <span class='cn'>NULL</span><span class='op'>)</span><span class='op'>$</span><span class='va'>prediction</span>
-<span class='op'>}</span>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/applyModel.html b/docs/reference/applyModel.html
deleted file mode 100644
index 2b5feb7af..000000000
--- a/docs/reference/applyModel.html
+++ /dev/null
@@ -1,281 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Apply train model on new data
-Apply a Patient Level Prediction model on Patient Level Prediction Data and get the predicted risk
-in [0,1] for each person in the population. If the user inputs a population with an outcomeCount
-column then the function also returns the evaluation of the prediction (AUC, brier score,
-calibration) — applyModel • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Apply train model on new data
-Apply a Patient Level Prediction model on Patient Level Prediction Data and get the predicted risk
-in [0,1] for each person in the population. If the user inputs a population with an outcomeCount
-column then the function also returns the evaluation of the prediction (AUC, brier score,
-calibration) — applyModel" />
-<meta property="og:description" content="Apply train model on new data
-Apply a Patient Level Prediction model on Patient Level Prediction Data and get the predicted risk
-in [0,1] for each person in the population. If the user inputs a population with an outcomeCount
-column then the function also returns the evaluation of the prediction (AUC, brier score,
-calibration)" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Apply train model on new data
-Apply a Patient Level Prediction model on Patient Level Prediction Data and get the predicted risk
-in [0,1] for each person in the population. If the user inputs a population with an outcomeCount
-column then the function also returns the evaluation of the prediction (AUC, brier score,
-calibration)</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ApplyPlp.R'><code>R/ApplyPlp.R</code></a></small>
-    <div class="hidden name"><code>applyModel.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Apply train model on new data
-Apply a Patient Level Prediction model on Patient Level Prediction Data and get the predicted risk
-in [0,1] for each person in the population. If the user inputs a population with an outcomeCount
-column then the function also returns the evaluation of the prediction (AUC, brier score,
-calibration)</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>applyModel</span><span class='op'>(</span>
-  <span class='va'>population</span>,
-  <span class='va'>plpData</span>,
-  <span class='va'>plpModel</span>,
-  calculatePerformance <span class='op'>=</span> <span class='cn'>T</span>,
-  databaseOutput <span class='op'>=</span> <span class='cn'>NULL</span>,
-  silent <span class='op'>=</span> <span class='cn'>F</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>population</th>
-      <td><p>The population of people who you want to predict the risk for</p></td>
-    </tr>
-    <tr>
-      <th>plpData</th>
-      <td><p>The plpData for the population</p></td>
-    </tr>
-    <tr>
-      <th>plpModel</th>
-      <td><p>The trained PatientLevelPrediction model</p></td>
-    </tr>
-    <tr>
-      <th>calculatePerformance</th>
-      <td><p>Whether to also calculate the performance metrics [default TRUE]</p></td>
-    </tr>
-    <tr>
-      <th>databaseOutput</th>
-      <td><p>Whether to save the details into the prediction database</p></td>
-    </tr>
-    <tr>
-      <th>silent</th>
-      <td><p>Whether to turn off progress reporting</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='co'># load the model and data</span>
-<span class='va'>plpData</span> <span class='op'>&lt;-</span> <span class='fu'><a href='loadPlpData.html'>loadPlpData</a></span><span class='op'>(</span><span class='st'>"C:/plpdata"</span><span class='op'>)</span>
-<span class='va'>plpModel</span> <span class='op'>&lt;-</span> <span class='fu'><a href='loadPlpModel.html'>loadPlpModel</a></span><span class='op'>(</span><span class='st'>"C:/plpmodel"</span><span class='op'>)</span>
-
-<span class='co'># use the same population settings as the model:</span>
-<span class='va'>populationSettings</span> <span class='op'>&lt;-</span> <span class='va'>plpModel</span><span class='op'>$</span><span class='va'>populationSettings</span>
-<span class='va'>populationSettings</span><span class='op'>$</span><span class='va'>plpData</span> <span class='op'>&lt;-</span> <span class='va'>plpData</span>
-<span class='va'>population</span> <span class='op'>&lt;-</span> <span class='fu'><a href='https://rdrr.io/r/base/do.call.html'>do.call</a></span><span class='op'>(</span><span class='va'>createStudyPopulation</span>, <span class='va'>populationSettings</span><span class='op'>)</span>
-
-<span class='co'># get the prediction:</span>
-<span class='va'>prediction</span> <span class='op'>&lt;-</span> <span class='fu'>applyModel</span><span class='op'>(</span><span class='va'>population</span>, <span class='va'>plpData</span>, <span class='va'>plpModel</span><span class='op'>)</span><span class='op'>$</span><span class='va'>prediction</span>
-<span class='op'>}</span>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/averagePrecision.html b/docs/reference/averagePrecision.html
index 35065ca7d..658174496 100644
--- a/docs/reference/averagePrecision.html
+++ b/docs/reference/averagePrecision.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculate the average precision — averagePrecision • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculate the average precision — averagePrecision" />
-<meta property="og:description" content="Calculate the average precision" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculate the average precision — averagePrecision • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculate the average precision — averagePrecision"><meta property="og:description" content="Calculate the average precision"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculate the average precision</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/EvaluationSummary.R'><code>R/EvaluationSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/EvaluationSummary.R" class="external-link"><code>R/EvaluationSummary.R</code></a></small>
     <div class="hidden name"><code>averagePrecision.Rd</code></div>
     </div>
 
@@ -183,49 +131,48 @@ <h1>Calculate the average precision</h1>
     <p>Calculate the average precision</p>
     </div>
 
-    <pre class="usage"><span class='fu'>averagePrecision</span><span class='op'>(</span><span class='va'>prediction</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>prediction</th>
-      <td><p>A prediction object</p></td>
-    </tr>
-    </table>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">averagePrecision</span><span class="op">(</span><span class="va">prediction</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>prediction</dt>
+<dd><p>A prediction object</p></dd>
 
-    <p>The average precision</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
+<p>The average precision</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculates the average precision from a predition object</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/brierScore.html b/docs/reference/brierScore.html
index d3606f9bf..678bf1efe 100644
--- a/docs/reference/brierScore.html
+++ b/docs/reference/brierScore.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>brierScore — brierScore • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="brierScore — brierScore" />
-<meta property="og:description" content="brierScore" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>brierScore — brierScore • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="brierScore — brierScore"><meta property="og:description" content="brierScore"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>brierScore</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/EvaluationSummary.R'><code>R/EvaluationSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/EvaluationSummary.R" class="external-link"><code>R/EvaluationSummary.R</code></a></small>
     <div class="hidden name"><code>brierScore.Rd</code></div>
     </div>
 
@@ -183,49 +131,48 @@ <h1>brierScore</h1>
     <p>brierScore</p>
     </div>
 
-    <pre class="usage"><span class='fu'>brierScore</span><span class='op'>(</span><span class='va'>prediction</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>prediction</th>
-      <td><p>A prediction object</p></td>
-    </tr>
-    </table>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">brierScore</span><span class="op">(</span><span class="va">prediction</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>prediction</dt>
+<dd><p>A prediction object</p></dd>
 
-    <p>A list containing the brier score and the scaled brier score</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
+<p>A list containing the brier score and the scaled brier score</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculates the brierScore from prediction object</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/bySumFf.html b/docs/reference/bySumFf.html
deleted file mode 100644
index 87ca32289..000000000
--- a/docs/reference/bySumFf.html
+++ /dev/null
@@ -1,214 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Compute sum of values binned by a second variable — bySumFf • PatientLevelPrediction</title>
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script>
-
-<!-- sticky kit -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-<meta property="og:title" content="Compute sum of values binned by a second variable — bySumFf" />
-
-<meta property="og:description" content="Compute sum of values binned by a second variable" />
-<meta name="twitter:card" content="summary" />
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-  </head>
-
-  <body>
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.5</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Compute sum of values binned by a second variable</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Predict.R'><code>R/Predict.R</code></a></small>
-    <div class="hidden name"><code>bySumFf.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    
-    <p>Compute sum of values binned by a second variable</p>
-    
-    </div>
-
-    <pre class="usage"><span class='fu'>bySumFf</span>(<span class='no'>values</span>, <span class='no'>bins</span>)</pre>
-    
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>values</th>
-      <td><p>An ff object containing the numeric values to be summed</p></td>
-    </tr>
-    <tr>
-      <th>bins</th>
-      <td><p>An ff object containing the numeric values to bin by</p></td>
-    </tr>
-    </table>
-    
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='no'>values</span> <span class='kw'>&lt;-</span> <span class='kw pkg'>ff</span><span class='kw ns'>::</span><span class='fu'><a href='https://www.rdocumentation.org/packages/ff/topics/as.ff'>as.ff</a></span>(<span class='fu'><a href='https://www.rdocumentation.org/packages/base/topics/c'>c</a></span>(<span class='fl'>1</span>, <span class='fl'>1</span>, <span class='fl'>2</span>, <span class='fl'>2</span>, <span class='fl'>1</span>))
-<span class='no'>bins</span> <span class='kw'>&lt;-</span> <span class='kw pkg'>ff</span><span class='kw ns'>::</span><span class='fu'><a href='https://www.rdocumentation.org/packages/ff/topics/as.ff'>as.ff</a></span>(<span class='fu'><a href='https://www.rdocumentation.org/packages/base/topics/c'>c</a></span>(<span class='fl'>1</span>, <span class='fl'>1</span>, <span class='fl'>1</span>, <span class='fl'>2</span>, <span class='fl'>2</span>))
-<span class='fu'>bySumFf</span>(<span class='no'>values</span>, <span class='no'>bins</span>)</div><div class='output co'>#&gt;   bins sums
-#&gt; 1    1    4
-#&gt; 2    2    3</div><div class='input'>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-    <h2>Contents</h2>
-    <ul class="nav nav-pills nav-stacked">
-      <li><a href="#arguments">Arguments</a></li>
-            
-      <li><a href="#examples">Examples</a></li>
-    </ul>
-
-  </div>
-</div>
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-   </div>
-
-  
-
-  </body>
-</html>
-
diff --git a/docs/reference/calibrationLine.html b/docs/reference/calibrationLine.html
index e0f6267d5..425cccd2b 100644
--- a/docs/reference/calibrationLine.html
+++ b/docs/reference/calibrationLine.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>calibrationLine — calibrationLine • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="calibrationLine — calibrationLine" />
-<meta property="og:description" content="calibrationLine" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>calibrationLine — calibrationLine • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="calibrationLine — calibrationLine"><meta property="og:description" content="calibrationLine"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>calibrationLine</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/EvaluationSummary.R'><code>R/EvaluationSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/EvaluationSummary.R" class="external-link"><code>R/EvaluationSummary.R</code></a></small>
     <div class="hidden name"><code>calibrationLine.Rd</code></div>
     </div>
 
@@ -183,50 +131,46 @@ <h1>calibrationLine</h1>
     <p>calibrationLine</p>
     </div>
 
-    <pre class="usage"><span class='fu'>calibrationLine</span><span class='op'>(</span><span class='va'>prediction</span>, numberOfStrata <span class='op'>=</span> <span class='fl'>10</span><span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">calibrationLine</span><span class="op">(</span><span class="va">prediction</span>, numberOfStrata <span class="op">=</span> <span class="fl">10</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>prediction</dt>
+<dd><p>A prediction object</p></dd>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>prediction</th>
-      <td><p>A prediction object</p></td>
-    </tr>
-    <tr>
-      <th>numberOfStrata</th>
-      <td><p>The number of groups to split the prediction into</p></td>
-    </tr>
-    </table>
 
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<dt>numberOfStrata</dt>
+<dd><p>The number of groups to split the prediction into</p></dd>
 
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculates the calibration from prediction object</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/checkPlpInstallation.html b/docs/reference/checkPlpInstallation.html
deleted file mode 100644
index c9d6dee27..000000000
--- a/docs/reference/checkPlpInstallation.html
+++ /dev/null
@@ -1,229 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Check PatientLevelPrediction and its dependencies are correctly installed — checkPlpInstallation • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Check PatientLevelPrediction and its dependencies are correctly installed — checkPlpInstallation" />
-<meta property="og:description" content="Check PatientLevelPrediction and its dependencies are correctly installed" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Check PatientLevelPrediction and its dependencies are correctly installed</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/HelperFunctions.R'><code>R/HelperFunctions.R</code></a></small>
-    <div class="hidden name"><code>checkPlpInstallation.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Check PatientLevelPrediction and its dependencies are correctly installed</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>checkPlpInstallation</span><span class='op'>(</span>connectionDetails <span class='op'>=</span> <span class='cn'>NULL</span>, python <span class='op'>=</span> <span class='cn'>T</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>connectionDetails</th>
-      <td><p>An R object of type<br /><code>connectionDetails</code> created using the function
-<code>createConnectionDetails</code> in the <code>DatabaseConnector</code> package.</p></td>
-    </tr>
-    <tr>
-      <th>python</th>
-      <td><p>Whether to test the python models</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>This function checks whether PatientLevelPrediction and its dependencies are correctly installed. This will
-check the database connectivity, some models, and large data object
-handling (ff).</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/checkffFolder.html b/docs/reference/checkffFolder.html
deleted file mode 100644
index 72bfe06df..000000000
--- a/docs/reference/checkffFolder.html
+++ /dev/null
@@ -1,198 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Check if the fftempdir is writable — checkffFolder • PatientLevelPrediction</title>
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script>
-
-<!-- sticky kit -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-<meta property="og:title" content="Check if the fftempdir is writable — checkffFolder" />
-
-<meta property="og:description" content="Check if the fftempdir is writable" />
-<meta name="twitter:card" content="summary" />
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-  </head>
-
-  <body>
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.5</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Check if the fftempdir is writable</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/FfHelperFunctions.R'><code>R/FfHelperFunctions.R</code></a></small>
-    <div class="hidden name"><code>checkffFolder.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    
-    <p>Check if the fftempdir is writable</p>
-    
-    </div>
-
-    <pre class="usage"><span class='fu'>checkffFolder</span>()</pre>
-        
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>This function checks whether the fftempdir is writable.
-If not, it will ask the use to specify a writable folder.</p>
-    
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-    <h2>Contents</h2>
-    <ul class="nav nav-pills nav-stacked">
-      
-      <li><a href="#details">Details</a></li>
-          </ul>
-
-  </div>
-</div>
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-   </div>
-
-  
-
-  </body>
-</html>
-
diff --git a/docs/reference/clearffTempDir.html b/docs/reference/clearffTempDir.html
deleted file mode 100644
index a742cfd80..000000000
--- a/docs/reference/clearffTempDir.html
+++ /dev/null
@@ -1,191 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>clearffTempDir — clearffTempDir • PatientLevelPrediction</title>
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script>
-
-<!-- sticky kit -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-<meta property="og:title" content="clearffTempDir — clearffTempDir" />
-
-<meta property="og:description" content="Clears the temporary ff directory to free up disk space." />
-<meta name="twitter:card" content="summary" />
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-  </head>
-
-  <body>
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.5</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>clearffTempDir</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/FfHelperFunctions.R'><code>R/FfHelperFunctions.R</code></a></small>
-    <div class="hidden name"><code>clearffTempDir.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    
-    <p>Clears the temporary ff directory to free up disk space.</p>
-    
-    </div>
-
-    <pre class="usage"><span class='fu'>clearffTempDir</span>()</pre>
-        
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-    <h2>Contents</h2>
-    <ul class="nav nav-pills nav-stacked">
-                </ul>
-
-  </div>
-</div>
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-   </div>
-
-  
-
-  </body>
-</html>
-
diff --git a/docs/reference/combinePlpModelSettings.html b/docs/reference/combinePlpModelSettings.html
deleted file mode 100644
index 471203a99..000000000
--- a/docs/reference/combinePlpModelSettings.html
+++ /dev/null
@@ -1,231 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>combine two objects specifying multiple Plp model settings — combinePlpModelSettings • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="combine two objects specifying multiple Plp model settings — combinePlpModelSettings" />
-<meta property="og:description" content="combine two objects specifying multiple Plp model settings" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>combine two objects specifying multiple Plp model settings</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/RunMultiplePlp.R'><code>R/RunMultiplePlp.R</code></a></small>
-    <div class="hidden name"><code>combinePlpModelSettings.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>combine two objects specifying multiple Plp model settings</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>combinePlpModelSettings</span><span class='op'>(</span><span class='va'>plpModelSetting1</span>, <span class='va'>plpModelSetting2</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpModelSetting1</th>
-      <td><p>A combination of model, covariate and population settings</p></td>
-    </tr>
-    <tr>
-      <th>plpModelSetting2</th>
-      <td><p>A combination of model, covariate and population settings</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A list containing a dataframe settingLookupTable containing all the model, covariate and popualtion combination details, 
-a list models containing all the model settings, a list covariateSettings containing all the covariate settings and a list 
-populationSettings containing all the population settings.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Takes two output of running createPlpModelSettings() and combined them</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/computeAuc.html b/docs/reference/computeAuc.html
index 69cdd4b86..9c79d5ffb 100644
--- a/docs/reference/computeAuc.html
+++ b/docs/reference/computeAuc.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Compute the area under the ROC curve — computeAuc • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Compute the area under the ROC curve — computeAuc" />
-<meta property="og:description" content="Compute the area under the ROC curve" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Compute the area under the ROC curve — computeAuc • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Compute the area under the ROC curve — computeAuc"><meta property="og:description" content="Compute the area under the ROC curve"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Compute the area under the ROC curve</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/EvaluationSummary.R'><code>R/EvaluationSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/EvaluationSummary.R" class="external-link"><code>R/EvaluationSummary.R</code></a></small>
     <div class="hidden name"><code>computeAuc.Rd</code></div>
     </div>
 
@@ -183,52 +131,48 @@ <h1>Compute the area under the ROC curve</h1>
     <p>Compute the area under the ROC curve</p>
     </div>
 
-    <pre class="usage"><span class='fu'>computeAuc</span><span class='op'>(</span><span class='va'>prediction</span>, confidenceInterval <span class='op'>=</span> <span class='cn'>FALSE</span><span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">computeAuc</span><span class="op">(</span><span class="va">prediction</span>, confidenceInterval <span class="op">=</span> <span class="cn">FALSE</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>prediction</dt>
+<dd><p>A prediction object as generated using the
+<code><a href="https://rdrr.io/r/stats/predict.html" class="external-link">predict</a></code> functions.</p></dd>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>prediction</th>
-      <td><p>A prediction object as generated using the
-<code><a href='https://rdrr.io/r/stats/predict.html'>predict</a></code> functions.</p></td>
-    </tr>
-    <tr>
-      <th>confidenceInterval</th>
-      <td><p>Should 95 percebt confidence intervals be computed?</p></td>
-    </tr>
-    </table>
 
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<dt>confidenceInterval</dt>
+<dd><p>Should 95 percebt confidence intervals be computed?</p></dd>
 
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Computes the area under the ROC curve for the predicted probabilities, given the true observed
 outcomes.</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/computeAucFromDataFrames.html b/docs/reference/computeAucFromDataFrames.html
deleted file mode 100644
index 0cbce670f..000000000
--- a/docs/reference/computeAucFromDataFrames.html
+++ /dev/null
@@ -1,251 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Compute the area under the ROC curve — computeAucFromDataFrames • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Compute the area under the ROC curve — computeAucFromDataFrames" />
-<meta property="og:description" content="Compute the area under the ROC curve" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Compute the area under the ROC curve</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/EvaluatePlp.R'><code>R/EvaluatePlp.R</code></a></small>
-    <div class="hidden name"><code>computeAucFromDataFrames.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Compute the area under the ROC curve</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>computeAucFromDataFrames</span><span class='op'>(</span>
-  <span class='va'>prediction</span>,
-  <span class='va'>status</span>,
-  time <span class='op'>=</span> <span class='cn'>NULL</span>,
-  confidenceInterval <span class='op'>=</span> <span class='cn'>FALSE</span>,
-  <span class='va'>timePoint</span>,
-  modelType <span class='op'>=</span> <span class='st'>"logistic"</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>prediction</th>
-      <td><p>A vector with the predicted hazard rate.</p></td>
-    </tr>
-    <tr>
-      <th>status</th>
-      <td><p>A vector with the status of 1 (event) or 0 (no event).</p></td>
-    </tr>
-    <tr>
-      <th>time</th>
-      <td><p>Only for survival models: a vector with the time to event or censor
-(which ever comes first).</p></td>
-    </tr>
-    <tr>
-      <th>confidenceInterval</th>
-      <td><p>Should 95 percebt confidence intervals be computed?</p></td>
-    </tr>
-    <tr>
-      <th>timePoint</th>
-      <td><p>Only for survival models: time point when the AUC should be evaluated</p></td>
-    </tr>
-    <tr>
-      <th>modelType</th>
-      <td><p>Type of model. Currently supported are "logistic" and "survival".</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Computes the area under the ROC curve for the predicted probabilities, given the true observed
-outcomes.</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/computeGridPerformance.html b/docs/reference/computeGridPerformance.html
new file mode 100644
index 000000000..5865fa6ea
--- /dev/null
+++ b/docs/reference/computeGridPerformance.html
@@ -0,0 +1,183 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Computes grid performance with a specified performance function — computeGridPerformance • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Computes grid performance with a specified performance function — computeGridPerformance"><meta property="og:description" content="Computes grid performance with a specified performance function"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Computes grid performance with a specified performance function</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SklearnClassifier.R" class="external-link"><code>R/SklearnClassifier.R</code></a></small>
+    <div class="hidden name"><code>computeGridPerformance.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>Computes grid performance with a specified performance function</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">computeGridPerformance</span><span class="op">(</span><span class="va">prediction</span>, <span class="va">param</span>, performanceFunct <span class="op">=</span> <span class="st">"computeAuc"</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>prediction</dt>
+<dd><p>a dataframe with predictions and outcomeCount per rowId</p></dd>
+
+
+<dt>param</dt>
+<dd><p>a list of hyperparameters</p></dd>
+
+
+<dt>performanceFunct</dt>
+<dd><p>a string specifying which performance function to use
+. Default ``'compute_AUC'``</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A list with overview of the performance</p>
+    </div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/configurePython.html b/docs/reference/configurePython.html
index 4ddcfe043..1abf63794 100644
--- a/docs/reference/configurePython.html
+++ b/docs/reference/configurePython.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Sets up a virtual environment to use for PLP (can be conda or python) — configurePython • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Sets up a virtual environment to use for PLP (can be conda or python) — configurePython" />
-<meta property="og:description" content="Sets up a virtual environment to use for PLP (can be conda or python)" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Sets up a virtual environment to use for PLP (can be conda or python) — configurePython • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Sets up a virtual environment to use for PLP (can be conda or python) — configurePython"><meta property="og:description" content="Sets up a virtual environment to use for PLP (can be conda or python)"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Sets up a virtual environment to use for PLP (can be conda or python)</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/HelperFunctions.R'><code>R/HelperFunctions.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/HelperFunctions.R" class="external-link"><code>R/HelperFunctions.R</code></a></small>
     <div class="hidden name"><code>configurePython.Rd</code></div>
     </div>
 
@@ -183,51 +131,51 @@ <h1>Sets up a virtual environment to use for PLP (can be conda or python)</h1>
     <p>Sets up a virtual environment to use for PLP (can be conda or python)</p>
     </div>
 
-    <pre class="usage"><span class='fu'>configurePython</span><span class='op'>(</span>envname <span class='op'>=</span> <span class='st'>"PLP"</span>, envtype <span class='op'>=</span> <span class='cn'>NULL</span><span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">configurePython</span><span class="op">(</span>envname <span class="op">=</span> <span class="st">"PLP"</span>, envtype <span class="op">=</span> <span class="cn">NULL</span>, condaPythonVersion <span class="op">=</span> <span class="st">"3.11"</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>envname</dt>
+<dd><p>A string for the name of the virtual environment (default is 'PLP')</p></dd>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>envname</th>
-      <td><p>A string for the name of the virtual environment (default is 'PLP')</p></td>
-    </tr>
-    <tr>
-      <th>envtype</th>
-      <td><p>An option for specifying the environment as'conda' or 'python'.  If NULL then the default is 'conda' for windows users and 'python' for non-windows users</p></td>
-    </tr>
-    </table>
 
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<dt>envtype</dt>
+<dd><p>An option for specifying the environment as'conda' or 'python'.  If NULL then the default is 'conda' for windows users and 'python' for non-windows users</p></dd>
 
+
+<dt>condaPythonVersion</dt>
+<dd><p>String, Python version to use when creating a conda environment</p></dd>
+
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>This function creates a virtual environment that can be used by PatientLevelPrediction
 and installs all the required package dependancies.  If using python, pip must be set up.</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/covariateSummary.html b/docs/reference/covariateSummary.html
index 5d58c34fa..0ca6ee4c9 100644
--- a/docs/reference/covariateSummary.html
+++ b/docs/reference/covariateSummary.html
@@ -1,70 +1,15 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>covariateSummary — covariateSummary • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="covariateSummary — covariateSummary" />
-<meta property="og:description" content="Summarises the covariateData to calculate the mean and standard deviation per covaraite
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>covariateSummary — covariateSummary • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="covariateSummary — covariateSummary"><meta property="og:description" content="Summarises the covariateData to calculate the mean and standard deviation per covaraite
 if the labels are input it also stratifies this by class label and if the trainRowIds and testRowIds
 specifying the patients in the train/test sets respectively are input, these values are also stratified
-by train and test set" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+by train and test set"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -75,13 +20,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -97,13 +41,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -116,10 +59,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -128,13 +71,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -142,43 +88,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>covariateSummary</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/CovariateSummary.R'><code>R/CovariateSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/CovariateSummary.R" class="external-link"><code>R/CovariateSummary.R</code></a></small>
     <div class="hidden name"><code>covariateSummary.Rd</code></div>
     </div>
 
@@ -189,81 +137,80 @@ <h1>covariateSummary</h1>
 by train and test set</p>
     </div>
 
-    <pre class="usage"><span class='fu'>covariateSummary</span><span class='op'>(</span>
-  <span class='va'>covariateData</span>,
-  <span class='va'>cohort</span>,
-  labels <span class='op'>=</span> <span class='cn'>NULL</span>,
-  strata <span class='op'>=</span> <span class='cn'>NULL</span>,
-  variableImportance <span class='op'>=</span> <span class='cn'>NULL</span>,
-  featureEngineering <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">covariateSummary</span><span class="op">(</span></span>
+<span>  <span class="va">covariateData</span>,</span>
+<span>  <span class="va">cohort</span>,</span>
+<span>  labels <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  strata <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  variableImportance <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  featureEngineering <span class="op">=</span> <span class="cn">NULL</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>covariateData</dt>
+<dd><p>The covariateData part of the plpData that is 
+extracted using <code>getPlpData</code></p></dd>
+
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>covariateData</th>
-      <td><p>The covariateData part of the plpData that is 
-extracted using <code>getPlpData</code></p></td>
-    </tr>
-    <tr>
-      <th>cohort</th>
-      <td><p>The patient cohort to calculate the summary</p></td>
-    </tr>
-    <tr>
-      <th>labels</th>
-      <td><p>A data.frame with the columns rowId and outcomeCount</p></td>
-    </tr>
-    <tr>
-      <th>strata</th>
-      <td><p>A data.frame containing the columns rowId, strataName</p></td>
-    </tr>
-    <tr>
-      <th>variableImportance</th>
-      <td><p>A data.frame with the columns covariateId and 
-value (the variable importance value)</p></td>
-    </tr>
-    <tr>
-      <th>featureEngineering</th>
-      <td><p>(currently not used )
+<dt>cohort</dt>
+<dd><p>The patient cohort to calculate the summary</p></dd>
+
+
+<dt>labels</dt>
+<dd><p>A data.frame with the columns rowId and outcomeCount</p></dd>
+
+
+<dt>strata</dt>
+<dd><p>A data.frame containing the columns rowId, strataName</p></dd>
+
+
+<dt>variableImportance</dt>
+<dd><p>A data.frame with the columns covariateId and 
+value (the variable importance value)</p></dd>
+
+
+<dt>featureEngineering</dt>
+<dd><p>(currently not used )
 A function or list of functions specifying any feature engineering
-to create covariates before summarising</p></td>
-    </tr>
-    </table>
+to create covariates before summarising</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
-    <p>A data.frame containing: CovariateCount	CovariateMean and CovariateStDev plus these values
+<p>A data.frame containing: CovariateCount	CovariateMean and CovariateStDev plus these values
 for any specified stratification</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>The function calculates various metrics to measure the performance of the model</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/createCohortCovariateSettings.html b/docs/reference/createCohortCovariateSettings.html
index df10200a9..1f4bc8bc9 100644
--- a/docs/reference/createCohortCovariateSettings.html
+++ b/docs/reference/createCohortCovariateSettings.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Extracts covariates based on cohorts — createCohortCovariateSettings • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Extracts covariates based on cohorts — createCohortCovariateSettings" />
-<meta property="og:description" content="Extracts covariates based on cohorts" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Extracts covariates based on cohorts — createCohortCovariateSettings • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Extracts covariates based on cohorts — createCohortCovariateSettings"><meta property="og:description" content="Extracts covariates based on cohorts"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Extracts covariates based on cohorts</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/AdditionalCovariates.R'><code>R/AdditionalCovariates.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/AdditionalCovariates.R" class="external-link"><code>R/AdditionalCovariates.R</code></a></small>
     <div class="hidden name"><code>createCohortCovariateSettings.Rd</code></div>
     </div>
 
@@ -183,104 +131,103 @@ <h1>Extracts covariates based on cohorts</h1>
     <p>Extracts covariates based on cohorts</p>
     </div>
 
-    <pre class="usage"><span class='fu'>createCohortCovariateSettings</span><span class='op'>(</span>
-  <span class='va'>cohortName</span>,
-  <span class='va'>settingId</span>,
-  <span class='va'>cohortDatabaseSchema</span>,
-  <span class='va'>cohortTable</span>,
-  <span class='va'>cohortId</span>,
-  startDay <span class='op'>=</span> <span class='op'>-</span><span class='fl'>30</span>,
-  endDay <span class='op'>=</span> <span class='fl'>0</span>,
-  count <span class='op'>=</span> <span class='cn'>F</span>,
-  ageInteraction <span class='op'>=</span> <span class='cn'>F</span>,
-  lnAgeInteraction <span class='op'>=</span> <span class='cn'>F</span>,
-  analysisId <span class='op'>=</span> <span class='fl'>456</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>cohortName</th>
-      <td><p>Name for the cohort</p></td>
-    </tr>
-    <tr>
-      <th>settingId</th>
-      <td><p>A unique id for the covariate time and</p></td>
-    </tr>
-    <tr>
-      <th>cohortDatabaseSchema</th>
-      <td><p>The schema of the database with the cohort</p></td>
-    </tr>
-    <tr>
-      <th>cohortTable</th>
-      <td><p>the table name that contains the covariate cohort</p></td>
-    </tr>
-    <tr>
-      <th>cohortId</th>
-      <td><p>cohort id for the covariate cohort</p></td>
-    </tr>
-    <tr>
-      <th>startDay</th>
-      <td><p>The number of days prior to index to start observing the cohort</p></td>
-    </tr>
-    <tr>
-      <th>endDay</th>
-      <td><p>The number of days prior to index to stop observing the cohort</p></td>
-    </tr>
-    <tr>
-      <th>count</th>
-      <td><p>If FALSE the covariate value is binary (1 means cohort occurred between index+startDay and index+endDay, 0 means it did not)
-If TRUE then the covariate value is the number of unique cohort_start_dates between index+startDay and index+endDay</p></td>
-    </tr>
-    <tr>
-      <th>ageInteraction</th>
-      <td><p>If TRUE multiple covariate value by the patient's age in years</p></td>
-    </tr>
-    <tr>
-      <th>lnAgeInteraction</th>
-      <td><p>If TRUE multiple covariate value by the log of the patient's age in years</p></td>
-    </tr>
-    <tr>
-      <th>analysisId</th>
-      <td><p>The analysisId for the covariate</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>An object of class covariateSettings specifying how to create the cohort covariate with the covariateId
- cohortId x 100000 + settingId x 1000 + analysisId</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createCohortCovariateSettings</span><span class="op">(</span></span>
+<span>  <span class="va">cohortName</span>,</span>
+<span>  <span class="va">settingId</span>,</span>
+<span>  <span class="va">cohortDatabaseSchema</span>,</span>
+<span>  <span class="va">cohortTable</span>,</span>
+<span>  <span class="va">cohortId</span>,</span>
+<span>  startDay <span class="op">=</span> <span class="op">-</span><span class="fl">30</span>,</span>
+<span>  endDay <span class="op">=</span> <span class="fl">0</span>,</span>
+<span>  count <span class="op">=</span> <span class="cn">F</span>,</span>
+<span>  ageInteraction <span class="op">=</span> <span class="cn">F</span>,</span>
+<span>  lnAgeInteraction <span class="op">=</span> <span class="cn">F</span>,</span>
+<span>  analysisId <span class="op">=</span> <span class="fl">456</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>cohortName</dt>
+<dd><p>Name for the cohort</p></dd>
+
+
+<dt>settingId</dt>
+<dd><p>A unique id for the covariate time and</p></dd>
+
+
+<dt>cohortDatabaseSchema</dt>
+<dd><p>The schema of the database with the cohort</p></dd>
+
+
+<dt>cohortTable</dt>
+<dd><p>the table name that contains the covariate cohort</p></dd>
+
+
+<dt>cohortId</dt>
+<dd><p>cohort id for the covariate cohort</p></dd>
+
+
+<dt>startDay</dt>
+<dd><p>The number of days prior to index to start observing the cohort</p></dd>
 
+
+<dt>endDay</dt>
+<dd><p>The number of days prior to index to stop observing the cohort</p></dd>
+
+
+<dt>count</dt>
+<dd><p>If FALSE the covariate value is binary (1 means cohort occurred between index+startDay and index+endDay, 0 means it did not)
+If TRUE then the covariate value is the number of unique cohort_start_dates between index+startDay and index+endDay</p></dd>
+
+
+<dt>ageInteraction</dt>
+<dd><p>If TRUE multiple covariate value by the patient's age in years</p></dd>
+
+
+<dt>lnAgeInteraction</dt>
+<dd><p>If TRUE multiple covariate value by the log of the patient's age in years</p></dd>
+
+
+<dt>analysisId</dt>
+<dd><p>The analysisId for the covariate</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>An object of class covariateSettings specifying how to create the cohort covariate with the covariateId
+ cohortId x 100000 + settingId x 1000 + analysisId</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>The user specifies a cohort and time period and then a covariate is constructed whether they are in the
 cohort during the time periods relative to target population cohort index</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/createDatabaseDetails.html b/docs/reference/createDatabaseDetails.html
index bac71f798..74b2f2d03 100644
--- a/docs/reference/createDatabaseDetails.html
+++ b/docs/reference/createDatabaseDetails.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create a setting that holds the details about the cdmDatabase connection for data extraction — createDatabaseDetails • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create a setting that holds the details about the cdmDatabase connection for data extraction — createDatabaseDetails" />
-<meta property="og:description" content="Create a setting that holds the details about the cdmDatabase connection for data extraction" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create a setting that holds the details about the cdmDatabase connection for data extraction — createDatabaseDetails • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create a setting that holds the details about the cdmDatabase connection for data extraction — createDatabaseDetails"><meta property="og:description" content="Create a setting that holds the details about the cdmDatabase connection for data extraction"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create a setting that holds the details about the cdmDatabase connection for data extraction</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ExtractData.R'><code>R/ExtractData.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ExtractData.R" class="external-link"><code>R/ExtractData.R</code></a></small>
     <div class="hidden name"><code>createDatabaseDetails.Rd</code></div>
     </div>
 
@@ -183,118 +131,126 @@ <h1>Create a setting that holds the details about the cdmDatabase connection for
     <p>Create a setting that holds the details about the cdmDatabase connection for data extraction</p>
     </div>
 
-    <pre class="usage"><span class='fu'>createDatabaseDetails</span><span class='op'>(</span>
-  <span class='va'>connectionDetails</span>,
-  <span class='va'>cdmDatabaseSchema</span>,
-  <span class='va'>cdmDatabaseName</span>,
-  tempEmulationSchema <span class='op'>=</span> <span class='va'>cdmDatabaseSchema</span>,
-  cohortDatabaseSchema <span class='op'>=</span> <span class='va'>cdmDatabaseSchema</span>,
-  cohortTable <span class='op'>=</span> <span class='st'>"cohort"</span>,
-  outcomeDatabaseSchema <span class='op'>=</span> <span class='va'>cdmDatabaseSchema</span>,
-  outcomeTable <span class='op'>=</span> <span class='st'>"cohort"</span>,
-  cohortId <span class='op'>=</span> <span class='cn'>NULL</span>,
-  outcomeIds <span class='op'>=</span> <span class='cn'>NULL</span>,
-  cdmVersion <span class='op'>=</span> <span class='fl'>5</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>connectionDetails</th>
-      <td><p>An R object of type <code>connectionDetails</code> created using the
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createDatabaseDetails</span><span class="op">(</span></span>
+<span>  <span class="va">connectionDetails</span>,</span>
+<span>  <span class="va">cdmDatabaseSchema</span>,</span>
+<span>  <span class="va">cdmDatabaseName</span>,</span>
+<span>  <span class="va">cdmDatabaseId</span>,</span>
+<span>  tempEmulationSchema <span class="op">=</span> <span class="va">cdmDatabaseSchema</span>,</span>
+<span>  cohortDatabaseSchema <span class="op">=</span> <span class="va">cdmDatabaseSchema</span>,</span>
+<span>  cohortTable <span class="op">=</span> <span class="st">"cohort"</span>,</span>
+<span>  outcomeDatabaseSchema <span class="op">=</span> <span class="va">cdmDatabaseSchema</span>,</span>
+<span>  outcomeTable <span class="op">=</span> <span class="st">"cohort"</span>,</span>
+<span>  targetId <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  outcomeIds <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  cdmVersion <span class="op">=</span> <span class="fl">5</span>,</span>
+<span>  cohortId <span class="op">=</span> <span class="cn">NULL</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>connectionDetails</dt>
+<dd><p>An R object of type <code>connectionDetails</code> created using the
 function <code>createConnectionDetails</code> in the
-<code>DatabaseConnector</code> package.</p></td>
-    </tr>
-    <tr>
-      <th>cdmDatabaseSchema</th>
-      <td><p>The name of the database schema that contains the OMOP CDM
+<code>DatabaseConnector</code> package.</p></dd>
+
+
+<dt>cdmDatabaseSchema</dt>
+<dd><p>The name of the database schema that contains the OMOP CDM
 instance. Requires read permissions to this database. On SQL
 Server, this should specifiy both the database and the schema,
-so for example 'cdm_instance.dbo'.</p></td>
-    </tr>
-    <tr>
-      <th>cdmDatabaseName</th>
-      <td><p>A string with a shareable name of the database (this will be shown to OHDSI researchers if the results get transported)</p></td>
-    </tr>
-    <tr>
-      <th>tempEmulationSchema</th>
-      <td><p>For dmbs like Oracle only: the name of the database schema where you
+so for example 'cdm_instance.dbo'.</p></dd>
+
+
+<dt>cdmDatabaseName</dt>
+<dd><p>A string with the name of the database - this is used in the shiny app and when externally validating models to name the result list and to specify the folder name when saving validation results (defaults to cdmDatabaseSchema if not specified)</p></dd>
+
+
+<dt>cdmDatabaseId</dt>
+<dd><p>A string with a unique identifier for the database and version - this is stored in the plp object for future reference and used by the shiny app (defaults to cdmDatabaseSchema if not specified)</p></dd>
+
+
+<dt>tempEmulationSchema</dt>
+<dd><p>For dmbs like Oracle only: the name of the database schema where you
 want all temporary tables to be managed. Requires
-create/insert permissions to this database.</p></td>
-    </tr>
-    <tr>
-      <th>cohortDatabaseSchema</th>
-      <td><p>The name of the database schema that is the location where the
+create/insert permissions to this database.</p></dd>
+
+
+<dt>cohortDatabaseSchema</dt>
+<dd><p>The name of the database schema that is the location where the
 target cohorts are available.  Requires read
-permissions to this database.</p></td>
-    </tr>
-    <tr>
-      <th>cohortTable</th>
-      <td><p>The tablename that contains the target cohorts.  Expectation is cohortTable
+permissions to this database.</p></dd>
+
+
+<dt>cohortTable</dt>
+<dd><p>The tablename that contains the target cohorts.  Expectation is cohortTable
 has format of COHORT table: COHORT_DEFINITION_ID, SUBJECT_ID,
-COHORT_START_DATE, COHORT_END_DATE.</p></td>
-    </tr>
-    <tr>
-      <th>outcomeDatabaseSchema</th>
-      <td><p>The name of the database schema that is the location where the
+COHORT_START_DATE, COHORT_END_DATE.</p></dd>
+
+
+<dt>outcomeDatabaseSchema</dt>
+<dd><p>The name of the database schema that is the location where the
 data used to define the outcome cohorts is available. Requires read permissions to
-this database.</p></td>
-    </tr>
-    <tr>
-      <th>outcomeTable</th>
-      <td><p>The tablename that contains the outcome cohorts.  Expectation is
+this database.</p></dd>
+
+
+<dt>outcomeTable</dt>
+<dd><p>The tablename that contains the outcome cohorts.  Expectation is
 outcomeTable has format of COHORT table: COHORT_DEFINITION_ID,
-SUBJECT_ID, COHORT_START_DATE, COHORT_END_DATE.</p></td>
-    </tr>
-    <tr>
-      <th>cohortId</th>
-      <td><p>An integer specifying the cohort id for the target cohort</p></td>
-    </tr>
-    <tr>
-      <th>outcomeIds</th>
-      <td><p>A single integer or vector of integers specifying the cohort ids for the outcome cohorts</p></td>
-    </tr>
-    <tr>
-      <th>cdmVersion</th>
-      <td><p>Define the OMOP CDM version used: currently support "4" and
-"5".</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A list with the the database specific settings (this is used by the runMultiplePlp function and the skeleton packages)</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+SUBJECT_ID, COHORT_START_DATE, COHORT_END_DATE.</p></dd>
+
+
+<dt>targetId</dt>
+<dd><p>An integer specifying the cohort id for the target cohort</p></dd>
+
+
+<dt>outcomeIds</dt>
+<dd><p>A single integer or vector of integers specifying the cohort ids for the outcome cohorts</p></dd>
+
 
+<dt>cdmVersion</dt>
+<dd><p>Define the OMOP CDM version used: currently support "4" and "5".</p></dd>
+
+
+<dt>cohortId</dt>
+<dd><p>(depreciated: use targetId) old input for the target cohort id</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A list with the the database specific settings (this is used by the runMultiplePlp function and the skeleton packages)</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>This function simply stores the settings for communicating with the cdmDatabase when extracting 
 the target cohort and outcomes</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/createDatabaseList.html b/docs/reference/createDatabaseList.html
new file mode 100644
index 000000000..47e5f277b
--- /dev/null
+++ b/docs/reference/createDatabaseList.html
@@ -0,0 +1,186 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create a list with the database details and database meta data entries — createDatabaseList • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create a list with the database details and database meta data entries — createDatabaseList"><meta property="og:description" content="This function creates a list with the database details and database meta data entries used in the study"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Create a list with the database details and database meta data entries</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/uploadToDatabase.R" class="external-link"><code>R/uploadToDatabase.R</code></a></small>
+    <div class="hidden name"><code>createDatabaseList.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>This function creates a list with the database details and database meta data entries used in the study</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createDatabaseList</span><span class="op">(</span><span class="va">cdmDatabaseSchemas</span>, <span class="va">cdmDatabaseNames</span>, databaseRefIds <span class="op">=</span> <span class="cn">NULL</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>cdmDatabaseSchemas</dt>
+<dd><p>(string vector) A vector of the cdmDatabaseSchemas used in the study - if the schemas are not unique per database please also specify databaseRefId</p></dd>
+
+
+<dt>cdmDatabaseNames</dt>
+<dd><p>Sharable names for the databases</p></dd>
+
+
+<dt>databaseRefIds</dt>
+<dd><p>(string vector) Unique database identifiers - what you specified as cdmDatabaseId in <code><a href="createDatabaseDetails.html">PatientLevelPrediction::createDatabaseDetails()</a></code> when developing the models</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>Returns a data.frame with the database details</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>This function is used when inserting database details into the PatientLevelPrediction database results schema</p>
+    </div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/createDatabaseSchemaSettings.html b/docs/reference/createDatabaseSchemaSettings.html
new file mode 100644
index 000000000..d5486f118
--- /dev/null
+++ b/docs/reference/createDatabaseSchemaSettings.html
@@ -0,0 +1,215 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create the PatientLevelPrediction database result schema settings — createDatabaseSchemaSettings • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create the PatientLevelPrediction database result schema settings — createDatabaseSchemaSettings"><meta property="og:description" content="This function specifies where the results schema is and lets you pick a different schema for the cohorts and databases"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Create the PatientLevelPrediction database result schema settings</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/uploadToDatabase.R" class="external-link"><code>R/uploadToDatabase.R</code></a></small>
+    <div class="hidden name"><code>createDatabaseSchemaSettings.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>This function specifies where the results schema is and lets you pick a different schema for the cohorts and databases</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createDatabaseSchemaSettings</span><span class="op">(</span></span>
+<span>  resultSchema <span class="op">=</span> <span class="st">"main"</span>,</span>
+<span>  tablePrefix <span class="op">=</span> <span class="st">""</span>,</span>
+<span>  targetDialect <span class="op">=</span> <span class="st">"sqlite"</span>,</span>
+<span>  tempEmulationSchema <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/options.html" class="external-link">getOption</a></span><span class="op">(</span><span class="st">"sqlRenderTempEmulationSchema"</span><span class="op">)</span>,</span>
+<span>  cohortDefinitionSchema <span class="op">=</span> <span class="va">resultSchema</span>,</span>
+<span>  tablePrefixCohortDefinitionTables <span class="op">=</span> <span class="va">tablePrefix</span>,</span>
+<span>  databaseDefinitionSchema <span class="op">=</span> <span class="va">resultSchema</span>,</span>
+<span>  tablePrefixDatabaseDefinitionTables <span class="op">=</span> <span class="va">tablePrefix</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>resultSchema</dt>
+<dd><p>(string) The name of the database schema with the result tables.</p></dd>
+
+
+<dt>tablePrefix</dt>
+<dd><p>(string) A string that appends to the PatientLevelPrediction result tables</p></dd>
+
+
+<dt>targetDialect</dt>
+<dd><p>(string) The database management system being used</p></dd>
+
+
+<dt>tempEmulationSchema</dt>
+<dd><p>(string) The temp schema used when the database management system is oracle</p></dd>
+
+
+<dt>cohortDefinitionSchema</dt>
+<dd><p>(string) The name of the database schema with the cohort definition tables (defaults to resultSchema).</p></dd>
+
+
+<dt>tablePrefixCohortDefinitionTables</dt>
+<dd><p>(string) A string that appends to the cohort definition tables</p></dd>
+
+
+<dt>databaseDefinitionSchema</dt>
+<dd><p>(string) The name of the database schema with the database definition tables (defaults to resultSchema).</p></dd>
+
+
+<dt>tablePrefixDatabaseDefinitionTables</dt>
+<dd><p>(string) A string that appends to the database definition tables</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>Returns a list of class 'plpDatabaseResultSchema' with all the database settings</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>This function can be used to specify the database settings used to upload PatientLevelPrediction results into a database</p>
+    </div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/createDefaultExecuteSettings.html b/docs/reference/createDefaultExecuteSettings.html
index 0b51578f2..e4e409d17 100644
--- a/docs/reference/createDefaultExecuteSettings.html
+++ b/docs/reference/createDefaultExecuteSettings.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Creates default list of settings specifying what parts of runPlp to execute — createDefaultExecuteSettings • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Creates default list of settings specifying what parts of runPlp to execute — createDefaultExecuteSettings" />
-<meta property="og:description" content="Creates default list of settings specifying what parts of runPlp to execute" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Creates default list of settings specifying what parts of runPlp to execute — createDefaultExecuteSettings • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Creates default list of settings specifying what parts of runPlp to execute — createDefaultExecuteSettings"><meta property="og:description" content="Creates default list of settings specifying what parts of runPlp to execute"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Creates default list of settings specifying what parts of runPlp to execute</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/RunPlpHelpers.R'><code>R/RunPlpHelpers.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/RunPlpHelpers.R" class="external-link"><code>R/RunPlpHelpers.R</code></a></small>
     <div class="hidden name"><code>createDefaultExecuteSettings.Rd</code></div>
     </div>
 
@@ -183,41 +131,42 @@ <h1>Creates default list of settings specifying what parts of runPlp to execute<
     <p>Creates default list of settings specifying what parts of runPlp to execute</p>
     </div>
 
-    <pre class="usage"><span class='fu'>createDefaultExecuteSettings</span><span class='op'>(</span><span class='op'>)</span></pre>
-
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createDefaultExecuteSettings</span><span class="op">(</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <p>list with TRUE for split, preprocess, model development and covariate summary</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="value">
+    <h2>Value</h2>
+    
 
+<p>list with TRUE for split, preprocess, model development and covariate summary</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>runs split, preprocess, model development and covariate summary</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/createDefaultSplitSetting.html b/docs/reference/createDefaultSplitSetting.html
index 3e9229228..f92e40aee 100644
--- a/docs/reference/createDefaultSplitSetting.html
+++ b/docs/reference/createDefaultSplitSetting.html
@@ -1,70 +1,15 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create the settings for defining how the plpData are split into test/validation/train sets using 
-default splitting functions (either random stratified by outcome, time or subject splitting). — createDefaultSplitSetting • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create the settings for defining how the plpData are split into test/validation/train sets using 
-default splitting functions (either random stratified by outcome, time or subject splitting). — createDefaultSplitSetting" />
-<meta property="og:description" content="Create the settings for defining how the plpData are split into test/validation/train sets using 
-default splitting functions (either random stratified by outcome, time or subject splitting)." />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create the settings for defining how the plpData are split into test/validation/train sets using 
+default splitting functions (either random stratified by outcome, time or subject splitting) — createDefaultSplitSetting • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create the settings for defining how the plpData are split into test/validation/train sets using 
+default splitting functions (either random stratified by outcome, time or subject splitting) — createDefaultSplitSetting"><meta property="og:description" content="Create the settings for defining how the plpData are split into test/validation/train sets using 
+default splitting functions (either random stratified by outcome, time or subject splitting)"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -75,13 +20,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -97,13 +41,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -116,10 +59,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -128,13 +71,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -142,122 +88,124 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create the settings for defining how the plpData are split into test/validation/train sets using 
-default splitting functions (either random stratified by outcome, time or subject splitting).</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/DataSplitting.R'><code>R/DataSplitting.R</code></a></small>
+default splitting functions (either random stratified by outcome, time or subject splitting)</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/DataSplitting.R" class="external-link"><code>R/DataSplitting.R</code></a></small>
     <div class="hidden name"><code>createDefaultSplitSetting.Rd</code></div>
     </div>
 
     <div class="ref-description">
     <p>Create the settings for defining how the plpData are split into test/validation/train sets using 
-default splitting functions (either random stratified by outcome, time or subject splitting).</p>
+default splitting functions (either random stratified by outcome, time or subject splitting)</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createDefaultSplitSetting</span><span class="op">(</span></span>
+<span>  testFraction <span class="op">=</span> <span class="fl">0.25</span>,</span>
+<span>  trainFraction <span class="op">=</span> <span class="fl">0.75</span>,</span>
+<span>  splitSeed <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/sample.html" class="external-link">sample</a></span><span class="op">(</span><span class="fl">1e+05</span>, <span class="fl">1</span><span class="op">)</span>,</span>
+<span>  nfold <span class="op">=</span> <span class="fl">3</span>,</span>
+<span>  type <span class="op">=</span> <span class="st">"stratified"</span></span>
+<span><span class="op">)</span></span></code></pre></div>
     </div>
 
-    <pre class="usage"><span class='fu'>createDefaultSplitSetting</span><span class='op'>(</span>
-  testFraction <span class='op'>=</span> <span class='fl'>0.25</span>,
-  trainFraction <span class='op'>=</span> <span class='fl'>0.75</span>,
-  splitSeed <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/sample.html'>sample</a></span><span class='op'>(</span><span class='fl'>1e+05</span>, <span class='fl'>1</span><span class='op'>)</span>,
-  nfold <span class='op'>=</span> <span class='fl'>3</span>,
-  type <span class='op'>=</span> <span class='st'>"stratified"</span>
-<span class='op'>)</span></pre>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>testFraction</dt>
+<dd><p>(numeric) A real number between 0 and 1 indicating the test set fraction of the data</p></dd>
+
+
+<dt>trainFraction</dt>
+<dd><p>(numeric) A real number between 0 and 1 indicating the train set fraction of the data.
+If not set train is equal to 1 - test</p></dd>
+
+
+<dt>splitSeed</dt>
+<dd><p>(numeric) A seed to use when splitting the data for reproducibility (if not set a random number will be generated)</p></dd>
+
+
+<dt>nfold</dt>
+<dd><p>(numeric) An integer &gt; 1 specifying the number of folds used in cross validation</p></dd>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>testFraction</th>
-      <td><p>(numeric) A real number between 0 and 1 indicating the test set fraction of the data</p></td>
-    </tr>
-    <tr>
-      <th>trainFraction</th>
-      <td><p>(numeric) A real number between 0 and 1 indicating the train set fraction of the data.
-If not set train is equal to 1 - test</p></td>
-    </tr>
-    <tr>
-      <th>splitSeed</th>
-      <td><p>(numeric) A seed to use when splitting the data for reproducibility (if not set a random number will be generated)</p></td>
-    </tr>
-    <tr>
-      <th>nfold</th>
-      <td><p>(numeric) An integer &gt; 1 specifying the number of folds used in cross validation</p></td>
-    </tr>
-    <tr>
-      <th>type</th>
-      <td><p>(character) Choice of:</p><ul>
-<li><p>'stratified' Each data point is randomly assigned into the test or a train fold set but this is done stratified such that the outcome rate is consistent in each partition</p></li>
-<li><p>'time') Older data are assigned into the training set and newer data are assigned into the test set</p></li>
+
+<dt>type</dt>
+<dd><p>(character) Choice of:</p><ul><li><p>'stratified' Each data point is randomly assigned into the test or a train fold set but this is done stratified such that the outcome rate is consistent in each partition</p></li>
+<li><p>'time' Older data are assigned into the training set and newer data are assigned into the test set</p></li>
 <li><p>'subject' Data are partitioned by subject, if a subject is in the data more than once, all the data points for the subject are assigned either into the test data or into the train data (not both).</p></li>
-</ul></td>
-    </tr>
-    </table>
+</ul></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+<p>An object of class <code>splitSettings</code></p>
 
-    <p>An object of class <code>splitSettings</code></p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
 
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Returns an object of class <code>splitSettings</code> that specifies the splitting function that will be called and the settings</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/createEnsemble.html b/docs/reference/createEnsemble.html
deleted file mode 100644
index 6b66f293b..000000000
--- a/docs/reference/createEnsemble.html
+++ /dev/null
@@ -1,227 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Combine models into an Ensemble — createEnsemble • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Combine models into an Ensemble — createEnsemble" />
-<meta property="og:description" content="Combine models into an Ensemble" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Combine models into an Ensemble</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Ensemble.R'><code>R/Ensemble.R</code></a></small>
-    <div class="hidden name"><code>createEnsemble.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Combine models into an Ensemble</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>createEnsemble</span><span class='op'>(</span><span class='va'>runPlpList</span>, weighted <span class='op'>=</span> <span class='cn'>F</span>, weights <span class='op'>=</span> <span class='cn'>NULL</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>runPlpList</th>
-      <td><p>The runPlp results for the different models to combine</p></td>
-    </tr>
-    <tr>
-      <th>weighted</th>
-      <td><p>If F then mean across models is used, if T must input weights or AUC weighting is used</p></td>
-    </tr>
-    <tr>
-      <th>weights</th>
-      <td><p>A vector of length(runPlpList) with the weights to assign each model</p></td>
-    </tr>
-    </table>
-
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/createExecuteSettings.html b/docs/reference/createExecuteSettings.html
index 02209dc22..dfb91aeed 100644
--- a/docs/reference/createExecuteSettings.html
+++ b/docs/reference/createExecuteSettings.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Creates list of settings specifying what parts of runPlp to execute — createExecuteSettings • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Creates list of settings specifying what parts of runPlp to execute — createExecuteSettings" />
-<meta property="og:description" content="Creates list of settings specifying what parts of runPlp to execute" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Creates list of settings specifying what parts of runPlp to execute — createExecuteSettings • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Creates list of settings specifying what parts of runPlp to execute — createExecuteSettings"><meta property="og:description" content="Creates list of settings specifying what parts of runPlp to execute"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Creates list of settings specifying what parts of runPlp to execute</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/RunPlpHelpers.R'><code>R/RunPlpHelpers.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/RunPlpHelpers.R" class="external-link"><code>R/RunPlpHelpers.R</code></a></small>
     <div class="hidden name"><code>createExecuteSettings.Rd</code></div>
     </div>
 
@@ -183,76 +131,75 @@ <h1>Creates list of settings specifying what parts of runPlp to execute</h1>
     <p>Creates list of settings specifying what parts of runPlp to execute</p>
     </div>
 
-    <pre class="usage"><span class='fu'>createExecuteSettings</span><span class='op'>(</span>
-  runSplitData <span class='op'>=</span> <span class='cn'>F</span>,
-  runSampleData <span class='op'>=</span> <span class='cn'>F</span>,
-  runfeatureEngineering <span class='op'>=</span> <span class='cn'>F</span>,
-  runPreprocessData <span class='op'>=</span> <span class='cn'>F</span>,
-  runModelDevelopment <span class='op'>=</span> <span class='cn'>F</span>,
-  runCovariateSummary <span class='op'>=</span> <span class='cn'>F</span>
-<span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createExecuteSettings</span><span class="op">(</span></span>
+<span>  runSplitData <span class="op">=</span> <span class="cn">F</span>,</span>
+<span>  runSampleData <span class="op">=</span> <span class="cn">F</span>,</span>
+<span>  runfeatureEngineering <span class="op">=</span> <span class="cn">F</span>,</span>
+<span>  runPreprocessData <span class="op">=</span> <span class="cn">F</span>,</span>
+<span>  runModelDevelopment <span class="op">=</span> <span class="cn">F</span>,</span>
+<span>  runCovariateSummary <span class="op">=</span> <span class="cn">F</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>runSplitData</dt>
+<dd><p>TRUE or FALSE whether to split data into train/test</p></dd>
+
+
+<dt>runSampleData</dt>
+<dd><p>TRUE or FALSE whether to over or under sample</p></dd>
+
+
+<dt>runfeatureEngineering</dt>
+<dd><p>TRUE or FALSE whether to do feature engineering</p></dd>
+
+
+<dt>runPreprocessData</dt>
+<dd><p>TRUE or FALSE whether to do preprocessing</p></dd>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>runSplitData</th>
-      <td><p>TRUE or FALSE whether to split data into train/test</p></td>
-    </tr>
-    <tr>
-      <th>runSampleData</th>
-      <td><p>TRUE or FALSE whether to over or under sample</p></td>
-    </tr>
-    <tr>
-      <th>runfeatureEngineering</th>
-      <td><p>TRUE or FALSE whether to do feature engineering</p></td>
-    </tr>
-    <tr>
-      <th>runPreprocessData</th>
-      <td><p>TRUE or FALSE whether to do preprocessing</p></td>
-    </tr>
-    <tr>
-      <th>runModelDevelopment</th>
-      <td><p>TRUE or FALSE whether to develop the model</p></td>
-    </tr>
-    <tr>
-      <th>runCovariateSummary</th>
-      <td><p>TRUE or FALSE whether to create covariate summary</p></td>
-    </tr>
-    </table>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+<dt>runModelDevelopment</dt>
+<dd><p>TRUE or FALSE whether to develop the model</p></dd>
 
-    <p>list with TRUE/FALSE for each part of runPlp</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
 
+<dt>runCovariateSummary</dt>
+<dd><p>TRUE or FALSE whether to create covariate summary</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>list with TRUE/FALSE for each part of runPlp</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>define what parts of runPlp to execute</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/createExistingModelSql.html b/docs/reference/createExistingModelSql.html
deleted file mode 100644
index 786f006d3..000000000
--- a/docs/reference/createExistingModelSql.html
+++ /dev/null
@@ -1,249 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Apply an existing logistic regression prediction model — createExistingModelSql • PatientLevelPrediction</title>
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script>
-
-<!-- sticky kit -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-<meta property="og:title" content="Apply an existing logistic regression prediction model — createExistingModelSql" />
-
-<meta property="og:description" content="Apply an existing logistic regression prediction model" />
-<meta name="twitter:card" content="summary" />
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-  </head>
-
-  <body>
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.5</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Apply an existing logistic regression prediction model</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ImportExport.R'><code>R/ImportExport.R</code></a></small>
-    <div class="hidden name"><code>createExistingModelSql.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    
-    <p>Apply an existing logistic regression prediction model</p>
-    
-    </div>
-
-    <pre class="usage"><span class='fu'>createExistingModelSql</span>(<span class='no'>modelTable</span>, <span class='no'>modelNames</span>, <span class='no'>interceptTable</span>, <span class='no'>covariateTable</span>,
-  <span class='kw'>type</span> <span class='kw'>=</span> <span class='st'>"logistic"</span>, <span class='kw'>analysisId</span> <span class='kw'>=</span> <span class='fl'>112</span>, <span class='no'>covariateSettings</span>, <span class='kw'>asFunctions</span> <span class='kw'>=</span> <span class='no'>F</span>,
-  <span class='kw'>customCovariates</span> <span class='kw'>=</span> <span class='kw'>NULL</span>, <span class='kw'>e</span> <span class='kw'>=</span> <span class='fu'><a href='https://www.rdocumentation.org/packages/base/topics/environment'>environment</a></span>(), <span class='kw'>covariateValues</span> <span class='kw'>=</span> <span class='no'>F</span>)</pre>
-    
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>modelTable</th>
-      <td><p>A dataframe or list of dataframes with columns: modelId, modelCovariateId, coefficientValue all doubles</p></td>
-    </tr>
-    <tr>
-      <th>modelNames</th>
-      <td><p>A name used in the covariate function names (no spaces)</p></td>
-    </tr>
-    <tr>
-      <th>interceptTable</th>
-      <td><p>A dataframe or list of dataframes with the columns: modelId, interceptValue</p></td>
-    </tr>
-    <tr>
-      <th>covariateTable</th>
-      <td><p>A dataframe or list of dataframes with columns: modelCovariateId, covariateId (the mapping of covariate_id to standard covariates)</p></td>
-    </tr>
-    <tr>
-      <th>type</th>
-      <td><p>The type of model: logistic or linear/score</p></td>
-    </tr>
-    <tr>
-      <th>analysisId</th>
-      <td><p>The covariate analysis_id (default 112)</p></td>
-    </tr>
-    <tr>
-      <th>covariateSettings</th>
-      <td><p>The settings for the standard covariates (needs for temporal settings)</p></td>
-    </tr>
-    <tr>
-      <th>asFunctions</th>
-      <td><p>If T then return two functions</p></td>
-    </tr>
-    <tr>
-      <th>customCovariates</th>
-      <td><p>enables custome SQL to be used to create custom covariates</p></td>
-    </tr>
-    <tr>
-      <th>e</th>
-      <td><p>The environment to output the covariate setting functions to</p></td>
-    </tr>
-    <tr>
-      <th>covariateValues</th>
-      <td><p>boolean Whether to also download the covariates that make up the risk score</p></td>
-    </tr>
-    </table>
-    
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>This function is used to create custom covariates corresponding to existing models</p>
-    
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-    <h2>Contents</h2>
-    <ul class="nav nav-pills nav-stacked">
-      <li><a href="#arguments">Arguments</a></li>
-      
-      <li><a href="#details">Details</a></li>
-          </ul>
-
-  </div>
-</div>
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-   </div>
-
-  
-
-  </body>
-</html>
-
diff --git a/docs/reference/createFeatureEngineeringSettings.html b/docs/reference/createFeatureEngineeringSettings.html
index 4a9ef8b11..18defa02b 100644
--- a/docs/reference/createFeatureEngineeringSettings.html
+++ b/docs/reference/createFeatureEngineeringSettings.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create the settings for defining any feature engineering that will be done — createFeatureEngineeringSettings • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create the settings for defining any feature engineering that will be done — createFeatureEngineeringSettings" />
-<meta property="og:description" content="Create the settings for defining any feature engineering that will be done" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create the settings for defining any feature engineering that will be done — createFeatureEngineeringSettings • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create the settings for defining any feature engineering that will be done — createFeatureEngineeringSettings"><meta property="og:description" content="Create the settings for defining any feature engineering that will be done"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create the settings for defining any feature engineering that will be done</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/FeatureEngineering.R'><code>R/FeatureEngineering.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/FeatureEngineering.R" class="external-link"><code>R/FeatureEngineering.R</code></a></small>
     <div class="hidden name"><code>createFeatureEngineeringSettings.Rd</code></div>
     </div>
 
@@ -183,51 +131,51 @@ <h1>Create the settings for defining any feature engineering that will be done</
     <p>Create the settings for defining any feature engineering that will be done</p>
     </div>
 
-    <pre class="usage"><span class='fu'>createFeatureEngineeringSettings</span><span class='op'>(</span>type <span class='op'>=</span> <span class='st'>"none"</span><span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createFeatureEngineeringSettings</span><span class="op">(</span>type <span class="op">=</span> <span class="st">"none"</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>type</th>
-      <td><p>(character) Choice of:</p><ul>
-<li><p>'none' No feature engineering - this is the default</p></li>
-</ul></td>
-    </tr>
-    </table>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>type</dt>
+<dd><p>(character) Choice of:</p><ul><li><p>'none' No feature engineering - this is the default</p></li>
+</ul></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
-    <p>An object of class <code>featureEngineeringSettings</code></p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<p>An object of class <code>featureEngineeringSettings</code></p>
 
+
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Returns an object of class <code>featureEngineeringSettings</code> that specifies the sampling function that will be called and the settings</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/createLearningCurve.html b/docs/reference/createLearningCurve.html
index ef6cc8335..f7e442e2f 100644
--- a/docs/reference/createLearningCurve.html
+++ b/docs/reference/createLearningCurve.html
@@ -1,68 +1,13 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>createLearningCurve — createLearningCurve • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="createLearningCurve — createLearningCurve" />
-<meta property="og:description" content="Creates a learning curve object, which can be plotted using the
- plotLearningCurve() function." />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>createLearningCurve — createLearningCurve • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="createLearningCurve — createLearningCurve"><meta property="og:description" content="Creates a learning curve object, which can be plotted using the
+ plotLearningCurve() function."><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -73,13 +18,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -95,13 +39,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -114,10 +57,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -126,13 +69,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -140,212 +86,208 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>createLearningCurve</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/LearningCurve.R'><code>R/LearningCurve.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/LearningCurve.R" class="external-link"><code>R/LearningCurve.R</code></a></small>
     <div class="hidden name"><code>createLearningCurve.Rd</code></div>
     </div>
 
     <div class="ref-description">
     <p>Creates a learning curve object, which can be plotted using the
- <code><a href='plotLearningCurve.html'>plotLearningCurve()</a></code> function.</p>
+ <code><a href="plotLearningCurve.html">plotLearningCurve()</a></code> function.</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createLearningCurve</span><span class="op">(</span></span>
+<span>  <span class="va">plpData</span>,</span>
+<span>  <span class="va">outcomeId</span>,</span>
+<span>  parallel <span class="op">=</span> <span class="cn">T</span>,</span>
+<span>  cores <span class="op">=</span> <span class="fl">4</span>,</span>
+<span>  <span class="va">modelSettings</span>,</span>
+<span>  saveDirectory <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  analysisId <span class="op">=</span> <span class="st">"learningCurve"</span>,</span>
+<span>  populationSettings <span class="op">=</span> <span class="fu"><a href="createStudyPopulationSettings.html">createStudyPopulationSettings</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  splitSettings <span class="op">=</span> <span class="fu"><a href="createDefaultSplitSetting.html">createDefaultSplitSetting</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  trainFractions <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">0.25</span>, <span class="fl">0.5</span>, <span class="fl">0.75</span><span class="op">)</span>,</span>
+<span>  trainEvents <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  sampleSettings <span class="op">=</span> <span class="fu"><a href="createSampleSettings.html">createSampleSettings</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  featureEngineeringSettings <span class="op">=</span> <span class="fu"><a href="createFeatureEngineeringSettings.html">createFeatureEngineeringSettings</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  preprocessSettings <span class="op">=</span> <span class="fu"><a href="createPreprocessSettings.html">createPreprocessSettings</a></span><span class="op">(</span>minFraction <span class="op">=</span> <span class="fl">0.001</span>, normalize <span class="op">=</span> <span class="cn">T</span><span class="op">)</span>,</span>
+<span>  logSettings <span class="op">=</span> <span class="fu"><a href="createLogSettings.html">createLogSettings</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  executeSettings <span class="op">=</span> <span class="fu"><a href="createExecuteSettings.html">createExecuteSettings</a></span><span class="op">(</span>runSplitData <span class="op">=</span> <span class="cn">T</span>, runSampleData <span class="op">=</span> <span class="cn">F</span>,</span>
+<span>    runfeatureEngineering <span class="op">=</span> <span class="cn">F</span>, runPreprocessData <span class="op">=</span> <span class="cn">T</span>, runModelDevelopment <span class="op">=</span> <span class="cn">T</span>,</span>
+<span>    runCovariateSummary <span class="op">=</span> <span class="cn">F</span><span class="op">)</span></span>
+<span><span class="op">)</span></span></code></pre></div>
     </div>
 
-    <pre class="usage"><span class='fu'>createLearningCurve</span><span class='op'>(</span>
-  <span class='va'>plpData</span>,
-  <span class='va'>outcomeId</span>,
-  parallel <span class='op'>=</span> <span class='cn'>T</span>,
-  cores <span class='op'>=</span> <span class='fl'>4</span>,
-  <span class='va'>modelSettings</span>,
-  saveDirectory <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/getwd.html'>getwd</a></span><span class='op'>(</span><span class='op'>)</span>,
-  analysisId <span class='op'>=</span> <span class='st'>"learningCurve"</span>,
-  populationSettings <span class='op'>=</span> <span class='fu'><a href='createStudyPopulationSettings.html'>createStudyPopulationSettings</a></span><span class='op'>(</span><span class='op'>)</span>,
-  splitSettings <span class='op'>=</span> <span class='fu'><a href='createDefaultSplitSetting.html'>createDefaultSplitSetting</a></span><span class='op'>(</span><span class='op'>)</span>,
-  trainFractions <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>0.25</span>, <span class='fl'>0.5</span>, <span class='fl'>0.75</span><span class='op'>)</span>,
-  trainEvents <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>500</span>, <span class='fl'>1000</span>, <span class='fl'>1500</span><span class='op'>)</span>,
-  sampleSettings <span class='op'>=</span> <span class='fu'><a href='createSampleSettings.html'>createSampleSettings</a></span><span class='op'>(</span><span class='op'>)</span>,
-  featureEngineeringSettings <span class='op'>=</span> <span class='fu'><a href='createFeatureEngineeringSettings.html'>createFeatureEngineeringSettings</a></span><span class='op'>(</span><span class='op'>)</span>,
-  preprocessSettings <span class='op'>=</span> <span class='fu'><a href='createPreprocessSettings.html'>createPreprocessSettings</a></span><span class='op'>(</span>minFraction <span class='op'>=</span> <span class='fl'>0.001</span>, normalize <span class='op'>=</span> <span class='cn'>T</span><span class='op'>)</span>,
-  logSettings <span class='op'>=</span> <span class='fu'><a href='createLogSettings.html'>createLogSettings</a></span><span class='op'>(</span><span class='op'>)</span>,
-  executeSettings <span class='op'>=</span> <span class='fu'><a href='createExecuteSettings.html'>createExecuteSettings</a></span><span class='op'>(</span>runSplitData <span class='op'>=</span> <span class='cn'>T</span>, runSampleData <span class='op'>=</span> <span class='cn'>F</span>,
-    runfeatureEngineering <span class='op'>=</span> <span class='cn'>F</span>, runPreprocessData <span class='op'>=</span> <span class='cn'>T</span>, runModelDevelopment <span class='op'>=</span> <span class='cn'>T</span>,
-    runCovariateSummary <span class='op'>=</span> <span class='cn'>F</span><span class='op'>)</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpData</th>
-      <td><p>An object of type <code>plpData</code> - the patient level prediction
-data extracted from the CDM.</p></td>
-    </tr>
-    <tr>
-      <th>outcomeId</th>
-      <td><p>(integer) The ID of the outcome.</p></td>
-    </tr>
-    <tr>
-      <th>parallel</th>
-      <td><p>Whether to run the code in parallel</p></td>
-    </tr>
-    <tr>
-      <th>cores</th>
-      <td><p>The number of computer cores to use if running in parallel</p></td>
-    </tr>
-    <tr>
-      <th>modelSettings</th>
-      <td><p>An object of class <code>modelSettings</code> created using one of the function:</p><ul>
-<li><p>setLassoLogisticRegression() A lasso logistic regression model</p></li>
-<li><p>setGradientBoostingMachine() A gradient boosting machine</p></li>
-<li><p>setAdaBoost() An ada boost model</p></li>
-<li><p>setRandomForest() A random forest model</p></li>
-<li><p>setDecisionTree() A decision tree model</p></li>
-<li><p>setCovNN()) A convolutional neural network model</p></li>
-<li><p>setCIReNN() A recurrent neural network model</p></li>
-<li><p>setMLP() A neural network model</p></li>
-<li><p>setDeepNN() A deep neural network model</p></li>
-<li><p>setKNN() A KNN model</p></li>
-</ul></td>
-    </tr>
-    <tr>
-      <th>saveDirectory</th>
-      <td><p>The path to the directory where the results will be saved (if NULL uses working directory)</p></td>
-    </tr>
-    <tr>
-      <th>analysisId</th>
-      <td><p>(integer) Identifier for the analysis. It is used to create, e.g., the result folder. Default is a timestamp.</p></td>
-    </tr>
-    <tr>
-      <th>populationSettings</th>
-      <td><p>An object of type <code>populationSettings</code> created using <code>createStudyPopulationSettings</code> that
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpData</dt>
+<dd><p>An object of type <code>plpData</code> - the patient level prediction
+data extracted from the CDM.</p></dd>
+
+
+<dt>outcomeId</dt>
+<dd><p>(integer) The ID of the outcome.</p></dd>
+
+
+<dt>parallel</dt>
+<dd><p>Whether to run the code in parallel</p></dd>
+
+
+<dt>cores</dt>
+<dd><p>The number of computer cores to use if running in parallel</p></dd>
+
+
+<dt>modelSettings</dt>
+<dd><p>An object of class <code>modelSettings</code> created using one of the function:</p><ul><li><p><code><a href="setLassoLogisticRegression.html">setLassoLogisticRegression()</a></code> A lasso logistic regression model</p></li>
+<li><p><code><a href="setGradientBoostingMachine.html">setGradientBoostingMachine()</a></code> A gradient boosting machine</p></li>
+<li><p><code><a href="setAdaBoost.html">setAdaBoost()</a></code> An ada boost model</p></li>
+<li><p><code><a href="setRandomForest.html">setRandomForest()</a></code> A random forest model</p></li>
+<li><p><code><a href="setDecisionTree.html">setDecisionTree()</a></code> A decision tree model</p></li>
+<li><p><code><a href="setKNN.html">setKNN()</a></code> A KNN model</p></li>
+</ul></dd>
+
+
+<dt>saveDirectory</dt>
+<dd><p>The path to the directory where the results will be saved (if NULL uses working directory)</p></dd>
+
+
+<dt>analysisId</dt>
+<dd><p>(integer) Identifier for the analysis. It is used to create, e.g., the result folder. Default is a timestamp.</p></dd>
+
+
+<dt>populationSettings</dt>
+<dd><p>An object of type <code>populationSettings</code> created using <code>createStudyPopulationSettings</code> that
 specifies how the data class labels are defined and addition any exclusions to apply to the 
-plpData cohort</p></td>
-    </tr>
-    <tr>
-      <th>splitSettings</th>
-      <td><p>An object of type <code>splitSettings</code> that specifies how to split the data into train/validation/test.
-The default settings can be created using <code>createDefaultSplitSetting</code>.</p></td>
-    </tr>
-    <tr>
-      <th>trainFractions</th>
-      <td><p>A list of training fractions to create models for.
+plpData cohort</p></dd>
+
+
+<dt>splitSettings</dt>
+<dd><p>An object of type <code>splitSettings</code> that specifies how to split the data into train/validation/test.
+The default settings can be created using <code>createDefaultSplitSetting</code>.</p></dd>
+
+
+<dt>trainFractions</dt>
+<dd><p>A list of training fractions to create models for.
 Note, providing <code>trainEvents</code> will override your input to
-<code>trainFractions</code>.</p></td>
-    </tr>
-    <tr>
-      <th>trainEvents</th>
-      <td><p>Events have shown to be determinant of model performance.
+<code>trainFractions</code>.</p></dd>
+
+
+<dt>trainEvents</dt>
+<dd><p>Events have shown to be determinant of model performance.
 Therefore, it is recommended to provide <code>trainEvents</code> rather than
 <code>trainFractions</code>. Note, providing <code>trainEvents</code> will override
-your input to <code>trainFractions</code>. The format should be as follows:</p><ul>
-<li><p><code><a href='https://rdrr.io/r/base/c.html'>c(500, 1000, 1500) </a></code> - a list of training events</p></li>
-</ul></td>
-    </tr>
-    <tr>
-      <th>sampleSettings</th>
-      <td><p>An object of type <code>sampleSettings</code> that specifies any under/over sampling to be done.
-The default is none.</p></td>
-    </tr>
-    <tr>
-      <th>featureEngineeringSettings</th>
-      <td><p>An object of <code>featureEngineeringSettings</code> specifying any feature engineering to be learned (using the train data)</p></td>
-    </tr>
-    <tr>
-      <th>preprocessSettings</th>
-      <td><p>An object of <code>preprocessSettings</code>. This setting specifies the minimum fraction of 
+your input to <code>trainFractions</code>. The format should be as follows:</p><ul><li><p><code>c(500, 1000, 1500) </code> - a list of training events</p></li>
+</ul></dd>
+
+
+<dt>sampleSettings</dt>
+<dd><p>An object of type <code>sampleSettings</code> that specifies any under/over sampling to be done.
+The default is none.</p></dd>
+
+
+<dt>featureEngineeringSettings</dt>
+<dd><p>An object of <code>featureEngineeringSettings</code> specifying any feature engineering to be learned (using the train data)</p></dd>
+
+
+<dt>preprocessSettings</dt>
+<dd><p>An object of <code>preprocessSettings</code>. This setting specifies the minimum fraction of 
 target population who must have a covariate for it to be included in the model training                            
-and whether to normalise the covariates before training</p></td>
-    </tr>
-    <tr>
-      <th>logSettings</th>
-      <td><p>An object of <code>logSettings</code> created using <code>createLogSettings</code> 
-specifying how the logging is done</p></td>
-    </tr>
-    <tr>
-      <th>executeSettings</th>
-      <td><p>An object of <code>executeSettings</code> specifying which parts of the analysis to run</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A learning curve object containing the various performance measures
- obtained by the model for each training set fraction. It can be plotted
- using <code>plotLearningCurve</code>.</p>
+and whether to normalise the covariates before training</p></dd>
+
 
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='co'># define model</span>
-<span class='va'>modelSettings</span> <span class='op'>=</span> <span class='fu'>PatientLevelPrediction</span><span class='fu'>::</span><span class='fu'><a href='setLassoLogisticRegression.html'>setLassoLogisticRegression</a></span><span class='op'>(</span><span class='op'>)</span>
+<dt>logSettings</dt>
+<dd><p>An object of <code>logSettings</code> created using <code>createLogSettings</code> 
+specifying how the logging is done</p></dd>
 
-<span class='co'># create learning curve</span>
-<span class='va'>learningCurve</span> <span class='op'>&lt;-</span> <span class='fu'>PatientLevelPrediction</span><span class='fu'>::</span><span class='fu'>createLearningCurve</span><span class='op'>(</span><span class='va'>population</span>,
-                                                             <span class='va'>plpData</span>,
-                                                             <span class='va'>modelSettings</span><span class='op'>)</span>
-<span class='co'># plot learning curve</span>
-<span class='fu'>PatientLevelPrediction</span><span class='fu'>::</span><span class='fu'><a href='plotLearningCurve.html'>plotLearningCurve</a></span><span class='op'>(</span><span class='va'>learningCurve</span><span class='op'>)</span>
-<span class='op'>}</span>
 
-</div></pre>
+<dt>executeSettings</dt>
+<dd><p>An object of <code>executeSettings</code> specifying which parts of the analysis to run</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A learning curve object containing the various performance measures
+ obtained by the model for each training set fraction. It can be plotted
+ using <code>plotLearningCurve</code>.</p>
+    </div>
+
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="kw">if</span> <span class="op">(</span><span class="cn">FALSE</span><span class="op">)</span> <span class="op">{</span></span></span>
+<span class="r-in"><span><span class="co"># define model</span></span></span>
+<span class="r-in"><span><span class="va">modelSettings</span> <span class="op">=</span> <span class="fu">PatientLevelPrediction</span><span class="fu">::</span><span class="fu"><a href="setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span></span></span>
+<span class="r-in"><span></span></span>
+<span class="r-in"><span><span class="co"># create learning curve</span></span></span>
+<span class="r-in"><span><span class="va">learningCurve</span> <span class="op">&lt;-</span> <span class="fu">PatientLevelPrediction</span><span class="fu">::</span><span class="fu">createLearningCurve</span><span class="op">(</span><span class="va">population</span>,</span></span>
+<span class="r-in"><span>                                                             <span class="va">plpData</span>,</span></span>
+<span class="r-in"><span>                                                             <span class="va">modelSettings</span><span class="op">)</span></span></span>
+<span class="r-in"><span><span class="co"># plot learning curve</span></span></span>
+<span class="r-in"><span><span class="fu">PatientLevelPrediction</span><span class="fu">::</span><span class="fu"><a href="plotLearningCurve.html">plotLearningCurve</a></span><span class="op">(</span><span class="va">learningCurve</span><span class="op">)</span></span></span>
+<span class="r-in"><span><span class="op">}</span></span></span>
+<span class="r-in"><span></span></span>
+</code></pre></div>
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/createLearningCurvePar.html b/docs/reference/createLearningCurvePar.html
deleted file mode 100644
index bf980a2e2..000000000
--- a/docs/reference/createLearningCurvePar.html
+++ /dev/null
@@ -1,382 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>createLearningCurvePar — createLearningCurvePar • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="createLearningCurvePar — createLearningCurvePar" />
-<meta property="og:description" content="Creates a learning curve in parallel, which can be plotted using
- the plotLearningCurve() function. Currently this functionality is
- only supported by Lasso Logistic Regression." />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>createLearningCurvePar</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/LearningCurve.R'><code>R/LearningCurve.R</code></a></small>
-    <div class="hidden name"><code>createLearningCurvePar.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Creates a learning curve in parallel, which can be plotted using
- the <code><a href='plotLearningCurve.html'>plotLearningCurve()</a></code> function. Currently this functionality is
- only supported by Lasso Logistic Regression.</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>createLearningCurvePar</span><span class='op'>(</span>
-  <span class='va'>population</span>,
-  <span class='va'>plpData</span>,
-  <span class='va'>modelSettings</span>,
-  testSplit <span class='op'>=</span> <span class='st'>"stratified"</span>,
-  testFraction <span class='op'>=</span> <span class='fl'>0.25</span>,
-  trainFractions <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>0.25</span>, <span class='fl'>0.5</span>, <span class='fl'>0.75</span><span class='op'>)</span>,
-  trainEvents <span class='op'>=</span> <span class='cn'>NULL</span>,
-  splitSeed <span class='op'>=</span> <span class='cn'>NULL</span>,
-  nfold <span class='op'>=</span> <span class='fl'>3</span>,
-  indexes <span class='op'>=</span> <span class='cn'>NULL</span>,
-  verbosity <span class='op'>=</span> <span class='st'>"TRACE"</span>,
-  minCovariateFraction <span class='op'>=</span> <span class='fl'>0.001</span>,
-  normalizeData <span class='op'>=</span> <span class='cn'>T</span>,
-  saveDirectory <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/getwd.html'>getwd</a></span><span class='op'>(</span><span class='op'>)</span>,
-  savePlpData <span class='op'>=</span> <span class='cn'>F</span>,
-  savePlpResult <span class='op'>=</span> <span class='cn'>F</span>,
-  savePlpPlots <span class='op'>=</span> <span class='cn'>F</span>,
-  saveEvaluation <span class='op'>=</span> <span class='cn'>F</span>,
-  timeStamp <span class='op'>=</span> <span class='cn'>FALSE</span>,
-  analysisId <span class='op'>=</span> <span class='st'>"lc-"</span>,
-  cores <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>population</th>
-      <td><p>The population created using <code><a href='createStudyPopulation.html'>createStudyPopulation()</a></code>
-that will be used to develop the model.</p></td>
-    </tr>
-    <tr>
-      <th>plpData</th>
-      <td><p>An object of type <code>plpData</code> - the patient level
-prediction data extracted from the CDM.</p></td>
-    </tr>
-    <tr>
-      <th>modelSettings</th>
-      <td><p>An object of class <code>modelSettings</code> created using
-one of the function. Currently only one model is supported:</p><ul>
-<li><p><code>setLassoLogisticRegression</code> - a lasso logistic regression
-    model</p></li>
-</ul></td>
-    </tr>
-    <tr>
-      <th>testSplit</th>
-      <td><p>Specifies the type of evaluation used. Can be either 
-<code>'person'</code> or <code>'time'</code>. The value <code>'time'</code> finds the date
-that splots the population into the testing and training fractions
-provided. Patients with an index after this date are assigned to the test
-set and patients with an index prior to this date are assigned to the
-training set. The value <code>'person'</code> splits the data randomly into
-testing and training sets according to fractions provided. The split is
-stratified by the class label.</p></td>
-    </tr>
-    <tr>
-      <th>testFraction</th>
-      <td><p>The fraction of the data, which will be used as the 
-testing set in the patient split evaluation.</p></td>
-    </tr>
-    <tr>
-      <th>trainFractions</th>
-      <td><p>A list of training fractions to create models for.
-Note, providing <code>trainEvents</code> will override your input to
-<code>trainFractions</code>.</p></td>
-    </tr>
-    <tr>
-      <th>trainEvents</th>
-      <td><p>Events have shown to be determinant of model performance.
-Therefore, it is recommended to provide <code>trainEvents</code> rather than
-<code>trainFractions</code>. Note, providing <code>trainEvents</code> will override
-your input to <code>trainFractions</code>. The format should be as follows:</p><ul>
-<li><p><code><a href='https://rdrr.io/r/base/c.html'>c(500, 1000, 1500)</a></code> - a list of training events</p></li>
-</ul></td>
-    </tr>
-    <tr>
-      <th>splitSeed</th>
-      <td><p>The seed used to split the testing and training set when
-using a 'person' type split</p></td>
-    </tr>
-    <tr>
-      <th>nfold</th>
-      <td><p>The number of folds used in the cross validation (default = 
-<code>3</code>).</p></td>
-    </tr>
-    <tr>
-      <th>indexes</th>
-      <td><p>A dataframe containing a rowId and index column where the 
-index value of -1 means in the test set, and positive integer represents
-the cross validation fold (default is <code>NULL</code>).</p></td>
-    </tr>
-    <tr>
-      <th>verbosity</th>
-      <td><p>Sets the level of the verbosity. If the log level is at or
-higher in priority than the logger threshold, a message will print. The 
-levels are:</p><ul>
-<li><p><code>DEBUG</code> - highest verbosity showing all debug statements</p></li>
-<li><p><code>TRACE</code> - showing information about start and end of steps</p></li>
-<li><p><code>INFO</code> - show informative messages (default)</p></li>
-<li><p><code>WARN</code> - show warning messages</p></li>
-<li><p><code>ERROR</code> - show error messages</p></li>
-<li><p><code>FATAL</code> - be silent except for fatal errors</p></li>
-</ul></td>
-    </tr>
-    <tr>
-      <th>minCovariateFraction</th>
-      <td><p>Minimum covariate prevalence in population to
-avoid removal during preprocssing.</p></td>
-    </tr>
-    <tr>
-      <th>normalizeData</th>
-      <td><p>Whether to normalise the data</p></td>
-    </tr>
-    <tr>
-      <th>saveDirectory</th>
-      <td><p>Location to save log and results</p></td>
-    </tr>
-    <tr>
-      <th>savePlpData</th>
-      <td><p>Whether to save the plpData</p></td>
-    </tr>
-    <tr>
-      <th>savePlpResult</th>
-      <td><p>Whether to save the plpResult</p></td>
-    </tr>
-    <tr>
-      <th>savePlpPlots</th>
-      <td><p>Whether to save the plp plots</p></td>
-    </tr>
-    <tr>
-      <th>saveEvaluation</th>
-      <td><p>Whether to save the plp performance csv files</p></td>
-    </tr>
-    <tr>
-      <th>timeStamp</th>
-      <td><p>Include a timestamp in the log</p></td>
-    </tr>
-    <tr>
-      <th>analysisId</th>
-      <td><p>The analysis unique identifier</p></td>
-    </tr>
-    <tr>
-      <th>cores</th>
-      <td><p>The number of cores to use</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A learning curve object containing the various performance measures
- obtained by the model for each training set fraction. It can be plotted
- using <code>plotLearningCurve</code>.</p>
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='co'># define model</span>
-<span class='va'>modelSettings</span> <span class='op'>=</span> <span class='fu'><a href='setLassoLogisticRegression.html'>setLassoLogisticRegression</a></span><span class='op'>(</span><span class='op'>)</span>
-
-<span class='co'># register parallel backend</span>
-<span class='fu'>registerParallelBackend</span><span class='op'>(</span><span class='op'>)</span>
-
-<span class='co'># create learning curve</span>
-<span class='va'>learningCurve</span> <span class='op'>&lt;-</span> <span class='fu'>createLearningCurvePar</span><span class='op'>(</span><span class='va'>population</span>,
-                                        <span class='va'>plpData</span>,
-                                        <span class='va'>modelSettings</span><span class='op'>)</span>
-<span class='co'># plot learning curve</span>
-<span class='fu'><a href='plotLearningCurve.html'>plotLearningCurve</a></span><span class='op'>(</span><span class='va'>learningCurve</span><span class='op'>)</span>
-<span class='op'>}</span>
-
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/createLogSettings.html b/docs/reference/createLogSettings.html
index a37373f1c..05c7e8c73 100644
--- a/docs/reference/createLogSettings.html
+++ b/docs/reference/createLogSettings.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create the settings for logging the progression of the analysis — createLogSettings • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create the settings for logging the progression of the analysis — createLogSettings" />
-<meta property="og:description" content="Create the settings for logging the progression of the analysis" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create the settings for logging the progression of the analysis — createLogSettings • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create the settings for logging the progression of the analysis — createLogSettings"><meta property="og:description" content="Create the settings for logging the progression of the analysis"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create the settings for logging the progression of the analysis</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Logging.R'><code>R/Logging.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Logging.R" class="external-link"><code>R/Logging.R</code></a></small>
     <div class="hidden name"><code>createLogSettings.Rd</code></div>
     </div>
 
@@ -183,64 +131,64 @@ <h1>Create the settings for logging the progression of the analysis</h1>
     <p>Create the settings for logging the progression of the analysis</p>
     </div>
 
-    <pre class="usage"><span class='fu'>createLogSettings</span><span class='op'>(</span>verbosity <span class='op'>=</span> <span class='st'>"DEBUG"</span>, timeStamp <span class='op'>=</span> <span class='cn'>T</span>, logName <span class='op'>=</span> <span class='st'>"runPlp Log"</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>verbosity</th>
-      <td><p>Sets the level of the verbosity. If the log level is at or higher in priority than the logger threshold, a message will print. The levels are:</p><ul>
-<li><p>DEBUGHighest verbosity showing all debug statements</p></li>
-<li><p>TRACEShowing information about start and end of steps</p></li>
-<li><p>INFOShow informative information (Default)</p></li>
-<li><p>WARNShow warning messages</p></li>
-<li><p>ERRORShow error messages</p></li>
-<li><p>FATALBe silent except for fatal errors</p></li>
-</ul></td>
-    </tr>
-    <tr>
-      <th>timeStamp</th>
-      <td><p>If TRUE a timestamp will be added to each logging statement. Automatically switched on for TRACE level.</p></td>
-    </tr>
-    <tr>
-      <th>logName</th>
-      <td><p>A string reference for the logger</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>An object of class <code>logSettings</code></p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createLogSettings</span><span class="op">(</span>verbosity <span class="op">=</span> <span class="st">"DEBUG"</span>, timeStamp <span class="op">=</span> <span class="cn">T</span>, logName <span class="op">=</span> <span class="st">"runPlp Log"</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>verbosity</dt>
+<dd><p>Sets the level of the verbosity. If the log level is at or higher in priority than the logger threshold, a message will print. The levels are:</p><ul><li><p>DEBUG Highest verbosity showing all debug statements</p></li>
+<li><p>TRACE Showing information about start and end of steps</p></li>
+<li><p>INFO Show informative information (Default)</p></li>
+<li><p>WARN Show warning messages</p></li>
+<li><p>ERROR Show error messages</p></li>
+<li><p>FATAL Be silent except for fatal errors</p></li>
+</ul></dd>
+
+
+<dt>timeStamp</dt>
+<dd><p>If TRUE a timestamp will be added to each logging statement. Automatically switched on for TRACE level.</p></dd>
+
+
+<dt>logName</dt>
+<dd><p>A string reference for the logger</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>An object of class <code>logSettings</code></p>
+
+
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Returns an object of class <code>logSettings</code> that specifies the logger settings</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/createLrSql.html b/docs/reference/createLrSql.html
deleted file mode 100644
index 754439f3c..000000000
--- a/docs/reference/createLrSql.html
+++ /dev/null
@@ -1,255 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Convert logistic regression model to sql code... — createLrSql • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Convert logistic regression model to sql code... — createLrSql" />
-<meta property="og:description" content="Convert logistic regression model to sql code..." />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.3.10</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Convert logistic regression model to sql code...</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ImportExport.R'><code>R/ImportExport.R</code></a></small>
-    <div class="hidden name"><code>createLrSql.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Convert logistic regression model to sql code...</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>createLrSql</span><span class='op'>(</span>
-  <span class='va'>models</span>,
-  <span class='va'>modelNames</span>,
-  covariateConstructionName <span class='op'>=</span> <span class='st'>"prediction"</span>,
-  modelTable <span class='op'>=</span> <span class='st'>"#model_table"</span>,
-  analysisId <span class='op'>=</span> <span class='fl'>111</span>,
-  e <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/environment.html'>environment</a></span><span class='op'>(</span><span class='op'>)</span>,
-  databaseOutput <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>models</th>
-      <td><p>A trianed plp model.</p></td>
-    </tr>
-    <tr>
-      <th>modelNames</th>
-      <td><p>A name used in the covariate function names (no spaces)</p></td>
-    </tr>
-    <tr>
-      <th>covariateConstructionName</th>
-      <td><p>the name used for the create covariate function</p></td>
-    </tr>
-    <tr>
-      <th>modelTable</th>
-      <td><p>The temporary table name storing the model details</p></td>
-    </tr>
-    <tr>
-      <th>analysisId</th>
-      <td><p>The covariate analysis_id</p></td>
-    </tr>
-    <tr>
-      <th>e</th>
-      <td><p>The environment to output the covariate setting functions to</p></td>
-    </tr>
-    <tr>
-      <th>databaseOutput</th>
-      <td><p>If you want to output to go inot a cohort table add the "database.schema.tablename" here</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>This function is used to create custom covariates for a logistic regression model
-(currently only supports, demographics/conditions/drug/procedures/observations and measurement concepts)</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/createModelDesign.html b/docs/reference/createModelDesign.html
index 1277b3cb9..f1fdfafec 100644
--- a/docs/reference/createModelDesign.html
+++ b/docs/reference/createModelDesign.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Specify settings for deceloping a single model — createModelDesign • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Specify settings for deceloping a single model — createModelDesign" />
-<meta property="og:description" content="Specify settings for deceloping a single model" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Specify settings for deceloping a single model — createModelDesign • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Specify settings for deceloping a single model — createModelDesign"><meta property="og:description" content="Specify settings for deceloping a single model"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Specify settings for deceloping a single model</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/RunMultiplePlp.R'><code>R/RunMultiplePlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/RunMultiplePlp.R" class="external-link"><code>R/RunMultiplePlp.R</code></a></small>
     <div class="hidden name"><code>createModelDesign.Rd</code></div>
     </div>
 
@@ -183,96 +131,101 @@ <h1>Specify settings for deceloping a single model</h1>
     <p>Specify settings for deceloping a single model</p>
     </div>
 
-    <pre class="usage"><span class='fu'>createModelDesign</span><span class='op'>(</span>
-  <span class='va'>targetId</span>,
-  <span class='va'>outcomeId</span>,
-  restrictPlpDataSettings <span class='op'>=</span> <span class='fu'><a href='createRestrictPlpDataSettings.html'>createRestrictPlpDataSettings</a></span><span class='op'>(</span><span class='op'>)</span>,
-  populationSettings <span class='op'>=</span> <span class='fu'><a href='createStudyPopulationSettings.html'>createStudyPopulationSettings</a></span><span class='op'>(</span><span class='op'>)</span>,
-  covariateSettings <span class='op'>=</span> <span class='fu'>FeatureExtraction</span><span class='fu'>::</span><span class='fu'><a href='https://rdrr.io/pkg/FeatureExtraction/man/createDefaultCovariateSettings.html'>createDefaultCovariateSettings</a></span><span class='op'>(</span><span class='op'>)</span>,
-  featureEngineeringSettings <span class='op'>=</span> <span class='cn'>NULL</span>,
-  sampleSettings <span class='op'>=</span> <span class='cn'>NULL</span>,
-  preprocessSettings <span class='op'>=</span> <span class='cn'>NULL</span>,
-  modelSettings <span class='op'>=</span> <span class='cn'>NULL</span>,
-  runCovariateSummary <span class='op'>=</span> <span class='cn'>T</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>targetId</th>
-      <td><p>The id of the target cohort that will be used for data extraction (e.g., the ATLAS id)</p></td>
-    </tr>
-    <tr>
-      <th>outcomeId</th>
-      <td><p>The id of the outcome that will be used for data extraction (e.g., the ATLAS id)</p></td>
-    </tr>
-    <tr>
-      <th>restrictPlpDataSettings</th>
-      <td><p>The settings specifying the extra restriction settings when extracting the data created using <code><a href='createRestrictPlpDataSettings.html'>createRestrictPlpDataSettings()</a></code>.</p></td>
-    </tr>
-    <tr>
-      <th>populationSettings</th>
-      <td><p>The population settings specified by <code><a href='createStudyPopulationSettings.html'>createStudyPopulationSettings()</a></code></p></td>
-    </tr>
-    <tr>
-      <th>covariateSettings</th>
-      <td><p>The covariate settings, this can be a list or a single <code>'covariateSetting'</code> object.</p></td>
-    </tr>
-    <tr>
-      <th>featureEngineeringSettings</th>
-      <td><p>Either NULL or an object of class <code>featureEngineeringSettings</code> specifying any feature engineering used during model development</p></td>
-    </tr>
-    <tr>
-      <th>sampleSettings</th>
-      <td><p>Either NULL or an object of class <code>sampleSettings</code> with the over/under sampling settings used for model development</p></td>
-    </tr>
-    <tr>
-      <th>preprocessSettings</th>
-      <td><p>Either NULL or an object of class <code>preprocessSettings</code> created using <code>createPreprocessingSettings()</code></p></td>
-    </tr>
-    <tr>
-      <th>modelSettings</th>
-      <td><p>The model settings such as <code><a href='setLassoLogisticRegression.html'>setLassoLogisticRegression()</a></code></p></td>
-    </tr>
-    <tr>
-      <th>runCovariateSummary</th>
-      <td><p>Whether to run the covariateSummary</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A list with analysis settings used to develop a single prediction model</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createModelDesign</span><span class="op">(</span></span>
+<span>  <span class="va">targetId</span>,</span>
+<span>  <span class="va">outcomeId</span>,</span>
+<span>  restrictPlpDataSettings <span class="op">=</span> <span class="fu"><a href="createRestrictPlpDataSettings.html">createRestrictPlpDataSettings</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  populationSettings <span class="op">=</span> <span class="fu"><a href="createStudyPopulationSettings.html">createStudyPopulationSettings</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  covariateSettings <span class="op">=</span> <span class="fu">FeatureExtraction</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/FeatureExtraction/man/createDefaultCovariateSettings.html" class="external-link">createDefaultCovariateSettings</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  featureEngineeringSettings <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  sampleSettings <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  preprocessSettings <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  modelSettings <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  splitSettings <span class="op">=</span> <span class="fu"><a href="createDefaultSplitSetting.html">createDefaultSplitSetting</a></span><span class="op">(</span>type <span class="op">=</span> <span class="st">"stratified"</span>, testFraction <span class="op">=</span> <span class="fl">0.25</span>,</span>
+<span>    trainFraction <span class="op">=</span> <span class="fl">0.75</span>, splitSeed <span class="op">=</span> <span class="fl">123</span>, nfold <span class="op">=</span> <span class="fl">3</span><span class="op">)</span>,</span>
+<span>  runCovariateSummary <span class="op">=</span> <span class="cn">T</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>targetId</dt>
+<dd><p>The id of the target cohort that will be used for data extraction (e.g., the ATLAS id)</p></dd>
+
+
+<dt>outcomeId</dt>
+<dd><p>The id of the outcome that will be used for data extraction (e.g., the ATLAS id)</p></dd>
+
+
+<dt>restrictPlpDataSettings</dt>
+<dd><p>The settings specifying the extra restriction settings when extracting the data created using <code><a href="createRestrictPlpDataSettings.html">createRestrictPlpDataSettings()</a></code>.</p></dd>
+
+
+<dt>populationSettings</dt>
+<dd><p>The population settings specified by <code><a href="createStudyPopulationSettings.html">createStudyPopulationSettings()</a></code></p></dd>
+
+
+<dt>covariateSettings</dt>
+<dd><p>The covariate settings, this can be a list or a single <code>'covariateSetting'</code> object.</p></dd>
+
 
+<dt>featureEngineeringSettings</dt>
+<dd><p>Either NULL or an object of class <code>featureEngineeringSettings</code> specifying any feature engineering used during model development</p></dd>
+
+
+<dt>sampleSettings</dt>
+<dd><p>Either NULL or an object of class <code>sampleSettings</code> with the over/under sampling settings used for model development</p></dd>
+
+
+<dt>preprocessSettings</dt>
+<dd><p>Either NULL or an object of class <code>preprocessSettings</code> created using <code>createPreprocessingSettings()</code></p></dd>
+
+
+<dt>modelSettings</dt>
+<dd><p>The model settings such as <code><a href="setLassoLogisticRegression.html">setLassoLogisticRegression()</a></code></p></dd>
+
+
+<dt>splitSettings</dt>
+<dd><p>The train/validation/test splitting used by all analyses created using <code><a href="createDefaultSplitSetting.html">createDefaultSplitSetting()</a></code></p></dd>
+
+
+<dt>runCovariateSummary</dt>
+<dd><p>Whether to run the covariateSummary</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A list with analysis settings used to develop a single prediction model</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>This specifies a single analysis for developing as single model</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/createPlpJournalDocument.html b/docs/reference/createPlpJournalDocument.html
deleted file mode 100644
index 7639d076c..000000000
--- a/docs/reference/createPlpJournalDocument.html
+++ /dev/null
@@ -1,287 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>createPlpJournalDocument — createPlpJournalDocument • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="createPlpJournalDocument — createPlpJournalDocument" />
-<meta property="og:description" content="Creates a template for a prediction journal paper with the characteristics/results filled in" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>createPlpJournalDocument</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/CreatePlpDocument.R'><code>R/CreatePlpDocument.R</code></a></small>
-    <div class="hidden name"><code>createPlpJournalDocument.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Creates a template for a prediction journal paper with the characteristics/results filled in</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>createPlpJournalDocument</span><span class='op'>(</span>
-  plpResult <span class='op'>=</span> <span class='cn'>NULL</span>,
-  plpValidation <span class='op'>=</span> <span class='cn'>NULL</span>,
-  plpData <span class='op'>=</span> <span class='cn'>NULL</span>,
-  targetName <span class='op'>=</span> <span class='st'>"&lt;target population&gt;"</span>,
-  outcomeName <span class='op'>=</span> <span class='st'>"&lt;outcome&gt;"</span>,
-  table1 <span class='op'>=</span> <span class='cn'>F</span>,
-  connectionDetails <span class='op'>=</span> <span class='cn'>NULL</span>,
-  includeTrain <span class='op'>=</span> <span class='cn'>FALSE</span>,
-  includeTest <span class='op'>=</span> <span class='cn'>TRUE</span>,
-  includePredictionPicture <span class='op'>=</span> <span class='cn'>TRUE</span>,
-  includeAttritionPlot <span class='op'>=</span> <span class='cn'>TRUE</span>,
-  outputLocation <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/file.path.html'>file.path</a></span><span class='op'>(</span><span class='fu'><a href='https://rdrr.io/r/base/getwd.html'>getwd</a></span><span class='op'>(</span><span class='op'>)</span>, <span class='st'>"plp_journal_document.docx"</span><span class='op'>)</span>,
-  save <span class='op'>=</span> <span class='cn'>T</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpResult</th>
-      <td><p>An object of type <code>plpResult</code> returned by running runPlp()</p></td>
-    </tr>
-    <tr>
-      <th>plpValidation</th>
-      <td><p>An object of type <code>validatePlp</code> returned by running externalValidatePlp()</p></td>
-    </tr>
-    <tr>
-      <th>plpData</th>
-      <td><p>The plpData</p></td>
-    </tr>
-    <tr>
-      <th>targetName</th>
-      <td><p>A string with the target description name</p></td>
-    </tr>
-    <tr>
-      <th>outcomeName</th>
-      <td><p>A string with the outcome description name</p></td>
-    </tr>
-    <tr>
-      <th>table1</th>
-      <td><p>Whether to include table1 (characteristics)</p></td>
-    </tr>
-    <tr>
-      <th>connectionDetails</th>
-      <td><p>The connection required to calcualte the characteristics</p></td>
-    </tr>
-    <tr>
-      <th>includeTrain</th>
-      <td><p>Whether to include the train set performance</p></td>
-    </tr>
-    <tr>
-      <th>includeTest</th>
-      <td><p>Whether to include the test set performance</p></td>
-    </tr>
-    <tr>
-      <th>includePredictionPicture</th>
-      <td><p>Whether to include a picture detailing the prediction problem</p></td>
-    </tr>
-    <tr>
-      <th>includeAttritionPlot</th>
-      <td><p>Whether to include the attriction plot</p></td>
-    </tr>
-    <tr>
-      <th>outputLocation</th>
-      <td><p>The location to write the document to</p></td>
-    </tr>
-    <tr>
-      <th>save</th>
-      <td><p>If false this fucntion returns the document and does not save to outputLocation</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A work document containing the selected outputs within the user's directory at location specified in outputLocation</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>The function creates a word document containing the analysis details, data summary and prediction model results.</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/createPlpModelSettings.html b/docs/reference/createPlpModelSettings.html
deleted file mode 100644
index 82d054b9e..000000000
--- a/docs/reference/createPlpModelSettings.html
+++ /dev/null
@@ -1,236 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>create a an object specifying the multiple Plp model settings — createPlpModelSettings • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="create a an object specifying the multiple Plp model settings — createPlpModelSettings" />
-<meta property="og:description" content="create a an object specifying the multiple Plp model settings" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>create a an object specifying the multiple Plp model settings</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/RunMultiplePlp.R'><code>R/RunMultiplePlp.R</code></a></small>
-    <div class="hidden name"><code>createPlpModelSettings.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>create a an object specifying the multiple Plp model settings</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>createPlpModelSettings</span><span class='op'>(</span><span class='va'>modelList</span>, <span class='va'>covariateSettingList</span>, <span class='va'>populationSettingList</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>modelList</th>
-      <td><p>A list of model settings</p></td>
-    </tr>
-    <tr>
-      <th>covariateSettingList</th>
-      <td><p>A list of covariate settings</p></td>
-    </tr>
-    <tr>
-      <th>populationSettingList</th>
-      <td><p>A list of population settings</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A list containing a dataframe settingLookupTable containing all the model, covariate and popualtion combination details, 
-a list models containing all the model settings, a list covariateSettings containing all the covariate settings and a list 
-populationSettings containing all the population settings.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Takes a list of models, covariates, population and returns the cartesian product combining all
-settings.</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/createPlpReport.html b/docs/reference/createPlpReport.html
deleted file mode 100644
index 6a956a89e..000000000
--- a/docs/reference/createPlpReport.html
+++ /dev/null
@@ -1,267 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>createPlpReport — createPlpReport • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="createPlpReport — createPlpReport" />
-<meta property="og:description" content="Creates a word document report of the prediction" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>createPlpReport</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/CreatePlpDocument.R'><code>R/CreatePlpDocument.R</code></a></small>
-    <div class="hidden name"><code>createPlpReport.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Creates a word document report of the prediction</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>createPlpReport</span><span class='op'>(</span>
-  plpResult <span class='op'>=</span> <span class='cn'>NULL</span>,
-  plpValidation <span class='op'>=</span> <span class='cn'>NULL</span>,
-  plpData <span class='op'>=</span> <span class='cn'>NULL</span>,
-  targetName <span class='op'>=</span> <span class='st'>"&lt;target population&gt;"</span>,
-  outcomeName <span class='op'>=</span> <span class='st'>"&lt;outcome&gt;"</span>,
-  targetDefinition <span class='op'>=</span> <span class='cn'>NULL</span>,
-  outcomeDefinition <span class='op'>=</span> <span class='cn'>NULL</span>,
-  outputLocation <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/file.path.html'>file.path</a></span><span class='op'>(</span><span class='fu'><a href='https://rdrr.io/r/base/getwd.html'>getwd</a></span><span class='op'>(</span><span class='op'>)</span>, <span class='st'>"plp_report.docx"</span><span class='op'>)</span>,
-  save <span class='op'>=</span> <span class='cn'>T</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpResult</th>
-      <td><p>An object of type <code>plpResult</code> returned by running runPlp()</p></td>
-    </tr>
-    <tr>
-      <th>plpValidation</th>
-      <td><p>An object of type <code>validatePlp</code> returned by running externalValidatePlp()</p></td>
-    </tr>
-    <tr>
-      <th>plpData</th>
-      <td><p>The plpData</p></td>
-    </tr>
-    <tr>
-      <th>targetName</th>
-      <td><p>A string with the target description name</p></td>
-    </tr>
-    <tr>
-      <th>outcomeName</th>
-      <td><p>A string with the outcome description name</p></td>
-    </tr>
-    <tr>
-      <th>targetDefinition</th>
-      <td><p>The cohort details</p></td>
-    </tr>
-    <tr>
-      <th>outcomeDefinition</th>
-      <td><p>The cohort details</p></td>
-    </tr>
-    <tr>
-      <th>outputLocation</th>
-      <td><p>The location to write the document to</p></td>
-    </tr>
-    <tr>
-      <th>save</th>
-      <td><p>If false the output of the function of the function is the document rather than creating the document in outputLocation</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A work document containing the selected outputs within the user's directory at location specified in outputLocation</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>The function creates a word document containing the analysis details, data summary and prediction model results.</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/createPlpResultTables.html b/docs/reference/createPlpResultTables.html
index 742a4f777..41c22b897 100644
--- a/docs/reference/createPlpResultTables.html
+++ b/docs/reference/createPlpResultTables.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create the results tables to store PatientLevelPrediction models and results into a database — createPlpResultTables • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create the results tables to store PatientLevelPrediction models and results into a database — createPlpResultTables" />
-<meta property="og:description" content="This function executes a large set of SQL statements to create tables that can store models and results" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create the results tables to store PatientLevelPrediction models and results into a database — createPlpResultTables • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create the results tables to store PatientLevelPrediction models and results into a database — createPlpResultTables"><meta property="og:description" content="This function executes a large set of SQL statements to create tables that can store models and results"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create the results tables to store PatientLevelPrediction models and results into a database</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/uploadPlpDbResults.R'><code>R/uploadPlpDbResults.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/uploadToDatabase.R" class="external-link"><code>R/uploadToDatabase.R</code></a></small>
     <div class="hidden name"><code>createPlpResultTables.Rd</code></div>
     </div>
 
@@ -183,88 +131,85 @@ <h1>Create the results tables to store PatientLevelPrediction models and results
     <p>This function executes a large set of SQL statements to create tables that can store models and results</p>
     </div>
 
-    <pre class="usage"><span class='fu'>createPlpResultTables</span><span class='op'>(</span>
-  <span class='va'>conn</span>,
-  <span class='va'>resultSchema</span>,
-  targetDialect <span class='op'>=</span> <span class='st'>"postgresql"</span>,
-  deleteExistingTables <span class='op'>=</span> <span class='cn'>T</span>,
-  createTables <span class='op'>=</span> <span class='cn'>T</span>,
-  stringAppendToTables <span class='op'>=</span> <span class='st'>""</span>,
-  tempEmulationSchema <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/options.html'>getOption</a></span><span class='op'>(</span><span class='st'>"sqlRenderTempEmulationSchema"</span><span class='op'>)</span>,
-  testFile <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>conn</th>
-      <td><p>A connection to a database created by using the
-function <code>connect</code> in the
-<code>DatabaseConnector</code> package.</p></td>
-    </tr>
-    <tr>
-      <th>resultSchema</th>
-      <td><p>The name of the database schema that the result tables will be created.</p></td>
-    </tr>
-    <tr>
-      <th>targetDialect</th>
-      <td><p>The database management system being used</p></td>
-    </tr>
-    <tr>
-      <th>deleteExistingTables</th>
-      <td><p>If true any existing tables matching the PatientLevelPrediction result tables names will be deleted</p></td>
-    </tr>
-    <tr>
-      <th>createTables</th>
-      <td><p>If true the PatientLevelPrediction result tables will be created</p></td>
-    </tr>
-    <tr>
-      <th>stringAppendToTables</th>
-      <td><p>A string that appends to the PatientLevelPrediction result tables</p></td>
-    </tr>
-    <tr>
-      <th>tempEmulationSchema</th>
-      <td><p>The temp schema used when the database management system is oracle</p></td>
-    </tr>
-    <tr>
-      <th>testFile</th>
-      <td><p>(used for testing) The location of an sql file with the table creation code</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>Returns NULL but creates the required tables into the specified database schema.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createPlpResultTables</span><span class="op">(</span></span>
+<span>  <span class="va">connectionDetails</span>,</span>
+<span>  targetDialect <span class="op">=</span> <span class="st">"postgresql"</span>,</span>
+<span>  <span class="va">resultSchema</span>,</span>
+<span>  deleteTables <span class="op">=</span> <span class="cn">T</span>,</span>
+<span>  createTables <span class="op">=</span> <span class="cn">T</span>,</span>
+<span>  tablePrefix <span class="op">=</span> <span class="st">""</span>,</span>
+<span>  tempEmulationSchema <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/options.html" class="external-link">getOption</a></span><span class="op">(</span><span class="st">"sqlRenderTempEmulationSchema"</span><span class="op">)</span>,</span>
+<span>  testFile <span class="op">=</span> <span class="cn">NULL</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>connectionDetails</dt>
+<dd><p>The database connection details</p></dd>
+
+
+<dt>targetDialect</dt>
+<dd><p>The database management system being used</p></dd>
+
+
+<dt>resultSchema</dt>
+<dd><p>The name of the database schema that the result tables will be created.</p></dd>
+
 
+<dt>deleteTables</dt>
+<dd><p>If true any existing tables matching the PatientLevelPrediction result tables names will be deleted</p></dd>
+
+
+<dt>createTables</dt>
+<dd><p>If true the PatientLevelPrediction result tables will be created</p></dd>
+
+
+<dt>tablePrefix</dt>
+<dd><p>A string that appends to the PatientLevelPrediction result tables</p></dd>
+
+
+<dt>tempEmulationSchema</dt>
+<dd><p>The temp schema used when the database management system is oracle</p></dd>
+
+
+<dt>testFile</dt>
+<dd><p>(used for testing) The location of an sql file with the table creation code</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>Returns NULL but creates the required tables into the specified database schema(s).</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>This function can be used to create (or delete) PatientLevelPrediction result tables</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/createPreprocessSettings.html b/docs/reference/createPreprocessSettings.html
index 0beec10c9..74d078c2b 100644
--- a/docs/reference/createPreprocessSettings.html
+++ b/docs/reference/createPreprocessSettings.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create the settings for preprocessing the trainData using  . — createPreprocessSettings • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create the settings for preprocessing the trainData using  . — createPreprocessSettings" />
-<meta property="og:description" content="Create the settings for preprocessing the trainData using  ." />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create the settings for preprocessing the trainData. — createPreprocessSettings • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create the settings for preprocessing the trainData. — createPreprocessSettings"><meta property="og:description" content="Create the settings for preprocessing the trainData."><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,105 +85,108 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
-    <h1>Create the settings for preprocessing the trainData using <code> </code>.</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/PreprocessingData.R'><code>R/PreprocessingData.R</code></a></small>
+    <h1>Create the settings for preprocessing the trainData.</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/PreprocessingData.R" class="external-link"><code>R/PreprocessingData.R</code></a></small>
     <div class="hidden name"><code>createPreprocessSettings.Rd</code></div>
     </div>
 
     <div class="ref-description">
-    <p>Create the settings for preprocessing the trainData using <code> </code>.</p>
+    <p>Create the settings for preprocessing the trainData.</p>
     </div>
 
-    <pre class="usage"><span class='fu'>createPreprocessSettings</span><span class='op'>(</span>
-  minFraction <span class='op'>=</span> <span class='fl'>0.001</span>,
-  normalize <span class='op'>=</span> <span class='cn'>TRUE</span>,
-  removeRedundancy <span class='op'>=</span> <span class='cn'>TRUE</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>minFraction</th>
-      <td><p>The minimum fraction of target population who must have a covariate for it to be included in the model training</p></td>
-    </tr>
-    <tr>
-      <th>normalize</th>
-      <td><p>Whether to normalise the covariates before training (Default: TRUE)</p></td>
-    </tr>
-    <tr>
-      <th>removeRedundancy</th>
-      <td><p>Whether to remove redundant features (Default: TRUE)</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>An object of class <code>preprocessingSettings</code></p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createPreprocessSettings</span><span class="op">(</span></span>
+<span>  minFraction <span class="op">=</span> <span class="fl">0.001</span>,</span>
+<span>  normalize <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>  removeRedundancy <span class="op">=</span> <span class="cn">TRUE</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>minFraction</dt>
+<dd><p>The minimum fraction of target population who must have a covariate for it to be included in the model training</p></dd>
+
+
+<dt>normalize</dt>
+<dd><p>Whether to normalise the covariates before training (Default: TRUE)</p></dd>
+
+
+<dt>removeRedundancy</dt>
+<dd><p>Whether to remove redundant features (Default: TRUE)</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>An object of class <code>preprocessingSettings</code></p>
+
+
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Returns an object of class <code>preprocessingSettings</code> that specifies how to preprocess the training data</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/createRandomForestFeatureSelection.html b/docs/reference/createRandomForestFeatureSelection.html
index 7cee442c3..3b2803c55 100644
--- a/docs/reference/createRandomForestFeatureSelection.html
+++ b/docs/reference/createRandomForestFeatureSelection.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create the settings for random foreat based feature selection — createRandomForestFeatureSelection • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create the settings for random foreat based feature selection — createRandomForestFeatureSelection" />
-<meta property="og:description" content="Create the settings for random foreat based feature selection" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create the settings for random foreat based feature selection — createRandomForestFeatureSelection • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create the settings for random foreat based feature selection — createRandomForestFeatureSelection"><meta property="og:description" content="Create the settings for random foreat based feature selection"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create the settings for random foreat based feature selection</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/FeatureEngineering.R'><code>R/FeatureEngineering.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/FeatureEngineering.R" class="external-link"><code>R/FeatureEngineering.R</code></a></small>
     <div class="hidden name"><code>createRandomForestFeatureSelection.Rd</code></div>
     </div>
 
@@ -183,53 +131,54 @@ <h1>Create the settings for random foreat based feature selection</h1>
     <p>Create the settings for random foreat based feature selection</p>
     </div>
 
-    <pre class="usage"><span class='fu'>createRandomForestFeatureSelection</span><span class='op'>(</span>ntrees <span class='op'>=</span> <span class='fl'>2000</span>, maxDepth <span class='op'>=</span> <span class='fl'>17</span><span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createRandomForestFeatureSelection</span><span class="op">(</span>ntrees <span class="op">=</span> <span class="fl">2000</span>, maxDepth <span class="op">=</span> <span class="fl">17</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>ntrees</dt>
+<dd><p>number of tree in forest</p></dd>
+
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>ntrees</th>
-      <td><p>number of tree in forest</p></td>
-    </tr>
-    <tr>
-      <th>maxDepth</th>
-      <td><p>MAx depth of each tree</p></td>
-    </tr>
-    </table>
+<dt>maxDepth</dt>
+<dd><p>MAx depth of each tree</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
-    <p>An object of class <code>featureEngineeringSettings</code></p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<p>An object of class <code>featureEngineeringSettings</code></p>
 
+
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Returns an object of class <code>featureEngineeringSettings</code> that specifies the sampling function that will be called and the settings</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/createRestrictPlpDataSettings.html b/docs/reference/createRestrictPlpDataSettings.html
index 2e1ab7462..923489204 100644
--- a/docs/reference/createRestrictPlpDataSettings.html
+++ b/docs/reference/createRestrictPlpDataSettings.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>createRestrictPlpDataSettings define extra restriction settings when calling getPlpData — createRestrictPlpDataSettings • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="createRestrictPlpDataSettings define extra restriction settings when calling getPlpData — createRestrictPlpDataSettings" />
-<meta property="og:description" content="This function creates the settings used to restrict the target cohort when calling getPlpData" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>createRestrictPlpDataSettings define extra restriction settings when calling getPlpData — createRestrictPlpDataSettings • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="createRestrictPlpDataSettings define extra restriction settings when calling getPlpData — createRestrictPlpDataSettings"><meta property="og:description" content="This function creates the settings used to restrict the target cohort when calling getPlpData"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>createRestrictPlpDataSettings define extra restriction settings when calling getPlpData</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ExtractData.R'><code>R/ExtractData.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ExtractData.R" class="external-link"><code>R/ExtractData.R</code></a></small>
     <div class="hidden name"><code>createRestrictPlpDataSettings.Rd</code></div>
     </div>
 
@@ -183,80 +131,79 @@ <h1>createRestrictPlpDataSettings define extra restriction settings when calling
     <p>This function creates the settings used to restrict the target cohort when calling getPlpData</p>
     </div>
 
-    <pre class="usage"><span class='fu'>createRestrictPlpDataSettings</span><span class='op'>(</span>
-  studyStartDate <span class='op'>=</span> <span class='st'>""</span>,
-  studyEndDate <span class='op'>=</span> <span class='st'>""</span>,
-  firstExposureOnly <span class='op'>=</span> <span class='cn'>F</span>,
-  washoutPeriod <span class='op'>=</span> <span class='fl'>0</span>,
-  sampleSize <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createRestrictPlpDataSettings</span><span class="op">(</span></span>
+<span>  studyStartDate <span class="op">=</span> <span class="st">""</span>,</span>
+<span>  studyEndDate <span class="op">=</span> <span class="st">""</span>,</span>
+<span>  firstExposureOnly <span class="op">=</span> <span class="cn">F</span>,</span>
+<span>  washoutPeriod <span class="op">=</span> <span class="fl">0</span>,</span>
+<span>  sampleSize <span class="op">=</span> <span class="cn">NULL</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>studyStartDate</dt>
+<dd><p>A calendar date specifying the minimum date that a cohort index
+date can appear. Date format is 'yyyymmdd'.</p></dd>
+
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>studyStartDate</th>
-      <td><p>A calendar date specifying the minimum date that a cohort index
-date can appear. Date format is 'yyyymmdd'.</p></td>
-    </tr>
-    <tr>
-      <th>studyEndDate</th>
-      <td><p>A calendar date specifying the maximum date that a cohort index
+<dt>studyEndDate</dt>
+<dd><p>A calendar date specifying the maximum date that a cohort index
 date can appear. Date format is 'yyyymmdd'. Important: the study
 end data is also used to truncate risk windows, meaning no outcomes
-beyond the study end date will be considered.</p></td>
-    </tr>
-    <tr>
-      <th>firstExposureOnly</th>
-      <td><p>Should only the first exposure per subject be included? Note that
+beyond the study end date will be considered.</p></dd>
+
+
+<dt>firstExposureOnly</dt>
+<dd><p>Should only the first exposure per subject be included? Note that
 this is typically done in the <code>createStudyPopulation</code> function,
-but can already be done here for efficiency reasons.</p></td>
-    </tr>
-    <tr>
-      <th>washoutPeriod</th>
-      <td><p>The mininum required continuous observation time prior to index
+but can already be done here for efficiency reasons.</p></dd>
+
+
+<dt>washoutPeriod</dt>
+<dd><p>The mininum required continuous observation time prior to index
 date for a person to be included in the at risk cohort. Note that
 this is typically done in the <code>createStudyPopulation</code> function,
-but can already be done here for efficiency reasons.</p></td>
-    </tr>
-    <tr>
-      <th>sampleSize</th>
-      <td><p>If not NULL, the number of people to sample from the target cohort</p></td>
-    </tr>
-    </table>
+but can already be done here for efficiency reasons.</p></dd>
+
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+<dt>sampleSize</dt>
+<dd><p>If not NULL, the number of people to sample from the target cohort</p></dd>
 
-    <p>A setting object of class <code>restrictPlpDataSettings</code> containing a list getPlpData extra settings</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
+<p>A setting object of class <code>restrictPlpDataSettings</code> containing a list getPlpData extra settings</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Users need to specify the extra restrictions to apply when downloading the target cohort</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/createSampleSettings.html b/docs/reference/createSampleSettings.html
index 049018f9a..f2723ab9e 100644
--- a/docs/reference/createSampleSettings.html
+++ b/docs/reference/createSampleSettings.html
@@ -1,70 +1,15 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create the settings for defining how the trainData from splitData are sampled using 
-default sample functions. — createSampleSettings • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create the settings for defining how the trainData from splitData are sampled using 
-default sample functions. — createSampleSettings" />
-<meta property="og:description" content="Create the settings for defining how the trainData from splitData are sampled using 
-default sample functions." />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create the settings for defining how the trainData from splitData are sampled using 
+default sample functions. — createSampleSettings • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create the settings for defining how the trainData from splitData are sampled using 
+default sample functions. — createSampleSettings"><meta property="og:description" content="Create the settings for defining how the trainData from splitData are sampled using 
+default sample functions."><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -75,13 +20,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -97,13 +41,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -116,10 +59,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -128,13 +71,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -142,44 +88,46 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create the settings for defining how the trainData from <code>splitData</code> are sampled using 
 default sample functions.</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Sampling.R'><code>R/Sampling.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Sampling.R" class="external-link"><code>R/Sampling.R</code></a></small>
     <div class="hidden name"><code>createSampleSettings.Rd</code></div>
     </div>
 
@@ -188,65 +136,65 @@ <h1>Create the settings for defining how the trainData from <code>splitData</cod
 default sample functions.</p>
     </div>
 
-    <pre class="usage"><span class='fu'>createSampleSettings</span><span class='op'>(</span>
-  type <span class='op'>=</span> <span class='st'>"none"</span>,
-  numberOutcomestoNonOutcomes <span class='op'>=</span> <span class='fl'>1</span>,
-  sampleSeed <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/sample.html'>sample</a></span><span class='op'>(</span><span class='fl'>10000</span>, <span class='fl'>1</span><span class='op'>)</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>type</th>
-      <td><p>(character) Choice of:</p><ul>
-<li><p>'none' No sampling is applied - this is the default</p></li>
-<li><p>'underSample')Undersample the non-outcome class to make the data more ballanced</p></li>
-<li><p>'overSample'Oversample the outcome class by adding in each outcome multiple times</p></li>
-</ul></td>
-    </tr>
-    <tr>
-      <th>numberOutcomestoNonOutcomes</th>
-      <td><p>(numeric) An numeric specifying the require number of non-outcomes per outcome</p></td>
-    </tr>
-    <tr>
-      <th>sampleSeed</th>
-      <td><p>(numeric) A seed to use when splitting the data for reproducibility (if not set a random number will be generated)</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>An object of class <code>sampleSettings</code></p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createSampleSettings</span><span class="op">(</span></span>
+<span>  type <span class="op">=</span> <span class="st">"none"</span>,</span>
+<span>  numberOutcomestoNonOutcomes <span class="op">=</span> <span class="fl">1</span>,</span>
+<span>  sampleSeed <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/sample.html" class="external-link">sample</a></span><span class="op">(</span><span class="fl">10000</span>, <span class="fl">1</span><span class="op">)</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>type</dt>
+<dd><p>(character) Choice of:</p><ul><li><p>'none' No sampling is applied - this is the default</p></li>
+<li><p>'underSample' Undersample the non-outcome class to make the data more ballanced</p></li>
+<li><p>'overSample' Oversample the outcome class by adding in each outcome multiple times</p></li>
+</ul></dd>
+
+
+<dt>numberOutcomestoNonOutcomes</dt>
+<dd><p>(numeric) An numeric specifying the require number of non-outcomes per outcome</p></dd>
+
+
+<dt>sampleSeed</dt>
+<dd><p>(numeric) A seed to use when splitting the data for reproducibility (if not set a random number will be generated)</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>An object of class <code>sampleSettings</code></p>
+
+
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Returns an object of class <code>sampleSettings</code> that specifies the sampling function that will be called and the settings</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/createSplineSettings.html b/docs/reference/createSplineSettings.html
new file mode 100644
index 000000000..9b04d1a1b
--- /dev/null
+++ b/docs/reference/createSplineSettings.html
@@ -0,0 +1,188 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create the settings for adding a spline for continuous variables — createSplineSettings • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create the settings for adding a spline for continuous variables — createSplineSettings"><meta property="og:description" content="Create the settings for adding a spline for continuous variables"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Create the settings for adding a spline for continuous variables</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/FeatureEngineering.R" class="external-link"><code>R/FeatureEngineering.R</code></a></small>
+    <div class="hidden name"><code>createSplineSettings.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>Create the settings for adding a spline for continuous variables</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createSplineSettings</span><span class="op">(</span><span class="va">continousCovariateId</span>, <span class="va">knots</span>, analysisId <span class="op">=</span> <span class="fl">683</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>continousCovariateId</dt>
+<dd><p>The covariateId to apply splines to</p></dd>
+
+
+<dt>knots</dt>
+<dd><p>Either number of knots of vector of split values</p></dd>
+
+
+<dt>analysisId</dt>
+<dd><p>The analysisId to use for the spline covariates</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>An object of class <code>featureEngineeringSettings</code></p>
+
+
+    </div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>Returns an object of class <code>featureEngineeringSettings</code> that specifies the sampling function that will be called and the settings</p>
+    </div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/createStratifiedImputationSettings.html b/docs/reference/createStratifiedImputationSettings.html
new file mode 100644
index 000000000..753176d9b
--- /dev/null
+++ b/docs/reference/createStratifiedImputationSettings.html
@@ -0,0 +1,184 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create the settings for adding a spline for continuous variables — createStratifiedImputationSettings • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create the settings for adding a spline for continuous variables — createStratifiedImputationSettings"><meta property="og:description" content="Create the settings for adding a spline for continuous variables"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Create the settings for adding a spline for continuous variables</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/FeatureEngineering.R" class="external-link"><code>R/FeatureEngineering.R</code></a></small>
+    <div class="hidden name"><code>createStratifiedImputationSettings.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>Create the settings for adding a spline for continuous variables</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createStratifiedImputationSettings</span><span class="op">(</span><span class="va">covariateId</span>, ageSplits <span class="op">=</span> <span class="cn">NULL</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>covariateId</dt>
+<dd><p>The covariateId that needs imputed values</p></dd>
+
+
+<dt>ageSplits</dt>
+<dd><p>A vector of age splits in years to create age groups</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>An object of class <code>featureEngineeringSettings</code></p>
+
+
+    </div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>Returns an object of class <code>featureEngineeringSettings</code> that specifies how to do stratified imputation</p>
+    </div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/createStudyPopulation.html b/docs/reference/createStudyPopulation.html
index fa9c31d7f..70fb48349 100644
--- a/docs/reference/createStudyPopulation.html
+++ b/docs/reference/createStudyPopulation.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create a study population — createStudyPopulation • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create a study population — createStudyPopulation" />
-<meta property="og:description" content="Create a study population" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create a study population — createStudyPopulation • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create a study population — createStudyPopulation"><meta property="og:description" content="Create a study population"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create a study population</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/PopulationSettings.R'><code>R/PopulationSettings.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/PopulationSettings.R" class="external-link"><code>R/PopulationSettings.R</code></a></small>
     <div class="hidden name"><code>createStudyPopulation.Rd</code></div>
     </div>
 
@@ -183,78 +131,86 @@ <h1>Create a study population</h1>
     <p>Create a study population</p>
     </div>
 
-    <pre class="usage"><span class='fu'>createStudyPopulation</span><span class='op'>(</span>
-  <span class='va'>plpData</span>,
-  <span class='va'>outcomeId</span>,
-  <span class='va'>populationSettings</span>,
-  population <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpData</th>
-      <td><p>An object of type <code>plpData</code> as generated using
-<code>getplpData</code>.</p></td>
-    </tr>
-    <tr>
-      <th>outcomeId</th>
-      <td><p>The  ID of the outcome.</p></td>
-    </tr>
-    <tr>
-      <th>populationSettings</th>
-      <td><p>An object of class populationSettings created using <code>createPopulationSettings</code></p></td>
-    </tr>
-    <tr>
-      <th>population</th>
-      <td><p>If specified, this population will be used as the starting point instead of the
-cohorts in the <code>plpData</code> object.</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A data frame specifying the study population. This data frame will have the following columns:</p><dl>
-<dt>rowId</dt><dd><p>A unique identifier for an exposure</p></dd>
-<dt>subjectId</dt><dd><p>The person ID of the subject</p></dd>
-<dt>cohortStartdate</dt><dd><p>The index date</p></dd>
-<dt>outcomeCount</dt><dd><p>The number of outcomes observed during the risk window</p></dd>
-<dt>timeAtRisk</dt><dd><p>The number of days in the risk window</p></dd>
-<dt>survivalTime</dt><dd><p>The number of days until either the outcome or the end of the risk window</p></dd>
-
-</dl>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createStudyPopulation</span><span class="op">(</span></span>
+<span>  <span class="va">plpData</span>,</span>
+<span>  <span class="va">outcomeId</span>,</span>
+<span>  <span class="va">populationSettings</span>,</span>
+<span>  population <span class="op">=</span> <span class="cn">NULL</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpData</dt>
+<dd><p>An object of type <code>plpData</code> as generated using
+<code>getplpData</code>.</p></dd>
+
+
+<dt>outcomeId</dt>
+<dd><p>The  ID of the outcome.</p></dd>
+
 
+<dt>populationSettings</dt>
+<dd><p>An object of class populationSettings created using <code>createPopulationSettings</code></p></dd>
+
+
+<dt>population</dt>
+<dd><p>If specified, this population will be used as the starting point instead of the
+cohorts in the <code>plpData</code> object.</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A data frame specifying the study population. This data frame will have the following columns:</p><dl><dt>rowId</dt>
+<dd><p>A unique identifier for an exposure</p></dd>
+
+<dt>subjectId</dt>
+<dd><p>The person ID of the subject</p></dd>
+
+<dt>cohortStartdate</dt>
+<dd><p>The index date</p></dd>
+
+<dt>outcomeCount</dt>
+<dd><p>The number of outcomes observed during the risk window</p></dd>
+
+<dt>timeAtRisk</dt>
+<dd><p>The number of days in the risk window</p></dd>
+
+<dt>survivalTime</dt>
+<dd><p>The number of days until either the outcome or the end of the risk window</p></dd>
+
+
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Create a study population by enforcing certain inclusion and exclusion criteria, defining
 a risk window, and determining which outcomes fall inside the risk window.</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/createStudyPopulationSettings.html b/docs/reference/createStudyPopulationSettings.html
index d6e0ed341..e40884cc2 100644
--- a/docs/reference/createStudyPopulationSettings.html
+++ b/docs/reference/createStudyPopulationSettings.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>create the study population settings — createStudyPopulationSettings • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="create the study population settings — createStudyPopulationSettings" />
-<meta property="og:description" content="create the study population settings" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>create the study population settings — createStudyPopulationSettings • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="create the study population settings — createStudyPopulationSettings"><meta property="og:description" content="create the study population settings"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>create the study population settings</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/PopulationSettings.R'><code>R/PopulationSettings.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/PopulationSettings.R" class="external-link"><code>R/PopulationSettings.R</code></a></small>
     <div class="hidden name"><code>createStudyPopulationSettings.Rd</code></div>
     </div>
 
@@ -183,117 +131,116 @@ <h1>create the study population settings</h1>
     <p>create the study population settings</p>
     </div>
 
-    <pre class="usage"><span class='fu'>createStudyPopulationSettings</span><span class='op'>(</span>
-  binary <span class='op'>=</span> <span class='cn'>T</span>,
-  includeAllOutcomes <span class='op'>=</span> <span class='cn'>T</span>,
-  firstExposureOnly <span class='op'>=</span> <span class='cn'>FALSE</span>,
-  washoutPeriod <span class='op'>=</span> <span class='fl'>0</span>,
-  removeSubjectsWithPriorOutcome <span class='op'>=</span> <span class='cn'>TRUE</span>,
-  priorOutcomeLookback <span class='op'>=</span> <span class='fl'>99999</span>,
-  requireTimeAtRisk <span class='op'>=</span> <span class='cn'>T</span>,
-  minTimeAtRisk <span class='op'>=</span> <span class='fl'>364</span>,
-  riskWindowStart <span class='op'>=</span> <span class='fl'>1</span>,
-  startAnchor <span class='op'>=</span> <span class='st'>"cohort start"</span>,
-  riskWindowEnd <span class='op'>=</span> <span class='fl'>365</span>,
-  endAnchor <span class='op'>=</span> <span class='st'>"cohort start"</span>,
-  restrictTarToCohortEnd <span class='op'>=</span> <span class='cn'>F</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>binary</th>
-      <td><p>Forces the outcomeCount to be 0 or 1 (use for binary prediction problems)</p></td>
-    </tr>
-    <tr>
-      <th>includeAllOutcomes</th>
-      <td><p>(binary) indicating whether to include people with outcomes who are not observed for the whole at risk period</p></td>
-    </tr>
-    <tr>
-      <th>firstExposureOnly</th>
-      <td><p>Should only the first exposure per subject be included? Note that
-this is typically done in the <code>createStudyPopulation</code> function,</p></td>
-    </tr>
-    <tr>
-      <th>washoutPeriod</th>
-      <td><p>The mininum required continuous observation time prior to index
-date for a person to be included in the cohort.</p></td>
-    </tr>
-    <tr>
-      <th>removeSubjectsWithPriorOutcome</th>
-      <td><p>Remove subjects that have the outcome prior to the risk window start?</p></td>
-    </tr>
-    <tr>
-      <th>priorOutcomeLookback</th>
-      <td><p>How many days should we look back when identifying prior outcomes?</p></td>
-    </tr>
-    <tr>
-      <th>requireTimeAtRisk</th>
-      <td><p>Should subject without time at risk be removed?</p></td>
-    </tr>
-    <tr>
-      <th>minTimeAtRisk</th>
-      <td><p>The minimum number of days at risk required to be included</p></td>
-    </tr>
-    <tr>
-      <th>riskWindowStart</th>
-      <td><p>The start of the risk window (in days) relative to the index date (+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createStudyPopulationSettings</span><span class="op">(</span></span>
+<span>  binary <span class="op">=</span> <span class="cn">T</span>,</span>
+<span>  includeAllOutcomes <span class="op">=</span> <span class="cn">T</span>,</span>
+<span>  firstExposureOnly <span class="op">=</span> <span class="cn">FALSE</span>,</span>
+<span>  washoutPeriod <span class="op">=</span> <span class="fl">0</span>,</span>
+<span>  removeSubjectsWithPriorOutcome <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>  priorOutcomeLookback <span class="op">=</span> <span class="fl">99999</span>,</span>
+<span>  requireTimeAtRisk <span class="op">=</span> <span class="cn">T</span>,</span>
+<span>  minTimeAtRisk <span class="op">=</span> <span class="fl">364</span>,</span>
+<span>  riskWindowStart <span class="op">=</span> <span class="fl">1</span>,</span>
+<span>  startAnchor <span class="op">=</span> <span class="st">"cohort start"</span>,</span>
+<span>  riskWindowEnd <span class="op">=</span> <span class="fl">365</span>,</span>
+<span>  endAnchor <span class="op">=</span> <span class="st">"cohort start"</span>,</span>
+<span>  restrictTarToCohortEnd <span class="op">=</span> <span class="cn">F</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>binary</dt>
+<dd><p>Forces the outcomeCount to be 0 or 1 (use for binary prediction problems)</p></dd>
+
+
+<dt>includeAllOutcomes</dt>
+<dd><p>(binary) indicating whether to include people with outcomes who are not observed for the whole at risk period</p></dd>
+
+
+<dt>firstExposureOnly</dt>
+<dd><p>Should only the first exposure per subject be included? Note that
+this is typically done in the <code>createStudyPopulation</code> function,</p></dd>
+
+
+<dt>washoutPeriod</dt>
+<dd><p>The mininum required continuous observation time prior to index
+date for a person to be included in the cohort.</p></dd>
+
+
+<dt>removeSubjectsWithPriorOutcome</dt>
+<dd><p>Remove subjects that have the outcome prior to the risk window start?</p></dd>
+
+
+<dt>priorOutcomeLookback</dt>
+<dd><p>How many days should we look back when identifying prior outcomes?</p></dd>
+
+
+<dt>requireTimeAtRisk</dt>
+<dd><p>Should subject without time at risk be removed?</p></dd>
+
+
+<dt>minTimeAtRisk</dt>
+<dd><p>The minimum number of days at risk required to be included</p></dd>
+
+
+<dt>riskWindowStart</dt>
+<dd><p>The start of the risk window (in days) relative to the index date (+
 days of exposure if the <code>addExposureDaysToStart</code> parameter is
-specified).</p></td>
-    </tr>
-    <tr>
-      <th>startAnchor</th>
-      <td><p>The anchor point for the start of the risk window. Can be "cohort start" or "cohort end".</p></td>
-    </tr>
-    <tr>
-      <th>riskWindowEnd</th>
-      <td><p>The end of the risk window (in days) relative to the index data (+
+specified).</p></dd>
+
+
+<dt>startAnchor</dt>
+<dd><p>The anchor point for the start of the risk window. Can be "cohort start" or "cohort end".</p></dd>
+
+
+<dt>riskWindowEnd</dt>
+<dd><p>The end of the risk window (in days) relative to the index data (+
 days of exposure if the <code>addExposureDaysToEnd</code> parameter is
-specified).</p></td>
-    </tr>
-    <tr>
-      <th>endAnchor</th>
-      <td><p>The anchor point for the end of the risk window. Can be "cohort start" or "cohort end".</p></td>
-    </tr>
-    <tr>
-      <th>restrictTarToCohortEnd</th>
-      <td><p>If using a survival model and you want the time-at-risk to end at the cohort end date set this to T</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A list containing all the settings required for creating the study population</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+specified).</p></dd>
+
 
+<dt>endAnchor</dt>
+<dd><p>The anchor point for the end of the risk window. Can be "cohort start" or "cohort end".</p></dd>
+
+
+<dt>restrictTarToCohortEnd</dt>
+<dd><p>If using a survival model and you want the time-at-risk to end at the cohort end date set this to T</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A list containing all the settings required for creating the study population</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Takes as input the inputs to create study population</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/createTempModelLoc.html b/docs/reference/createTempModelLoc.html
new file mode 100644
index 000000000..e91a74cf3
--- /dev/null
+++ b/docs/reference/createTempModelLoc.html
@@ -0,0 +1,162 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create a temporary model location — createTempModelLoc • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create a temporary model location — createTempModelLoc"><meta property="og:description" content="Create a temporary model location"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Create a temporary model location</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/HelperFunctions.R" class="external-link"><code>R/HelperFunctions.R</code></a></small>
+    <div class="hidden name"><code>createTempModelLoc.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>Create a temporary model location</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createTempModelLoc</span><span class="op">(</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/createUnivariateFeatureSelection.html b/docs/reference/createUnivariateFeatureSelection.html
index 8c7eab965..4f4c8fb19 100644
--- a/docs/reference/createUnivariateFeatureSelection.html
+++ b/docs/reference/createUnivariateFeatureSelection.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create the settings for defining any feature selection that will be done — createUnivariateFeatureSelection • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create the settings for defining any feature selection that will be done — createUnivariateFeatureSelection" />
-<meta property="og:description" content="Create the settings for defining any feature selection that will be done" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create the settings for defining any feature selection that will be done — createUnivariateFeatureSelection • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create the settings for defining any feature selection that will be done — createUnivariateFeatureSelection"><meta property="og:description" content="Create the settings for defining any feature selection that will be done"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create the settings for defining any feature selection that will be done</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/FeatureEngineering.R'><code>R/FeatureEngineering.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/FeatureEngineering.R" class="external-link"><code>R/FeatureEngineering.R</code></a></small>
     <div class="hidden name"><code>createUnivariateFeatureSelection.Rd</code></div>
     </div>
 
@@ -183,49 +131,50 @@ <h1>Create the settings for defining any feature selection that will be done</h1
     <p>Create the settings for defining any feature selection that will be done</p>
     </div>
 
-    <pre class="usage"><span class='fu'>createUnivariateFeatureSelection</span><span class='op'>(</span>k <span class='op'>=</span> <span class='fl'>100</span><span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createUnivariateFeatureSelection</span><span class="op">(</span>k <span class="op">=</span> <span class="fl">100</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>k</th>
-      <td><p>This function returns the K features most associated (univariately) to the outcome</p></td>
-    </tr>
-    </table>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>k</dt>
+<dd><p>This function returns the K features most associated (univariately) to the outcome</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
-    <p>An object of class <code>featureEngineeringSettings</code></p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<p>An object of class <code>featureEngineeringSettings</code></p>
 
+
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Returns an object of class <code>featureEngineeringSettings</code> that specifies the sampling function that will be called and the settings</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/createValidationDesign.html b/docs/reference/createValidationDesign.html
new file mode 100644
index 000000000..f3601d2c9
--- /dev/null
+++ b/docs/reference/createValidationDesign.html
@@ -0,0 +1,200 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>createValidationDesign - Define the validation design for external validation — createValidationDesign • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="createValidationDesign - Define the validation design for external validation — createValidationDesign"><meta property="og:description" content="createValidationDesign - Define the validation design for external validation"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>createValidationDesign - Define the validation design for external validation</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ExternalValidatePlp.R" class="external-link"><code>R/ExternalValidatePlp.R</code></a></small>
+    <div class="hidden name"><code>createValidationDesign.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>createValidationDesign - Define the validation design for external validation</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createValidationDesign</span><span class="op">(</span></span>
+<span>  <span class="va">targetId</span>,</span>
+<span>  <span class="va">outcomeId</span>,</span>
+<span>  <span class="va">populationSettings</span>,</span>
+<span>  <span class="va">restrictPlpDataSettings</span>,</span>
+<span>  <span class="va">plpModelList</span>,</span>
+<span>  recalibrate <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  runCovariateSummary <span class="op">=</span> <span class="cn">TRUE</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>targetId</dt>
+<dd><p>The targetId of the target cohort to validate on</p></dd>
+
+
+<dt>outcomeId</dt>
+<dd><p>The outcomeId of the outcome cohort to validate on</p></dd>
+
+
+<dt>populationSettings</dt>
+<dd><p>A list of population restriction settings created by <code>createPopulationSettings</code></p></dd>
+
+
+<dt>restrictPlpDataSettings</dt>
+<dd><p>A list of plpData restriction settings created by <code>createRestrictPlpDataSettings</code></p></dd>
+
+
+<dt>plpModelList</dt>
+<dd><p>A list of plpModels objects created by <code>runPlp</code> or a path to such objects</p></dd>
+
+
+<dt>recalibrate</dt>
+<dd><p>A vector of characters specifying the recalibration method to apply,</p></dd>
+
+
+<dt>runCovariateSummary</dt>
+<dd><p>whether to run the covariate summary for the validation data</p></dd>
+
+</dl></div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/createValidationSettings.html b/docs/reference/createValidationSettings.html
index cd816a609..c21539127 100644
--- a/docs/reference/createValidationSettings.html
+++ b/docs/reference/createValidationSettings.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>createValidationSettings define optional settings for performing external validation — createValidationSettings • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="createValidationSettings define optional settings for performing external validation — createValidationSettings" />
-<meta property="og:description" content="This function creates the settings required by externalValidatePlp" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>createValidationSettings define optional settings for performing external validation — createValidationSettings • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="createValidationSettings define optional settings for performing external validation — createValidationSettings"><meta property="og:description" content="This function creates the settings required by externalValidatePlp"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>createValidationSettings define optional settings for performing external validation</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ExternalValidatePlp.R'><code>R/ExternalValidatePlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ExternalValidatePlp.R" class="external-link"><code>R/ExternalValidatePlp.R</code></a></small>
     <div class="hidden name"><code>createValidationSettings.Rd</code></div>
     </div>
 
@@ -183,53 +131,52 @@ <h1>createValidationSettings define optional settings for performing external va
     <p>This function creates the settings required by externalValidatePlp</p>
     </div>
 
-    <pre class="usage"><span class='fu'>createValidationSettings</span><span class='op'>(</span>recalibrate <span class='op'>=</span> <span class='cn'>NULL</span>, runCovariateSummary <span class='op'>=</span> <span class='cn'>T</span><span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">createValidationSettings</span><span class="op">(</span>recalibrate <span class="op">=</span> <span class="cn">NULL</span>, runCovariateSummary <span class="op">=</span> <span class="cn">T</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>recalibrate</th>
-      <td><p>A vector of characters specifying the recalibration method to apply</p></td>
-    </tr>
-    <tr>
-      <th>runCovariateSummary</th>
-      <td><p>Whether to run the covariate summary for the validation data</p></td>
-    </tr>
-    </table>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>recalibrate</dt>
+<dd><p>A vector of characters specifying the recalibration method to apply</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
 
-    <p>A setting object of class <code>validationSettings</code> containing a list of settings for externalValidatePlp</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<dt>runCovariateSummary</dt>
+<dd><p>Whether to run the covariate summary for the validation data</p></dd>
 
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A setting object of class <code>validationSettings</code> containing a list of settings for externalValidatePlp</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Users need to specify whether they want to sample or recalibate when performing external validation</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/diagnoseMultiplePlp.html b/docs/reference/diagnoseMultiplePlp.html
new file mode 100644
index 000000000..5c24f5c05
--- /dev/null
+++ b/docs/reference/diagnoseMultiplePlp.html
@@ -0,0 +1,203 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Run a list of predictions diagnoses — diagnoseMultiplePlp • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Run a list of predictions diagnoses — diagnoseMultiplePlp"><meta property="og:description" content="Run a list of predictions diagnoses"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Run a list of predictions diagnoses</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/DiagnosePlp.R" class="external-link"><code>R/DiagnosePlp.R</code></a></small>
+    <div class="hidden name"><code>diagnoseMultiplePlp.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>Run a list of predictions diagnoses</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">diagnoseMultiplePlp</span><span class="op">(</span></span>
+<span>  databaseDetails <span class="op">=</span> <span class="fu"><a href="createDatabaseDetails.html">createDatabaseDetails</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  modelDesignList <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fu"><a href="createModelDesign.html">createModelDesign</a></span><span class="op">(</span>targetId <span class="op">=</span> <span class="fl">1</span>, outcomeId <span class="op">=</span> <span class="fl">2</span>, modelSettings <span class="op">=</span></span>
+<span>    <span class="fu"><a href="setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span><span class="op">)</span>, <span class="fu"><a href="createModelDesign.html">createModelDesign</a></span><span class="op">(</span>targetId <span class="op">=</span> <span class="fl">1</span>, outcomeId <span class="op">=</span> <span class="fl">3</span>,</span>
+<span>    modelSettings <span class="op">=</span> <span class="fu"><a href="setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>,</span>
+<span>  cohortDefinitions <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  logSettings <span class="op">=</span> <span class="fu"><a href="createLogSettings.html">createLogSettings</a></span><span class="op">(</span>verbosity <span class="op">=</span> <span class="st">"DEBUG"</span>, timeStamp <span class="op">=</span> <span class="cn">T</span>, logName <span class="op">=</span></span>
+<span>    <span class="st">"diagnosePlp Log"</span><span class="op">)</span>,</span>
+<span>  saveDirectory <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>databaseDetails</dt>
+<dd><p>The database settings created using <code><a href="createDatabaseDetails.html">createDatabaseDetails()</a></code></p></dd>
+
+
+<dt>modelDesignList</dt>
+<dd><p>A list of model designs created using <code><a href="createModelDesign.html">createModelDesign()</a></code></p></dd>
+
+
+<dt>cohortDefinitions</dt>
+<dd><p>A list of cohort definitions for the target and outcome cohorts</p></dd>
+
+
+<dt>logSettings</dt>
+<dd><p>The setting spexcifying the logging for the analyses created using <code><a href="createLogSettings.html">createLogSettings()</a></code></p></dd>
+
+
+<dt>saveDirectory</dt>
+<dd><p>Name of the folder where all the outputs will written to.</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A data frame with the following columns:</p><table class="table table"><tr><td><code>analysisId</code></td><td>The unique identifier
+for a set of analysis choices.</td></tr><tr><td><code>targetId</code></td><td>The ID of the target cohort populations.</td></tr><tr><td><code>outcomeId</code></td><td>The ID of the outcomeId.</td></tr><tr><td><code>dataLocation</code></td><td>The location where the plpData was saved</td></tr><tr><td><code>the settings ids</code></td><td>The ids for all other settings used for model development.</td></tr></table></div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>This function will run all specified prediction design diagnoses as defined using .</p>
+    </div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/diagnosePlp.html b/docs/reference/diagnosePlp.html
new file mode 100644
index 000000000..f4282a426
--- /dev/null
+++ b/docs/reference/diagnosePlp.html
@@ -0,0 +1,267 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>diagnostic - Investigates the prediction problem settings - use before training a model — diagnosePlp • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="diagnostic - Investigates the prediction problem settings - use before training a model — diagnosePlp"><meta property="og:description" content="This function runs a set of prediction diagnoses to help pick a suitable T, O, TAR and determine 
+whether the prediction problem is worth executing."><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>diagnostic - Investigates the prediction problem settings - use before training a model</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/DiagnosePlp.R" class="external-link"><code>R/DiagnosePlp.R</code></a></small>
+    <div class="hidden name"><code>diagnosePlp.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>This function runs a set of prediction diagnoses to help pick a suitable T, O, TAR and determine 
+whether the prediction problem is worth executing.</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">diagnosePlp</span><span class="op">(</span></span>
+<span>  plpData <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  <span class="va">outcomeId</span>,</span>
+<span>  <span class="va">analysisId</span>,</span>
+<span>  <span class="va">populationSettings</span>,</span>
+<span>  splitSettings <span class="op">=</span> <span class="fu"><a href="createDefaultSplitSetting.html">createDefaultSplitSetting</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  sampleSettings <span class="op">=</span> <span class="fu"><a href="createSampleSettings.html">createSampleSettings</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  saveDirectory <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  featureEngineeringSettings <span class="op">=</span> <span class="fu"><a href="createFeatureEngineeringSettings.html">createFeatureEngineeringSettings</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  modelSettings <span class="op">=</span> <span class="fu"><a href="setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  logSettings <span class="op">=</span> <span class="fu"><a href="createLogSettings.html">createLogSettings</a></span><span class="op">(</span>verbosity <span class="op">=</span> <span class="st">"DEBUG"</span>, timeStamp <span class="op">=</span> <span class="cn">T</span>, logName <span class="op">=</span></span>
+<span>    <span class="st">"diagnosePlp Log"</span><span class="op">)</span>,</span>
+<span>  preprocessSettings <span class="op">=</span> <span class="fu"><a href="createPreprocessSettings.html">createPreprocessSettings</a></span><span class="op">(</span><span class="op">)</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpData</dt>
+<dd><p>An object of type <code>plpData</code> - the patient level prediction
+data extracted from the CDM.  Can also include an initial population as 
+plpData$popualtion.</p></dd>
+
+
+<dt>outcomeId</dt>
+<dd><p>(integer) The ID of the outcome.</p></dd>
+
+
+<dt>analysisId</dt>
+<dd><p>(integer) Identifier for the analysis. It is used to create, e.g., the result folder. Default is a timestamp.</p></dd>
+
+
+<dt>populationSettings</dt>
+<dd><p>An object of type <code>populationSettings</code> created using <code>createStudyPopulationSettings</code> that
+specifies how the data class labels are defined and addition any exclusions to apply to the 
+plpData cohort</p></dd>
+
+
+<dt>splitSettings</dt>
+<dd><p>An object of type <code>splitSettings</code> that specifies how to split the data into train/validation/test.
+The default settings can be created using <code>createDefaultSplitSetting</code>.</p></dd>
+
+
+<dt>sampleSettings</dt>
+<dd><p>An object of type <code>sampleSettings</code> that specifies any under/over sampling to be done.
+The default is none.</p></dd>
+
+
+<dt>saveDirectory</dt>
+<dd><p>The path to the directory where the results will be saved (if NULL uses working directory)</p></dd>
+
+
+<dt>featureEngineeringSettings</dt>
+<dd><p>An object of <code>featureEngineeringSettings</code> specifying any feature engineering to be learned (using the train data)</p></dd>
+
+
+<dt>modelSettings</dt>
+<dd><p>An object of class <code>modelSettings</code> created using one of the function:</p><ul><li><p>setLassoLogisticRegression() A lasso logistic regression model</p></li>
+<li><p>setGradientBoostingMachine() A gradient boosting machine</p></li>
+<li><p>setAdaBoost() An ada boost model</p></li>
+<li><p>setRandomForest() A random forest model</p></li>
+<li><p>setDecisionTree() A decision tree model</p></li>
+<li><p>setKNN() A KNN model</p></li>
+</ul></dd>
+
+
+<dt>logSettings</dt>
+<dd><p>An object of <code>logSettings</code> created using <code>createLogSettings</code> 
+specifying how the logging is done</p></dd>
+
+
+<dt>preprocessSettings</dt>
+<dd><p>An object of <code>preprocessSettings</code>. This setting specifies the minimum fraction of 
+target population who must have a covariate for it to be included in the model training                            
+and whether to normalise the covariates before training</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>An object containing the model or location where the model is save, the data selection settings, the preprocessing
+and training settings as well as various performance measures obtained by the model.</p>
+<dl><dt>distribution</dt>
+<dd><p>list for each O of a data.frame containing: i) Time to observation end distribution, ii) Time from observation start distribution, iii) Time to event distribution and iv) Time from last prior event to index distribution (only for patients in T who have O before index)</p></dd>
+
+<dt>incident</dt>
+<dd><p>list for each O of incidence of O in T during TAR</p></dd>
+
+<dt>characterization</dt>
+<dd><p>list for each O of Characterization of T, TnO, Tn~O</p></dd>
+
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>Users can define set of Ts, Os, databases and population settings.  A list of data.frames containing details such as
+follow-up time distribution, time-to-event information, characteriszation details, time from last prior event, 
+observation time distribution.</p>
+    </div>
+
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="kw">if</span> <span class="op">(</span><span class="cn">FALSE</span><span class="op">)</span> <span class="op">{</span></span></span>
+<span class="r-in"><span><span class="co">#******** EXAMPLE 1 ********* </span></span></span>
+<span class="r-in"><span><span class="op">}</span> </span></span>
+</code></pre></div>
+    </div>
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/diagnostic.html b/docs/reference/diagnostic.html
deleted file mode 100644
index b66905ad0..000000000
--- a/docs/reference/diagnostic.html
+++ /dev/null
@@ -1,287 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>diagnostic - Investigates the prediction problem settings - use before training a model — diagnostic • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="diagnostic - Investigates the prediction problem settings - use before training a model — diagnostic" />
-<meta property="og:description" content="This function runs a set of prediction diagnoses to help pick a suitable T, O, TAR and determine 
-whether the prediction problem is worth executing." />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/InstallationGuide.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
-    </li>
-    <li>
-      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
-    </li>
-    <li>
-      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>diagnostic - Investigates the prediction problem settings - use before training a model</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Diagnostics.R'><code>R/Diagnostics.R</code></a></small>
-    <div class="hidden name"><code>diagnostic.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>This function runs a set of prediction diagnoses to help pick a suitable T, O, TAR and determine 
-whether the prediction problem is worth executing.</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>diagnostic</span><span class='op'>(</span>
-  plpData <span class='op'>=</span> <span class='cn'>NULL</span>,
-  cdmDatabaseName <span class='op'>=</span> <span class='st'>"none"</span>,
-  <span class='va'>cohortName</span>,
-  <span class='va'>outcomeNames</span>,
-  <span class='va'>databaseDetails</span>,
-  <span class='va'>restrictPlpDataSettings</span>,
-  <span class='va'>populationSettings</span>,
-  outputFolder <span class='op'>=</span> <span class='cn'>NULL</span>,
-  minCellCount <span class='op'>=</span> <span class='fl'>5</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpData</th>
-      <td><p>The data object to do the diagnostic on - if NULL you need to specify the connection settings below</p></td>
-    </tr>
-    <tr>
-      <th>cdmDatabaseName</th>
-      <td><p>The name of the database being diagnosed</p></td>
-    </tr>
-    <tr>
-      <th>cohortName</th>
-      <td><p>Name of the target cohort</p></td>
-    </tr>
-    <tr>
-      <th>outcomeNames</th>
-      <td><p>Vector of outcome names</p></td>
-    </tr>
-    <tr>
-      <th>databaseDetails</th>
-      <td><p>(only used is plpData is NULL) The database details created using <code>createDatabaseDetails</code></p></td>
-    </tr>
-    <tr>
-      <th>restrictPlpDataSettings</th>
-      <td><p>(only used is plpData is NULL) The restrictPlpDataSettings created using <code>createRestrictPlpDataSettings</code></p></td>
-    </tr>
-    <tr>
-      <th>populationSettings</th>
-      <td><p>The population setting details created using <code>createPopulationSettings</code></p></td>
-    </tr>
-    <tr>
-      <th>outputFolder</th>
-      <td><p>Location to save results for shiny app</p></td>
-    </tr>
-    <tr>
-      <th>minCellCount</th>
-      <td><p>The minimum count that will be displayed</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>An object containing the model or location where the model is save, the data selection settings, the preprocessing
-and training settings as well as various performance measures obtained by the model.</p>
-<dt>distribution</dt><dd><p>list for each O of a data.frame containing: i) Time to observation end distribution, ii) Time from observation start distribution, iii) Time to event distribution and iv) Time from last prior event to index distribution (only for patients in T who have O before index)</p></dd>
-<dt>incident</dt><dd><p>list for each O of incidence of O in T during TAR</p></dd>
-<dt>characterization</dt><dd><p>list for each O of Characterization of T, TnO, Tn~O</p></dd>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Users can define set of Ts, Os, databases and population settings.  A list of data.frames containing details such as
-follow-up time distribution, time-to-event information, characteriszation details, time from last prior event, 
-observation time distribution.</p>
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='co'>#******** EXAMPLE 1 ********* </span>
-<span class='op'>}</span> 
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/diagnosticOddsRatio.html b/docs/reference/diagnosticOddsRatio.html
index d6c34055f..806f33f02 100644
--- a/docs/reference/diagnosticOddsRatio.html
+++ b/docs/reference/diagnosticOddsRatio.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculate the diagnostic odds ratio — diagnosticOddsRatio • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculate the diagnostic odds ratio — diagnosticOddsRatio" />
-<meta property="og:description" content="Calculate the diagnostic odds ratio" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculate the diagnostic odds ratio — diagnosticOddsRatio • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculate the diagnostic odds ratio — diagnosticOddsRatio"><meta property="og:description" content="Calculate the diagnostic odds ratio"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculate the diagnostic odds ratio</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ThresholdSummary.R'><code>R/ThresholdSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ThresholdSummary.R" class="external-link"><code>R/ThresholdSummary.R</code></a></small>
     <div class="hidden name"><code>diagnosticOddsRatio.Rd</code></div>
     </div>
 
@@ -183,61 +131,60 @@ <h1>Calculate the diagnostic odds ratio</h1>
     <p>Calculate the diagnostic odds ratio</p>
     </div>
 
-    <pre class="usage"><span class='fu'>diagnosticOddsRatio</span><span class='op'>(</span><span class='va'>TP</span>, <span class='va'>TN</span>, <span class='va'>FN</span>, <span class='va'>FP</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>TP</th>
-      <td><p>Number of true positives</p></td>
-    </tr>
-    <tr>
-      <th>TN</th>
-      <td><p>Number of true negatives</p></td>
-    </tr>
-    <tr>
-      <th>FN</th>
-      <td><p>Number of false negatives</p></td>
-    </tr>
-    <tr>
-      <th>FP</th>
-      <td><p>Number of false positives</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>diagnosticOddsRatio value</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">diagnosticOddsRatio</span><span class="op">(</span><span class="va">TP</span>, <span class="va">TN</span>, <span class="va">FN</span>, <span class="va">FP</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>TP</dt>
+<dd><p>Number of true positives</p></dd>
+
+
+<dt>TN</dt>
+<dd><p>Number of true negatives</p></dd>
+
+
+<dt>FN</dt>
+<dd><p>Number of false negatives</p></dd>
+
+
+<dt>FP</dt>
+<dd><p>Number of false positives</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>diagnosticOddsRatio value</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculate the diagnostic odds ratio</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/drawAttritionDiagramPlp.html b/docs/reference/drawAttritionDiagramPlp.html
deleted file mode 100644
index 13bdca5d4..000000000
--- a/docs/reference/drawAttritionDiagramPlp.html
+++ /dev/null
@@ -1,244 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Draw the attrition diagram — drawAttritionDiagramPlp • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Draw the attrition diagram — drawAttritionDiagramPlp" />
-<meta property="og:description" content="drawAttritionDiagramPlp draws the attition diagram, showing how many people were excluded from
-the study population, and for what reasons." />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Draw the attrition diagram</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/AttritionDiagramPlp.R'><code>R/AttritionDiagramPlp.R</code></a></small>
-    <div class="hidden name"><code>drawAttritionDiagramPlp.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p><code>drawAttritionDiagramPlp</code> draws the attition diagram, showing how many people were excluded from
-the study population, and for what reasons.</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>drawAttritionDiagramPlp</span><span class='op'>(</span>
-  <span class='va'>attrition</span>,
-  targetLabel <span class='op'>=</span> <span class='st'>"Target Population"</span>,
-  outcomeLabel <span class='op'>=</span> <span class='st'>"Outcome Count"</span>,
-  fileName <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>attrition</th>
-      <td><p>The table of attrition details return from the population attr(popualtion, 'metaData')$attrition</p></td>
-    </tr>
-    <tr>
-      <th>targetLabel</th>
-      <td><p>A label to us for the treated cohort.</p></td>
-    </tr>
-    <tr>
-      <th>outcomeLabel</th>
-      <td><p>A label to us for the comparator cohort.</p></td>
-    </tr>
-    <tr>
-      <th>fileName</th>
-      <td><p>Name of the file where the plot should be saved, for example 'plot.png'.
-See the function <code>ggsave</code> in the ggplot2 package for supported file
-formats.</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A ggplot object. Use the <code><a href='https://ggplot2.tidyverse.org/reference/ggsave.html'>ggsave</a></code> function to save to file in a different
-format.</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/evaluateExistingModel.html b/docs/reference/evaluateExistingModel.html
deleted file mode 100644
index 256889239..000000000
--- a/docs/reference/evaluateExistingModel.html
+++ /dev/null
@@ -1,347 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>evaluateExistingModel — evaluateExistingModel • PatientLevelPrediction</title>
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script>
-
-<!-- sticky kit -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-<meta property="og:title" content="evaluateExistingModel — evaluateExistingModel" />
-
-<meta property="og:description" content="This function implements an existing model" />
-<meta name="twitter:card" content="summary" />
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-  </head>
-
-  <body>
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.5</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>evaluateExistingModel</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ExternalValidatePlp.R'><code>R/ExternalValidatePlp.R</code></a></small>
-    <div class="hidden name"><code>evaluateExistingModel.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    
-    <p>This function implements an existing model</p>
-    
-    </div>
-
-    <pre class="usage"><span class='fu'>evaluateExistingModel</span>(<span class='no'>modelTable</span>, <span class='no'>covariateTable</span>, <span class='kw'>interceptTable</span> <span class='kw'>=</span> <span class='kw'>NULL</span>,
-  <span class='kw'>type</span> <span class='kw'>=</span> <span class='st'>"score"</span>, <span class='no'>covariateSettings</span>, <span class='kw'>customCovariates</span> <span class='kw'>=</span> <span class='kw'>NULL</span>,
-  <span class='kw'>addExposureDaysToStart</span> <span class='kw'>=</span> <span class='no'>F</span>, <span class='kw'>riskWindowStart</span> <span class='kw'>=</span> <span class='fl'>1</span>,
-  <span class='kw'>addExposureDaysToEnd</span> <span class='kw'>=</span> <span class='no'>F</span>, <span class='kw'>riskWindowEnd</span> <span class='kw'>=</span> <span class='fl'>365</span>, <span class='kw'>requireTimeAtRisk</span> <span class='kw'>=</span> <span class='no'>T</span>,
-  <span class='kw'>minTimeAtRisk</span> <span class='kw'>=</span> <span class='fl'>364</span>, <span class='kw'>includeAllOutcomes</span> <span class='kw'>=</span> <span class='no'>T</span>,
-  <span class='kw'>removeSubjectsWithPriorOutcome</span> <span class='kw'>=</span> <span class='no'>T</span>, <span class='kw'>priorOutcomeLookback</span> <span class='kw'>=</span> <span class='fl'>99999</span>,
-  <span class='kw'>verbosity</span> <span class='kw'>=</span> <span class='st'>"INFO"</span>, <span class='kw'>washoutPeriod</span> <span class='kw'>=</span> <span class='fl'>0</span>, <span class='kw'>firstExposureOnly</span> <span class='kw'>=</span> <span class='no'>F</span>,
-  <span class='kw'>binary</span> <span class='kw'>=</span> <span class='no'>T</span>, <span class='no'>connectionDetails</span>, <span class='no'>cdmDatabaseSchema</span>, <span class='no'>cohortDatabaseSchema</span>,
-  <span class='no'>cohortTable</span>, <span class='no'>cohortId</span>, <span class='no'>outcomeDatabaseSchema</span>, <span class='no'>outcomeTable</span>, <span class='no'>outcomeId</span>,
-  <span class='kw'>oracleTempSchema</span> <span class='kw'>=</span> <span class='no'>cdmDatabaseSchema</span>, <span class='kw'>modelName</span> <span class='kw'>=</span> <span class='st'>"existingModel"</span>,
-  <span class='kw'>calibrationPopulation</span> <span class='kw'>=</span> <span class='kw'>NULL</span>, <span class='kw'>covariateSummary</span> <span class='kw'>=</span> <span class='no'>T</span>, <span class='kw'>cdmVersion</span> <span class='kw'>=</span> <span class='fl'>5</span>)</pre>
-    
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>modelTable</th>
-      <td><p>The model covariates and scores</p></td>
-    </tr>
-    <tr>
-      <th>covariateTable</th>
-      <td><p>The mapping from model covariates to standard covariates</p></td>
-    </tr>
-    <tr>
-      <th>interceptTable</th>
-      <td><p>The model intercepts</p></td>
-    </tr>
-    <tr>
-      <th>type</th>
-      <td><p>Model type (score or logistic)</p></td>
-    </tr>
-    <tr>
-      <th>covariateSettings</th>
-      <td><p>The standard covariate settings (specify covariate lookback time)</p></td>
-    </tr>
-    <tr>
-      <th>customCovariates</th>
-      <td><p>A table of covariateId, sql (sql creates the custom covariate)</p></td>
-    </tr>
-    <tr>
-      <th>addExposureDaysToStart</th>
-      <td><p>riskWindowStart relative to the cohort end date instead of the cohort start date?</p></td>
-    </tr>
-    <tr>
-      <th>riskWindowStart</th>
-      <td><p>The day after index to start predicting the outcome</p></td>
-    </tr>
-    <tr>
-      <th>addExposureDaysToEnd</th>
-      <td><p>riskWindowEnd relative to the cohort end date instead of the cohort start date?</p></td>
-    </tr>
-    <tr>
-      <th>riskWindowEnd</th>
-      <td><p>The day after index to stop predicting the outcome</p></td>
-    </tr>
-    <tr>
-      <th>requireTimeAtRisk</th>
-      <td><p>Do you want to ignore people who leave the database some point between the riskWindowStart and riskWindowEnd</p></td>
-    </tr>
-    <tr>
-      <th>minTimeAtRisk</th>
-      <td><p>If requireTimeAtRisk is TRUE, how many days must they be observed before leaving to get included (default recommendation is all risk period: riskWindowEnd-riskWindowStart)</p></td>
-    </tr>
-    <tr>
-      <th>includeAllOutcomes</th>
-      <td><p>Setting this to TRUE means people with the outcome who leave the data during the risk period are still included, so only non-outcome people who leave during the risk period are removed</p></td>
-    </tr>
-    <tr>
-      <th>removeSubjectsWithPriorOutcome</th>
-      <td><p>Remove people from the target population if they have the outcome prior to target cohort start date</p></td>
-    </tr>
-    <tr>
-      <th>priorOutcomeLookback</th>
-      <td><p>Lookback for removeSubjectsWithPriorOutcome</p></td>
-    </tr>
-    <tr>
-      <th>verbosity</th>
-      <td><p>The study population creation verbosity</p></td>
-    </tr>
-    <tr>
-      <th>washoutPeriod</th>
-      <td><p>Remove patients from the population with less than washoutPeriod of days prior observation</p></td>
-    </tr>
-    <tr>
-      <th>firstExposureOnly</th>
-      <td><p>If patients are in the target population multiple times, use only the first date</p></td>
-    </tr>
-    <tr>
-      <th>binary</th>
-      <td><p>Binary classificsation (T or F)</p></td>
-    </tr>
-    <tr>
-      <th>connectionDetails</th>
-      <td><p>The details to connect to the CDM</p></td>
-    </tr>
-    <tr>
-      <th>cdmDatabaseSchema</th>
-      <td><p>A string specifying the database containing the cdm</p></td>
-    </tr>
-    <tr>
-      <th>cohortDatabaseSchema</th>
-      <td><p>A string specifying the database containing the target cohorts</p></td>
-    </tr>
-    <tr>
-      <th>cohortTable</th>
-      <td><p>A string specifying the table containing the target cohorts</p></td>
-    </tr>
-    <tr>
-      <th>cohortId</th>
-      <td><p>An iteger specifying the cohort id for the target cohorts</p></td>
-    </tr>
-    <tr>
-      <th>outcomeDatabaseSchema</th>
-      <td><p>A string specifying the database containing the outcome cohorts</p></td>
-    </tr>
-    <tr>
-      <th>outcomeTable</th>
-      <td><p>A string specifying the table containing the outcome cohorts</p></td>
-    </tr>
-    <tr>
-      <th>outcomeId</th>
-      <td><p>An iteger specifying the cohort id for the outcome cohorts</p></td>
-    </tr>
-    <tr>
-      <th>oracleTempSchema</th>
-      <td><p>The temp oracle schema</p></td>
-    </tr>
-    <tr>
-      <th>modelName</th>
-      <td><p>The name of the model</p></td>
-    </tr>
-    <tr>
-      <th>calibrationPopulation</th>
-      <td><p>A data.frame of subjectId, cohortStartDate, indexes used to recalibrate the model on new data</p></td>
-    </tr>
-    <tr>
-      <th>covariateSummary</th>
-      <td><p>Whether to calculate the covariateSummary</p></td>
-    </tr>
-    <tr>
-      <th>cdmVersion</th>
-      <td><p>The CDM version being used</p></td>
-    </tr>
-    </table>
-    
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>The performance of the existing model and prediction</p>
-    
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Implements an existing model and evaluates its performance</p>
-    
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-    <h2>Contents</h2>
-    <ul class="nav nav-pills nav-stacked">
-      <li><a href="#arguments">Arguments</a></li>
-      
-      <li><a href="#value">Value</a></li>
-
-      <li><a href="#details">Details</a></li>
-          </ul>
-
-  </div>
-</div>
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-   </div>
-
-  
-
-  </body>
-</html>
-
diff --git a/docs/reference/evaluateMultiplePlp.html b/docs/reference/evaluateMultiplePlp.html
deleted file mode 100644
index 63876ebea..000000000
--- a/docs/reference/evaluateMultiplePlp.html
+++ /dev/null
@@ -1,309 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>externally validate the multiple plp models across new datasets — evaluateMultiplePlp • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="externally validate the multiple plp models across new datasets — evaluateMultiplePlp" />
-<meta property="og:description" content="This function loads all the models in a multiple plp analysis folder and
-validates the models on new data" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>externally validate the multiple plp models across new datasets</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/RunMultiplePlp.R'><code>R/RunMultiplePlp.R</code></a></small>
-    <div class="hidden name"><code>evaluateMultiplePlp.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>This function loads all the models in a multiple plp analysis folder and
-validates the models on new data</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>evaluateMultiplePlp</span><span class='op'>(</span>
-  <span class='va'>analysesLocation</span>,
-  <span class='va'>outputLocation</span>,
-  <span class='va'>connectionDetails</span>,
-  <span class='va'>validationSchemaTarget</span>,
-  <span class='va'>validationSchemaOutcome</span>,
-  <span class='va'>validationSchemaCdm</span>,
-  <span class='va'>databaseNames</span>,
-  <span class='va'>validationTableTarget</span>,
-  <span class='va'>validationTableOutcome</span>,
-  validationIdTarget <span class='op'>=</span> <span class='cn'>NULL</span>,
-  validationIdOutcome <span class='op'>=</span> <span class='cn'>NULL</span>,
-  oracleTempSchema <span class='op'>=</span> <span class='cn'>NULL</span>,
-  verbosity <span class='op'>=</span> <span class='st'>"INFO"</span>,
-  keepPrediction <span class='op'>=</span> <span class='cn'>F</span>,
-  recalibrate <span class='op'>=</span> <span class='cn'>NULL</span>,
-  sampleSize <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>analysesLocation</th>
-      <td><p>The location where the multiple plp analyses are</p></td>
-    </tr>
-    <tr>
-      <th>outputLocation</th>
-      <td><p>The location to save to validation results</p></td>
-    </tr>
-    <tr>
-      <th>connectionDetails</th>
-      <td><p>The connection details for extracting the new data</p></td>
-    </tr>
-    <tr>
-      <th>validationSchemaTarget</th>
-      <td><p>A string or list of strings specifying the database containing the target cohorts</p></td>
-    </tr>
-    <tr>
-      <th>validationSchemaOutcome</th>
-      <td><p>A string or list of strings specifying the database containing the outcome cohorts</p></td>
-    </tr>
-    <tr>
-      <th>validationSchemaCdm</th>
-      <td><p>A string or list of strings specifying the database containing the cdm</p></td>
-    </tr>
-    <tr>
-      <th>databaseNames</th>
-      <td><p>A string of lift of strings specifying sharing friendly database names corresponding to validationSchemaCdm</p></td>
-    </tr>
-    <tr>
-      <th>validationTableTarget</th>
-      <td><p>A string or list of strings specifying the table containing the target cohorts</p></td>
-    </tr>
-    <tr>
-      <th>validationTableOutcome</th>
-      <td><p>A string or list of strings specifying the table containing the outcome cohorts</p></td>
-    </tr>
-    <tr>
-      <th>validationIdTarget</th>
-      <td><p>An iteger or list of integers specifying the cohort id for the target cohorts</p></td>
-    </tr>
-    <tr>
-      <th>validationIdOutcome</th>
-      <td><p>An iteger or list of integers specifying the cohort id for the outcome cohorts</p></td>
-    </tr>
-    <tr>
-      <th>oracleTempSchema</th>
-      <td><p>The temp oracle schema requires read/write</p></td>
-    </tr>
-    <tr>
-      <th>verbosity</th>
-      <td><p>Sets the level of the verbosity. If the log level is at or higher in priority than the logger threshold, a message will print. The levels are:</p><ul>
-<li><p>DEBUGHighest verbosity showing all debug statements</p></li>
-<li><p>TRACEShowing information about start and end of steps</p></li>
-<li><p>INFOShow informative information (Default)</p></li>
-<li><p>WARNShow warning messages</p></li>
-<li><p>ERRORShow error messages</p></li>
-<li><p>FATALBe silent except for fatal errors</p></li>
-</ul></td>
-    </tr>
-    <tr>
-      <th>keepPrediction</th>
-      <td><p>Whether to keep the predicitons for the new data</p></td>
-    </tr>
-    <tr>
-      <th>recalibrate</th>
-      <td><p>A vector of recalibration methods (currently supports 'RecalibrationintheLarge' and/or 'weakRecalibration')</p></td>
-    </tr>
-    <tr>
-      <th>sampleSize</th>
-      <td><p>If not NULL, the number of people to sample from the target cohort</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Users need to input a location where the results of the multiple plp analyses
-are found and the connection and database settings for the new data</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/evaluatePlp.html b/docs/reference/evaluatePlp.html
index 1060d323c..7ad688156 100644
--- a/docs/reference/evaluatePlp.html
+++ b/docs/reference/evaluatePlp.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>evaluatePlp — evaluatePlp • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="evaluatePlp — evaluatePlp" />
-<meta property="og:description" content="Evaluates the performance of the patient level prediction model" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>evaluatePlp — evaluatePlp • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="evaluatePlp — evaluatePlp"><meta property="og:description" content="Evaluates the performance of the patient level prediction model"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>evaluatePlp</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/EvaluatePlp.R'><code>R/EvaluatePlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/EvaluatePlp.R" class="external-link"><code>R/EvaluatePlp.R</code></a></small>
     <div class="hidden name"><code>evaluatePlp.Rd</code></div>
     </div>
 
@@ -183,54 +131,53 @@ <h1>evaluatePlp</h1>
     <p>Evaluates the performance of the patient level prediction model</p>
     </div>
 
-    <pre class="usage"><span class='fu'>evaluatePlp</span><span class='op'>(</span><span class='va'>prediction</span>, typeColumn <span class='op'>=</span> <span class='st'>"evaluationType"</span><span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">evaluatePlp</span><span class="op">(</span><span class="va">prediction</span>, typeColumn <span class="op">=</span> <span class="st">"evaluationType"</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>prediction</th>
-      <td><p>The patient level prediction model's prediction</p></td>
-    </tr>
-    <tr>
-      <th>typeColumn</th>
-      <td><p>The column name in the prediction object that is used to 
-stratify the evaluation</p></td>
-    </tr>
-    </table>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>prediction</dt>
+<dd><p>The patient level prediction model's prediction</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
 
-    <p>A list containing the performance values</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<dt>typeColumn</dt>
+<dd><p>The column name in the prediction object that is used to 
+stratify the evaluation</p></dd>
 
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A list containing the performance values</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>The function calculates various metrics to measure the performance of the model</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/exportPlpDataToCsv.html b/docs/reference/exportPlpDataToCsv.html
deleted file mode 100644
index 32eabfb24..000000000
--- a/docs/reference/exportPlpDataToCsv.html
+++ /dev/null
@@ -1,231 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Export all data in a plpData object to CSV files — exportPlpDataToCsv • PatientLevelPrediction</title>
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script>
-
-<!-- sticky kit -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-<meta property="og:title" content="Export all data in a plpData object to CSV files — exportPlpDataToCsv" />
-
-<meta property="og:description" content="Export all data in a plpData object to CSV files" />
-<meta name="twitter:card" content="summary" />
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-  </head>
-
-  <body>
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.5</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Export all data in a plpData object to CSV files</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ImportExport.R'><code>R/ImportExport.R</code></a></small>
-    <div class="hidden name"><code>exportPlpDataToCsv.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    
-    <p>Export all data in a plpData object to CSV files</p>
-    
-    </div>
-
-    <pre class="usage"><span class='fu'>exportPlpDataToCsv</span>(<span class='no'>plpData</span>, <span class='no'>outputFolder</span>)</pre>
-    
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpData</th>
-      <td><p>An object of type <code>plpData</code>.</p></td>
-    </tr>
-    <tr>
-      <th>outputFolder</th>
-      <td><p>The folder on the file system where the CSV files will be created. If the
-folder does not yet exist it will be created.</p></td>
-    </tr>
-    </table>
-    
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Created a set of CSV files in the output folder with all the data in the plplData object. This
-function is intended to be used for research into prediction methods. The following files will be
-created:</p><dl class='dl-horizontal'>
-<dt>cohort.csv</dt><dd><p>Listing all persons and their prediction periods. This file
-will have these fields: row_id (a unique ID per period), person_id, cohort_start_date, cohort_id,
-time (number of days in the window).</p></dd> <dt>outcomes.csv</dt><dd><p>Listing all outcomes per period. This
-file will have these fields: row_id, outcome_id, outcome_count, time_to_event.</p></dd>
-<dt>exclude.csv</dt><dd><p>Either not exported or a file listing per outcome ID which windows had the
-outcome prior to the window and should therefore be removed prior to fitting the model. This object
-will have these fields: rowId, outcomeId.</p></dd> <dt>covariates.csv</dt><dd><p>Listing the baseline covariates
-per person in the cohorts. This is done using a sparse representation: covariates with a value of 0
-are omitted to save space. The covariates file will have three columns: rowId, covariateId, and
-covariateValue.</p></dd> <dt>covariateRef.csv</dt><dd><p>A file describing the covariates that have been
-extracted.</p></dd> <dt>metaData</dt><dd><p>Some information on how the plpData object was constructed.</p></dd> </dl>
-    
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><span class='co'># NOT RUN {</span>
-<span class='fu'>exportPlpDataToCsv</span>(<span class='no'>plpData</span>, <span class='st'>"s:/temp/exportTest"</span>)
-<span class='co'># }</span></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-    <h2>Contents</h2>
-    <ul class="nav nav-pills nav-stacked">
-      <li><a href="#arguments">Arguments</a></li>
-      
-      <li><a href="#details">Details</a></li>
-      
-      <li><a href="#examples">Examples</a></li>
-    </ul>
-
-  </div>
-</div>
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-   </div>
-
-  
-
-  </body>
-</html>
-
diff --git a/docs/reference/externalValidateDbPlp.html b/docs/reference/externalValidateDbPlp.html
index 8c8dd4311..79f46b721 100644
--- a/docs/reference/externalValidateDbPlp.html
+++ b/docs/reference/externalValidateDbPlp.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>externalValidateDbPlp - Validate a model on new databases — externalValidateDbPlp • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="externalValidateDbPlp - Validate a model on new databases — externalValidateDbPlp" />
-<meta property="og:description" content="This function extracts data using a user specified connection and cdm_schema, applied the model and then calcualtes the performance" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>externalValidateDbPlp - Validate a model on new databases — externalValidateDbPlp • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="externalValidateDbPlp - Validate a model on new databases — externalValidateDbPlp"><meta property="og:description" content="This function extracts data using a user specified connection and cdm_schema, applied the model and then calcualtes the performance"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>externalValidateDbPlp - Validate a model on new databases</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ExternalValidatePlp.R'><code>R/ExternalValidatePlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ExternalValidatePlp.R" class="external-link"><code>R/ExternalValidatePlp.R</code></a></small>
     <div class="hidden name"><code>externalValidateDbPlp.Rd</code></div>
     </div>
 
@@ -183,78 +131,77 @@ <h1>externalValidateDbPlp - Validate a model on new databases</h1>
     <p>This function extracts data using a user specified connection and cdm_schema, applied the model and then calcualtes the performance</p>
     </div>
 
-    <pre class="usage"><span class='fu'>externalValidateDbPlp</span><span class='op'>(</span>
-  <span class='va'>plpModel</span>,
-  validationDatabaseDetails <span class='op'>=</span> <span class='fu'><a href='createDatabaseDetails.html'>createDatabaseDetails</a></span><span class='op'>(</span><span class='op'>)</span>,
-  validationRestrictPlpDataSettings <span class='op'>=</span> <span class='fu'><a href='createRestrictPlpDataSettings.html'>createRestrictPlpDataSettings</a></span><span class='op'>(</span><span class='op'>)</span>,
-  settings <span class='op'>=</span> <span class='fu'><a href='createValidationSettings.html'>createValidationSettings</a></span><span class='op'>(</span>recalibrate <span class='op'>=</span> <span class='st'>"weakRecalibration"</span><span class='op'>)</span>,
-  logSettings <span class='op'>=</span> <span class='fu'><a href='createLogSettings.html'>createLogSettings</a></span><span class='op'>(</span>verbosity <span class='op'>=</span> <span class='st'>"INFO"</span>, logName <span class='op'>=</span> <span class='st'>"validatePLP"</span><span class='op'>)</span>,
-  outputFolder <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/getwd.html'>getwd</a></span><span class='op'>(</span><span class='op'>)</span>
-<span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">externalValidateDbPlp</span><span class="op">(</span></span>
+<span>  <span class="va">plpModel</span>,</span>
+<span>  validationDatabaseDetails <span class="op">=</span> <span class="fu"><a href="createDatabaseDetails.html">createDatabaseDetails</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  validationRestrictPlpDataSettings <span class="op">=</span> <span class="fu"><a href="createRestrictPlpDataSettings.html">createRestrictPlpDataSettings</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  settings <span class="op">=</span> <span class="fu"><a href="createValidationSettings.html">createValidationSettings</a></span><span class="op">(</span>recalibrate <span class="op">=</span> <span class="st">"weakRecalibration"</span><span class="op">)</span>,</span>
+<span>  logSettings <span class="op">=</span> <span class="fu"><a href="createLogSettings.html">createLogSettings</a></span><span class="op">(</span>verbosity <span class="op">=</span> <span class="st">"INFO"</span>, logName <span class="op">=</span> <span class="st">"validatePLP"</span><span class="op">)</span>,</span>
+<span>  outputFolder <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpModel</dt>
+<dd><p>The model object returned by runPlp() containing the trained model</p></dd>
+
+
+<dt>validationDatabaseDetails</dt>
+<dd><p>A list of objects of class <code>databaseDetails</code> created using <code>createDatabaseDetails</code></p></dd>
+
+
+<dt>validationRestrictPlpDataSettings</dt>
+<dd><p>A list of population restriction settings created by <code><a href="createRestrictPlpDataSettings.html">createRestrictPlpDataSettings()</a></code></p></dd>
+
+
+<dt>settings</dt>
+<dd><p>A settings object of class <code>validationSettings</code> created using <code>createValidationSettings</code></p></dd>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpModel</th>
-      <td><p>The model object returned by runPlp() containing the trained model</p></td>
-    </tr>
-    <tr>
-      <th>validationDatabaseDetails</th>
-      <td><p>A list of objects of class <code>databaseDetails</code> created using <code>createDatabaseDetails</code></p></td>
-    </tr>
-    <tr>
-      <th>validationRestrictPlpDataSettings</th>
-      <td><p>A list of population restriction settings created by <code><a href='createRestrictPlpDataSettings.html'>createRestrictPlpDataSettings()</a></code></p></td>
-    </tr>
-    <tr>
-      <th>settings</th>
-      <td><p>A settings object of class <code>validationSettings</code> created using <code>createValidationSettings</code></p></td>
-    </tr>
-    <tr>
-      <th>logSettings</th>
-      <td><p>An object of <code>logSettings</code> created using <code>createLogSettings</code> 
-specifying how the logging is done</p></td>
-    </tr>
-    <tr>
-      <th>outputFolder</th>
-      <td><p>The directory to save the validation results to (subfolders are created per database in validationDatabaseDetails)</p></td>
-    </tr>
-    </table>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+<dt>logSettings</dt>
+<dd><p>An object of <code>logSettings</code> created using <code>createLogSettings</code>
+specifying how the logging is done</p></dd>
 
-    <p>A list containing the performance for each validation_schema</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
 
-    <p>Users need to input a trained model (the output of runPlp()) and new database connections. The function will return a list of length equal to the 
+<dt>outputFolder</dt>
+<dd><p>The directory to save the validation results to (subfolders are created per database in validationDatabaseDetails)</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A list containing the performance for each validation_schema</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>Users need to input a trained model (the output of runPlp()) and new database connections. The function will return a list of length equal to the
 number of cdm_schemas input with the performance on the new data</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/externalValidatePlp.html b/docs/reference/externalValidatePlp.html
deleted file mode 100644
index 51f1df281..000000000
--- a/docs/reference/externalValidatePlp.html
+++ /dev/null
@@ -1,310 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>externalValidatePlp - Validate a model on new databases — externalValidatePlp • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="externalValidatePlp - Validate a model on new databases — externalValidatePlp" />
-<meta property="og:description" content="This function extracts data using a user specified connection and cdm_schema, applied the model and then calcualtes the performance" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>externalValidatePlp - Validate a model on new databases</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ExternalValidatePlp.R'><code>R/ExternalValidatePlp.R</code></a></small>
-    <div class="hidden name"><code>externalValidatePlp.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>This function extracts data using a user specified connection and cdm_schema, applied the model and then calcualtes the performance</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>externalValidatePlp</span><span class='op'>(</span>
-  <span class='va'>plpResult</span>,
-  <span class='va'>connectionDetails</span>,
-  <span class='va'>validationSchemaTarget</span>,
-  <span class='va'>validationSchemaOutcome</span>,
-  <span class='va'>validationSchemaCdm</span>,
-  <span class='va'>databaseNames</span>,
-  validationTableTarget <span class='op'>=</span> <span class='st'>"cohort"</span>,
-  validationTableOutcome <span class='op'>=</span> <span class='st'>"cohort"</span>,
-  validationIdTarget <span class='op'>=</span> <span class='cn'>NULL</span>,
-  validationIdOutcome <span class='op'>=</span> <span class='cn'>NULL</span>,
-  oracleTempSchema <span class='op'>=</span> <span class='cn'>NULL</span>,
-  verbosity <span class='op'>=</span> <span class='st'>"INFO"</span>,
-  keepPrediction <span class='op'>=</span> <span class='cn'>F</span>,
-  recalibrate <span class='op'>=</span> <span class='cn'>NULL</span>,
-  sampleSize <span class='op'>=</span> <span class='cn'>NULL</span>,
-  <span class='va'>outputFolder</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpResult</th>
-      <td><p>The object returned by runPlp() containing the trained model</p></td>
-    </tr>
-    <tr>
-      <th>connectionDetails</th>
-      <td><p>The connection details for extracting the new data</p></td>
-    </tr>
-    <tr>
-      <th>validationSchemaTarget</th>
-      <td><p>A string or vector of strings specifying the database containing the target cohorts</p></td>
-    </tr>
-    <tr>
-      <th>validationSchemaOutcome</th>
-      <td><p>A string or vector of strings specifying the database containing the outcome cohorts</p></td>
-    </tr>
-    <tr>
-      <th>validationSchemaCdm</th>
-      <td><p>A string or vector of strings specifying the database containing the cdm</p></td>
-    </tr>
-    <tr>
-      <th>databaseNames</th>
-      <td><p>A string of vector of strings specifying sharing friendly database names corresponding to validationSchemaCdm</p></td>
-    </tr>
-    <tr>
-      <th>validationTableTarget</th>
-      <td><p>A string or vector of strings specifying the table containing the target cohorts</p></td>
-    </tr>
-    <tr>
-      <th>validationTableOutcome</th>
-      <td><p>A string or vector of strings specifying the table containing the outcome cohorts</p></td>
-    </tr>
-    <tr>
-      <th>validationIdTarget</th>
-      <td><p>An iteger specifying the cohort id for the target cohort</p></td>
-    </tr>
-    <tr>
-      <th>validationIdOutcome</th>
-      <td><p>An iteger specifying the cohort id for the outcome cohort</p></td>
-    </tr>
-    <tr>
-      <th>oracleTempSchema</th>
-      <td><p>The temp oracle schema requires read/write</p></td>
-    </tr>
-    <tr>
-      <th>verbosity</th>
-      <td><p>Sets the level of the verbosity. If the log level is at or higher in priority than the logger threshold, a message will print. The levels are:</p><ul>
-<li><p>DEBUGHighest verbosity showing all debug statements</p></li>
-<li><p>TRACEShowing information about start and end of steps</p></li>
-<li><p>INFOShow informative information (Default)</p></li>
-<li><p>WARNShow warning messages</p></li>
-<li><p>ERRORShow error messages</p></li>
-<li><p>FATALBe silent except for fatal errors</p></li>
-</ul></td>
-    </tr>
-    <tr>
-      <th>keepPrediction</th>
-      <td><p>Whether to keep the predicitons for the new data</p></td>
-    </tr>
-    <tr>
-      <th>recalibrate</th>
-      <td><p>A vector of characters specifying the recalibration method to apply</p></td>
-    </tr>
-    <tr>
-      <th>sampleSize</th>
-      <td><p>If not NULL, the number of people to sample from the target cohort</p></td>
-    </tr>
-    <tr>
-      <th>outputFolder</th>
-      <td><p>If you want to save the results enter the directory to save here</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A list containing the performance for each validation_schema</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Users need to input a trained model (the output of runPlp()) and new database connections. The function will return a list of length equal to the 
-number of cdm_schemas input with the performance on the new data</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/extractDatabaseToCsv.html b/docs/reference/extractDatabaseToCsv.html
new file mode 100644
index 000000000..8e90243ea
--- /dev/null
+++ b/docs/reference/extractDatabaseToCsv.html
@@ -0,0 +1,204 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Exports all the results from a database into csv files — extractDatabaseToCsv • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Exports all the results from a database into csv files — extractDatabaseToCsv"><meta property="og:description" content="Exports all the results from a database into csv files"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Exports all the results from a database into csv files</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SaveLoadPlp.R" class="external-link"><code>R/SaveLoadPlp.R</code></a></small>
+    <div class="hidden name"><code>extractDatabaseToCsv.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>Exports all the results from a database into csv files</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">extractDatabaseToCsv</span><span class="op">(</span></span>
+<span>  conn <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  <span class="va">connectionDetails</span>,</span>
+<span>  databaseSchemaSettings <span class="op">=</span> <span class="fu"><a href="createDatabaseSchemaSettings.html">createDatabaseSchemaSettings</a></span><span class="op">(</span>resultSchema <span class="op">=</span> <span class="st">"main"</span><span class="op">)</span>,</span>
+<span>  <span class="va">csvFolder</span>,</span>
+<span>  minCellCount <span class="op">=</span> <span class="fl">5</span>,</span>
+<span>  sensitiveColumns <span class="op">=</span> <span class="fu">getPlpSensitiveColumns</span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  fileAppend <span class="op">=</span> <span class="cn">NULL</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>conn</dt>
+<dd><p>The connection to the database with the results</p></dd>
+
+
+<dt>connectionDetails</dt>
+<dd><p>The connectionDetails for the result database</p></dd>
+
+
+<dt>databaseSchemaSettings</dt>
+<dd><p>The result database schema settings</p></dd>
+
+
+<dt>csvFolder</dt>
+<dd><p>Location to save the csv files</p></dd>
+
+
+<dt>minCellCount</dt>
+<dd><p>The min value to show in cells that are sensitive (values less than this value will be replaced with -1)</p></dd>
+
+
+<dt>sensitiveColumns</dt>
+<dd><p>A named list (name of table columns belong to) with a list of columns to apply the minCellCount to.</p></dd>
+
+
+<dt>fileAppend</dt>
+<dd><p>If set to a string this will be appended to the start of the csv file names</p></dd>
+
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>Extracts the results from a database into a set of csv files</p>
+    </div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/f1Score.html b/docs/reference/f1Score.html
index 2f25b115f..0a2fb3974 100644
--- a/docs/reference/f1Score.html
+++ b/docs/reference/f1Score.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculate the f1Score — f1Score • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculate the f1Score — f1Score" />
-<meta property="og:description" content="Calculate the f1Score" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculate the f1Score — f1Score • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculate the f1Score — f1Score"><meta property="og:description" content="Calculate the f1Score"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculate the f1Score</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ThresholdSummary.R'><code>R/ThresholdSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ThresholdSummary.R" class="external-link"><code>R/ThresholdSummary.R</code></a></small>
     <div class="hidden name"><code>f1Score.Rd</code></div>
     </div>
 
@@ -183,61 +131,60 @@ <h1>Calculate the f1Score</h1>
     <p>Calculate the f1Score</p>
     </div>
 
-    <pre class="usage"><span class='fu'>f1Score</span><span class='op'>(</span><span class='va'>TP</span>, <span class='va'>TN</span>, <span class='va'>FN</span>, <span class='va'>FP</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>TP</th>
-      <td><p>Number of true positives</p></td>
-    </tr>
-    <tr>
-      <th>TN</th>
-      <td><p>Number of true negatives</p></td>
-    </tr>
-    <tr>
-      <th>FN</th>
-      <td><p>Number of false negatives</p></td>
-    </tr>
-    <tr>
-      <th>FP</th>
-      <td><p>Number of false positives</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>f1Score value</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">f1Score</span><span class="op">(</span><span class="va">TP</span>, <span class="va">TN</span>, <span class="va">FN</span>, <span class="va">FP</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>TP</dt>
+<dd><p>Number of true positives</p></dd>
+
+
+<dt>TN</dt>
+<dd><p>Number of true negatives</p></dd>
+
+
+<dt>FN</dt>
+<dd><p>Number of false negatives</p></dd>
+
+
+<dt>FP</dt>
+<dd><p>Number of false positives</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>f1Score value</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculate the f1Score</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/falseDiscoveryRate.html b/docs/reference/falseDiscoveryRate.html
index 5b2a7df58..73f4dc46b 100644
--- a/docs/reference/falseDiscoveryRate.html
+++ b/docs/reference/falseDiscoveryRate.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculate the falseDiscoveryRate — falseDiscoveryRate • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculate the falseDiscoveryRate — falseDiscoveryRate" />
-<meta property="og:description" content="Calculate the falseDiscoveryRate" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculate the falseDiscoveryRate — falseDiscoveryRate • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculate the falseDiscoveryRate — falseDiscoveryRate"><meta property="og:description" content="Calculate the falseDiscoveryRate"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculate the falseDiscoveryRate</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ThresholdSummary.R'><code>R/ThresholdSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ThresholdSummary.R" class="external-link"><code>R/ThresholdSummary.R</code></a></small>
     <div class="hidden name"><code>falseDiscoveryRate.Rd</code></div>
     </div>
 
@@ -183,61 +131,60 @@ <h1>Calculate the falseDiscoveryRate</h1>
     <p>Calculate the falseDiscoveryRate</p>
     </div>
 
-    <pre class="usage"><span class='fu'>falseDiscoveryRate</span><span class='op'>(</span><span class='va'>TP</span>, <span class='va'>TN</span>, <span class='va'>FN</span>, <span class='va'>FP</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>TP</th>
-      <td><p>Number of true positives</p></td>
-    </tr>
-    <tr>
-      <th>TN</th>
-      <td><p>Number of true negatives</p></td>
-    </tr>
-    <tr>
-      <th>FN</th>
-      <td><p>Number of false negatives</p></td>
-    </tr>
-    <tr>
-      <th>FP</th>
-      <td><p>Number of false positives</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>falseDiscoveryRate value</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">falseDiscoveryRate</span><span class="op">(</span><span class="va">TP</span>, <span class="va">TN</span>, <span class="va">FN</span>, <span class="va">FP</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>TP</dt>
+<dd><p>Number of true positives</p></dd>
+
+
+<dt>TN</dt>
+<dd><p>Number of true negatives</p></dd>
+
+
+<dt>FN</dt>
+<dd><p>Number of false negatives</p></dd>
+
+
+<dt>FP</dt>
+<dd><p>Number of false positives</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>falseDiscoveryRate value</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculate the falseDiscoveryRate</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/falseNegativeRate.html b/docs/reference/falseNegativeRate.html
index b58d5a5ec..dc984b78f 100644
--- a/docs/reference/falseNegativeRate.html
+++ b/docs/reference/falseNegativeRate.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculate the falseNegativeRate — falseNegativeRate • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculate the falseNegativeRate — falseNegativeRate" />
-<meta property="og:description" content="Calculate the falseNegativeRate" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculate the falseNegativeRate — falseNegativeRate • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculate the falseNegativeRate — falseNegativeRate"><meta property="og:description" content="Calculate the falseNegativeRate"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculate the falseNegativeRate</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ThresholdSummary.R'><code>R/ThresholdSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ThresholdSummary.R" class="external-link"><code>R/ThresholdSummary.R</code></a></small>
     <div class="hidden name"><code>falseNegativeRate.Rd</code></div>
     </div>
 
@@ -183,61 +131,60 @@ <h1>Calculate the falseNegativeRate</h1>
     <p>Calculate the falseNegativeRate</p>
     </div>
 
-    <pre class="usage"><span class='fu'>falseNegativeRate</span><span class='op'>(</span><span class='va'>TP</span>, <span class='va'>TN</span>, <span class='va'>FN</span>, <span class='va'>FP</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>TP</th>
-      <td><p>Number of true positives</p></td>
-    </tr>
-    <tr>
-      <th>TN</th>
-      <td><p>Number of true negatives</p></td>
-    </tr>
-    <tr>
-      <th>FN</th>
-      <td><p>Number of false negatives</p></td>
-    </tr>
-    <tr>
-      <th>FP</th>
-      <td><p>Number of false positives</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>falseNegativeRate  value</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">falseNegativeRate</span><span class="op">(</span><span class="va">TP</span>, <span class="va">TN</span>, <span class="va">FN</span>, <span class="va">FP</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>TP</dt>
+<dd><p>Number of true positives</p></dd>
+
+
+<dt>TN</dt>
+<dd><p>Number of true negatives</p></dd>
+
+
+<dt>FN</dt>
+<dd><p>Number of false negatives</p></dd>
+
+
+<dt>FP</dt>
+<dd><p>Number of false positives</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>falseNegativeRate  value</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculate the falseNegativeRate</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/falseOmissionRate.html b/docs/reference/falseOmissionRate.html
index 254724bd1..7cf484107 100644
--- a/docs/reference/falseOmissionRate.html
+++ b/docs/reference/falseOmissionRate.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculate the falseOmissionRate — falseOmissionRate • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculate the falseOmissionRate — falseOmissionRate" />
-<meta property="og:description" content="Calculate the falseOmissionRate" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculate the falseOmissionRate — falseOmissionRate • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculate the falseOmissionRate — falseOmissionRate"><meta property="og:description" content="Calculate the falseOmissionRate"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculate the falseOmissionRate</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ThresholdSummary.R'><code>R/ThresholdSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ThresholdSummary.R" class="external-link"><code>R/ThresholdSummary.R</code></a></small>
     <div class="hidden name"><code>falseOmissionRate.Rd</code></div>
     </div>
 
@@ -183,61 +131,60 @@ <h1>Calculate the falseOmissionRate</h1>
     <p>Calculate the falseOmissionRate</p>
     </div>
 
-    <pre class="usage"><span class='fu'>falseOmissionRate</span><span class='op'>(</span><span class='va'>TP</span>, <span class='va'>TN</span>, <span class='va'>FN</span>, <span class='va'>FP</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>TP</th>
-      <td><p>Number of true positives</p></td>
-    </tr>
-    <tr>
-      <th>TN</th>
-      <td><p>Number of true negatives</p></td>
-    </tr>
-    <tr>
-      <th>FN</th>
-      <td><p>Number of false negatives</p></td>
-    </tr>
-    <tr>
-      <th>FP</th>
-      <td><p>Number of false positives</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>falseOmissionRate value</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">falseOmissionRate</span><span class="op">(</span><span class="va">TP</span>, <span class="va">TN</span>, <span class="va">FN</span>, <span class="va">FP</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>TP</dt>
+<dd><p>Number of true positives</p></dd>
+
+
+<dt>TN</dt>
+<dd><p>Number of true negatives</p></dd>
+
+
+<dt>FN</dt>
+<dd><p>Number of false negatives</p></dd>
+
+
+<dt>FP</dt>
+<dd><p>Number of false positives</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>falseOmissionRate value</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculate the falseOmissionRate</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/falsePositiveRate.html b/docs/reference/falsePositiveRate.html
index 8e25c27bc..0f378c3c8 100644
--- a/docs/reference/falsePositiveRate.html
+++ b/docs/reference/falsePositiveRate.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculate the falsePositiveRate — falsePositiveRate • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculate the falsePositiveRate — falsePositiveRate" />
-<meta property="og:description" content="Calculate the falsePositiveRate" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculate the falsePositiveRate — falsePositiveRate • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculate the falsePositiveRate — falsePositiveRate"><meta property="og:description" content="Calculate the falsePositiveRate"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculate the falsePositiveRate</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ThresholdSummary.R'><code>R/ThresholdSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ThresholdSummary.R" class="external-link"><code>R/ThresholdSummary.R</code></a></small>
     <div class="hidden name"><code>falsePositiveRate.Rd</code></div>
     </div>
 
@@ -183,61 +131,60 @@ <h1>Calculate the falsePositiveRate</h1>
     <p>Calculate the falsePositiveRate</p>
     </div>
 
-    <pre class="usage"><span class='fu'>falsePositiveRate</span><span class='op'>(</span><span class='va'>TP</span>, <span class='va'>TN</span>, <span class='va'>FN</span>, <span class='va'>FP</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>TP</th>
-      <td><p>Number of true positives</p></td>
-    </tr>
-    <tr>
-      <th>TN</th>
-      <td><p>Number of true negatives</p></td>
-    </tr>
-    <tr>
-      <th>FN</th>
-      <td><p>Number of false negatives</p></td>
-    </tr>
-    <tr>
-      <th>FP</th>
-      <td><p>Number of false positives</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>falsePositiveRate  value</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">falsePositiveRate</span><span class="op">(</span><span class="va">TP</span>, <span class="va">TN</span>, <span class="va">FN</span>, <span class="va">FP</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>TP</dt>
+<dd><p>Number of true positives</p></dd>
+
+
+<dt>TN</dt>
+<dd><p>Number of true negatives</p></dd>
+
+
+<dt>FN</dt>
+<dd><p>Number of false negatives</p></dd>
+
+
+<dt>FP</dt>
+<dd><p>Number of false positives</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>falsePositiveRate  value</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculate the falsePositiveRate</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/fitGLMModel.html b/docs/reference/fitGLMModel.html
deleted file mode 100644
index 69c0fd178..000000000
--- a/docs/reference/fitGLMModel.html
+++ /dev/null
@@ -1,258 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Fit a predictive model — fitGLMModel • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Fit a predictive model — fitGLMModel" />
-<meta property="og:description" content="Fit a predictive model" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Fit a predictive model</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/FitGLMModel.R'><code>R/FitGLMModel.R</code></a></small>
-    <div class="hidden name"><code>fitGLMModel.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Fit a predictive model</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>fitGLMModel</span><span class='op'>(</span>
-  <span class='va'>population</span>,
-  <span class='va'>plpData</span>,
-  modelType <span class='op'>=</span> <span class='st'>"logistic"</span>,
-  excludeCovariateIds <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='op'>)</span>,
-  includeCovariateIds <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='op'>)</span>,
-  prior <span class='op'>=</span> <span class='fu'>Cyclops</span><span class='fu'>::</span><span class='fu'><a href='https://rdrr.io/pkg/Cyclops/man/createPrior.html'>createPrior</a></span><span class='op'>(</span><span class='st'>"laplace"</span>, useCrossValidation <span class='op'>=</span> <span class='cn'>TRUE</span><span class='op'>)</span>,
-  control <span class='op'>=</span> <span class='fu'>Cyclops</span><span class='fu'>::</span><span class='fu'><a href='https://rdrr.io/pkg/Cyclops/man/createControl.html'>createControl</a></span><span class='op'>(</span>cvType <span class='op'>=</span> <span class='st'>"auto"</span>, fold <span class='op'>=</span> <span class='fl'>3</span>, startingVariance <span class='op'>=</span> <span class='fl'>0.01</span>,
-    lowerLimit <span class='op'>=</span> <span class='fl'>0.01</span>, upperLimit <span class='op'>=</span> <span class='fl'>20</span>, tolerance <span class='op'>=</span> <span class='fl'>2e-06</span>, cvRepetitions <span class='op'>=</span> <span class='fl'>1</span>,
-    selectorType <span class='op'>=</span> <span class='st'>"byPid"</span>, noiseLevel <span class='op'>=</span> <span class='st'>"silent"</span>, threads <span class='op'>=</span> <span class='op'>-</span><span class='fl'>1</span>, maxIterations <span class='op'>=</span> <span class='fl'>3000</span><span class='op'>)</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>population</th>
-      <td><p>A population object generated by <code>createStudyPopulation</code>, potentially filtered by other functions.</p></td>
-    </tr>
-    <tr>
-      <th>plpData</th>
-      <td><p>An object of type <code>plpData</code> as generated using
-<code>getPlpData</code>.</p></td>
-    </tr>
-    <tr>
-      <th>modelType</th>
-      <td><p>The type of outcome model that will be used. Possible values are
-"logistic", "poisson", or "cox".</p></td>
-    </tr>
-    <tr>
-      <th>excludeCovariateIds</th>
-      <td><p>Exclude these covariates from the outcome model.</p></td>
-    </tr>
-    <tr>
-      <th>includeCovariateIds</th>
-      <td><p>Include only these covariates in the outcome model.</p></td>
-    </tr>
-    <tr>
-      <th>prior</th>
-      <td><p>The prior used to fit the model. See
-<code><a href='https://rdrr.io/pkg/Cyclops/man/createPrior.html'>createPrior</a></code> for details.</p></td>
-    </tr>
-    <tr>
-      <th>control</th>
-      <td><p>The control object used to control the cross-validation used to
-determine the hyperparameters of the prior (if applicable). See
-<code><a href='https://rdrr.io/pkg/Cyclops/man/createControl.html'>createControl</a></code> for details.</p></td>
-    </tr>
-    </table>
-
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/fitPlp.html b/docs/reference/fitPlp.html
index c8d40ca24..6a394ecec 100644
--- a/docs/reference/fitPlp.html
+++ b/docs/reference/fitPlp.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>fitPlp — fitPlp • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="fitPlp — fitPlp" />
-<meta property="og:description" content="Train various models using a default parameter gird search or user specified parameters" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>fitPlp — fitPlp • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="fitPlp — fitPlp"><meta property="og:description" content="Train various models using a default parameter gird search or user specified parameters"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>fitPlp</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Fit.R'><code>R/Fit.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Fit.R" class="external-link"><code>R/Fit.R</code></a></small>
     <div class="hidden name"><code>fitPlp.Rd</code></div>
     </div>
 
@@ -183,77 +131,88 @@ <h1>fitPlp</h1>
     <p>Train various models using a default parameter gird search or user specified parameters</p>
     </div>
 
-    <pre class="usage"><span class='fu'>fitPlp</span><span class='op'>(</span><span class='va'>trainData</span>, <span class='va'>modelSettings</span>, search <span class='op'>=</span> <span class='st'>"grid"</span>, <span class='va'>analysisId</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>trainData</th>
-      <td><p>An object of type <code>TrainData</code> created using <code>splitData</code>
-data extracted from the CDM.</p></td>
-    </tr>
-    <tr>
-      <th>modelSettings</th>
-      <td><p>An object of class <code>modelSettings</code> created using one of the function:</p><ul>
-<li><p>logisticRegressionModel() A lasso logistic regression model</p></li>
-<li><p>GBMclassifier() A gradient boosting machine</p></li>
-<li><p>RFclassifier() A random forest model</p></li>
-<li><p>GLMclassifier () A generalised linear model</p></li>
-<li><p>KNNclassifier() A KNN model</p></li>
-</ul></td>
-    </tr>
-    <tr>
-      <th>search</th>
-      <td><p>The search strategy for the hyper-parameter selection (currently not used)</p></td>
-    </tr>
-    <tr>
-      <th>analysisId</th>
-      <td><p>The id of the analysis</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>An object of class <code>plpModel</code> containing:</p>
-<dt>model</dt><dd><p>The trained prediction model</p></dd>
-<dt>modelLoc</dt><dd><p>The path to where the model is saved (if saved)</p></dd>
-<dt>trainAuc</dt><dd><p>The AUC obtained on the training set</p></dd>
-<dt>trainCalibration</dt><dd><p>The calibration obtained on the training set</p></dd>
-<dt>modelSettings</dt><dd><p>A list specifiying the model, preprocessing, outcomeId and cohortId</p></dd>
-<dt>metaData</dt><dd><p>The model meta data</p></dd>
-<dt>trainingTime</dt><dd><p>The time taken to train the classifier</p></dd>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">fitPlp</span><span class="op">(</span><span class="va">trainData</span>, <span class="va">modelSettings</span>, search <span class="op">=</span> <span class="st">"grid"</span>, <span class="va">analysisId</span>, <span class="va">analysisPath</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>trainData</dt>
+<dd><p>An object of type <code>TrainData</code> created using <code>splitData</code>
+data extracted from the CDM.</p></dd>
+
+
+<dt>modelSettings</dt>
+<dd><p>An object of class <code>modelSettings</code> created using one of the function:</p><ul><li><p>setLassoLogisticRegression() A lasso logistic regression model</p></li>
+<li><p>setGradientBoostingMachine() A gradient boosting machine</p></li>
+<li><p>setRandomForest() A random forest model</p></li>
+<li><p>setKNN() A KNN model</p></li>
+</ul></dd>
+
+
+<dt>search</dt>
+<dd><p>The search strategy for the hyper-parameter selection (currently not used)</p></dd>
 
+
+<dt>analysisId</dt>
+<dd><p>The id of the analysis</p></dd>
+
+
+<dt>analysisPath</dt>
+<dd><p>The path of the analysis</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>An object of class <code>plpModel</code> containing:</p>
+<dl><dt>model</dt>
+<dd><p>The trained prediction model</p></dd>
+
+<dt>preprocessing</dt>
+<dd><p>The preprocessing required when applying the model</p></dd>
+
+<dt>prediction</dt>
+<dd><p>The cohort data.frame with the predicted risk column added</p></dd>
+
+<dt>modelDesign</dt>
+<dd><p>A list specifiying the modelDesign settings used to fit the model</p></dd>
+
+<dt>trainDetails</dt>
+<dd><p>The model meta data</p></dd>
+
+<dt>covariateImportance</dt>
+<dd><p>The covariate importance for the model</p></dd>
+
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>The user can define the machine learning model to train (regularised logistic regression, random forest,
 gradient boosting machine, neural network and )</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/getAttritionTable.html b/docs/reference/getAttritionTable.html
deleted file mode 100644
index 34c3b36c1..000000000
--- a/docs/reference/getAttritionTable.html
+++ /dev/null
@@ -1,217 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Get the attrition table for a population — getAttritionTable • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Get the attrition table for a population — getAttritionTable" />
-<meta property="og:description" content="Get the attrition table for a population" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.0.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Get the attrition table for a population</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/StudyPopulation.R'><code>R/StudyPopulation.R</code></a></small>
-    <div class="hidden name"><code>getAttritionTable.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Get the attrition table for a population</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>getAttritionTable</span>(<span class='no'>object</span>)</pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>object</th>
-      <td><p>Either an object of type <code>plpData</code>, a population object generated by functions
-like <code>createStudyPopulation</code>, or an object of type <code>outcomeModel</code>.</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A data frame specifying the number of people and exposures in the population after specific steps of filtering.</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.5.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/getCalibration.html b/docs/reference/getCalibration.html
deleted file mode 100644
index 938f1c319..000000000
--- a/docs/reference/getCalibration.html
+++ /dev/null
@@ -1,236 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Get a sparse summary of the calibration — getCalibration • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Get a sparse summary of the calibration — getCalibration" />
-<meta property="og:description" content="Get a sparse summary of the calibration" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Get a sparse summary of the calibration</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/EvaluatePlp.R'><code>R/EvaluatePlp.R</code></a></small>
-    <div class="hidden name"><code>getCalibration.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Get a sparse summary of the calibration</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>getCalibration</span><span class='op'>(</span><span class='va'>prediction</span>, numberOfStrata <span class='op'>=</span> <span class='fl'>10</span>, truncateFraction <span class='op'>=</span> <span class='fl'>0.01</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>prediction</th>
-      <td><p>A prediction object as generated using the
-<code><a href='https://rdrr.io/r/stats/predict.html'>predict</a></code> functions.</p></td>
-    </tr>
-    <tr>
-      <th>numberOfStrata</th>
-      <td><p>The number of strata in the plot.</p></td>
-    </tr>
-    <tr>
-      <th>truncateFraction</th>
-      <td><p>This fraction of probability values will be ignored when plotting, to
-avoid the x-axis scale being dominated by a few outliers.</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A dataframe with the calibration summary</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Generates a sparse summary showing the predicted probabilities and the observed fractions. Predictions are
-stratefied into equally sized bins of predicted probabilities.</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/getCalibrationSummary.html b/docs/reference/getCalibrationSummary.html
index 03ff5060e..f32e764d0 100644
--- a/docs/reference/getCalibrationSummary.html
+++ b/docs/reference/getCalibrationSummary.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Get a sparse summary of the calibration — getCalibrationSummary • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Get a sparse summary of the calibration — getCalibrationSummary" />
-<meta property="og:description" content="Get a sparse summary of the calibration" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Get a sparse summary of the calibration — getCalibrationSummary • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Get a sparse summary of the calibration — getCalibrationSummary"><meta property="og:description" content="Get a sparse summary of the calibration"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Get a sparse summary of the calibration</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/CalibrationSummary.R'><code>R/CalibrationSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/CalibrationSummary.R" class="external-link"><code>R/CalibrationSummary.R</code></a></small>
     <div class="hidden name"><code>getCalibrationSummary.Rd</code></div>
     </div>
 
@@ -183,74 +131,73 @@ <h1>Get a sparse summary of the calibration</h1>
     <p>Get a sparse summary of the calibration</p>
     </div>
 
-    <pre class="usage"><span class='fu'>getCalibrationSummary</span><span class='op'>(</span>
-  <span class='va'>prediction</span>,
-  <span class='va'>predictionType</span>,
-  typeColumn <span class='op'>=</span> <span class='st'>"evaluation"</span>,
-  numberOfStrata <span class='op'>=</span> <span class='fl'>100</span>,
-  truncateFraction <span class='op'>=</span> <span class='fl'>0.05</span>
-<span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">getCalibrationSummary</span><span class="op">(</span></span>
+<span>  <span class="va">prediction</span>,</span>
+<span>  <span class="va">predictionType</span>,</span>
+<span>  typeColumn <span class="op">=</span> <span class="st">"evaluation"</span>,</span>
+<span>  numberOfStrata <span class="op">=</span> <span class="fl">100</span>,</span>
+<span>  truncateFraction <span class="op">=</span> <span class="fl">0.05</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>prediction</dt>
+<dd><p>A prediction object as generated using the
+<code><a href="https://rdrr.io/r/stats/predict.html" class="external-link">predict</a></code> functions.</p></dd>
+
+
+<dt>predictionType</dt>
+<dd><p>The type of prediction (binary or survival)</p></dd>
+
+
+<dt>typeColumn</dt>
+<dd><p>A column that is used to stratify the results</p></dd>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>prediction</th>
-      <td><p>A prediction object as generated using the
-<code><a href='https://rdrr.io/r/stats/predict.html'>predict</a></code> functions.</p></td>
-    </tr>
-    <tr>
-      <th>predictionType</th>
-      <td><p>The type of prediction (binary or survival)</p></td>
-    </tr>
-    <tr>
-      <th>typeColumn</th>
-      <td><p>A column that is used to stratify the results</p></td>
-    </tr>
-    <tr>
-      <th>numberOfStrata</th>
-      <td><p>The number of strata in the plot.</p></td>
-    </tr>
-    <tr>
-      <th>truncateFraction</th>
-      <td><p>This fraction of probability values will be ignored when plotting, to
-avoid the x-axis scale being dominated by a few outliers.</p></td>
-    </tr>
-    </table>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+<dt>numberOfStrata</dt>
+<dd><p>The number of strata in the plot.</p></dd>
 
-    <p>A dataframe with the calibration summary</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
 
+<dt>truncateFraction</dt>
+<dd><p>This fraction of probability values will be ignored when plotting, to
+avoid the x-axis scale being dominated by a few outliers.</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A dataframe with the calibration summary</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Generates a sparse summary showing the predicted probabilities and the observed fractions. Predictions are
 stratefied into equally sized bins of predicted probabilities.</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/getCohortCovariateData.html b/docs/reference/getCohortCovariateData.html
index 43ae2546e..b5dcda0a1 100644
--- a/docs/reference/getCohortCovariateData.html
+++ b/docs/reference/getCohortCovariateData.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Extracts covariates based on cohorts — getCohortCovariateData • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Extracts covariates based on cohorts — getCohortCovariateData" />
-<meta property="og:description" content="Extracts covariates based on cohorts" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Extracts covariates based on cohorts — getCohortCovariateData • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Extracts covariates based on cohorts — getCohortCovariateData"><meta property="og:description" content="Extracts covariates based on cohorts"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Extracts covariates based on cohorts</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/AdditionalCovariates.R'><code>R/AdditionalCovariates.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/AdditionalCovariates.R" class="external-link"><code>R/AdditionalCovariates.R</code></a></small>
     <div class="hidden name"><code>getCohortCovariateData.Rd</code></div>
     </div>
 
@@ -183,92 +131,96 @@ <h1>Extracts covariates based on cohorts</h1>
     <p>Extracts covariates based on cohorts</p>
     </div>
 
-    <pre class="usage"><span class='fu'>getCohortCovariateData</span><span class='op'>(</span>
-  <span class='va'>connection</span>,
-  oracleTempSchema <span class='op'>=</span> <span class='cn'>NULL</span>,
-  <span class='va'>cdmDatabaseSchema</span>,
-  cdmVersion <span class='op'>=</span> <span class='st'>"5"</span>,
-  cohortTable <span class='op'>=</span> <span class='st'>"#cohort_person"</span>,
-  rowIdField <span class='op'>=</span> <span class='st'>"row_id"</span>,
-  <span class='va'>aggregated</span>,
-  <span class='va'>cohortId</span>,
-  <span class='va'>covariateSettings</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>connection</th>
-      <td><p>The database connection</p></td>
-    </tr>
-    <tr>
-      <th>oracleTempSchema</th>
-      <td><p>The temp schema if using oracle</p></td>
-    </tr>
-    <tr>
-      <th>cdmDatabaseSchema</th>
-      <td><p>The schema of the OMOP CDM data</p></td>
-    </tr>
-    <tr>
-      <th>cdmVersion</th>
-      <td><p>version of the OMOP CDM data</p></td>
-    </tr>
-    <tr>
-      <th>cohortTable</th>
-      <td><p>the table name that contains the target population cohort</p></td>
-    </tr>
-    <tr>
-      <th>rowIdField</th>
-      <td><p>string representing the unique identifier in the target population cohort</p></td>
-    </tr>
-    <tr>
-      <th>aggregated</th>
-      <td><p>whether the covariate should be aggregated</p></td>
-    </tr>
-    <tr>
-      <th>cohortId</th>
-      <td><p>cohort id for the target population cohort</p></td>
-    </tr>
-    <tr>
-      <th>covariateSettings</th>
-      <td><p>settings for the covariate cohorts and time periods</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>The models will now be in the package</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">getCohortCovariateData</span><span class="op">(</span></span>
+<span>  <span class="va">connection</span>,</span>
+<span>  oracleTempSchema <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  <span class="va">cdmDatabaseSchema</span>,</span>
+<span>  cdmVersion <span class="op">=</span> <span class="st">"5"</span>,</span>
+<span>  cohortTable <span class="op">=</span> <span class="st">"#cohort_person"</span>,</span>
+<span>  rowIdField <span class="op">=</span> <span class="st">"row_id"</span>,</span>
+<span>  <span class="va">aggregated</span>,</span>
+<span>  <span class="va">cohortIds</span>,</span>
+<span>  <span class="va">covariateSettings</span>,</span>
+<span>  <span class="va">...</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>connection</dt>
+<dd><p>The database connection</p></dd>
+
+
+<dt>oracleTempSchema</dt>
+<dd><p>The temp schema if using oracle</p></dd>
+
+
+<dt>cdmDatabaseSchema</dt>
+<dd><p>The schema of the OMOP CDM data</p></dd>
+
+
+<dt>cdmVersion</dt>
+<dd><p>version of the OMOP CDM data</p></dd>
+
+
+<dt>cohortTable</dt>
+<dd><p>the table name that contains the target population cohort</p></dd>
 
+
+<dt>rowIdField</dt>
+<dd><p>string representing the unique identifier in the target population cohort</p></dd>
+
+
+<dt>aggregated</dt>
+<dd><p>whether the covariate should be aggregated</p></dd>
+
+
+<dt>cohortIds</dt>
+<dd><p>cohort id for the target cohort</p></dd>
+
+
+<dt>covariateSettings</dt>
+<dd><p>settings for the covariate cohorts and time periods</p></dd>
+
+
+<dt>...</dt>
+<dd><p>additional arguments from FeatureExtraction</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>The models will now be in the package</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>The user specifies a cohort and time period and then a covariate is constructed whether they are in the
 cohort during the time periods relative to target population cohort index</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/getCovariateData.html b/docs/reference/getCovariateData.html
deleted file mode 100644
index e7ab32589..000000000
--- a/docs/reference/getCovariateData.html
+++ /dev/null
@@ -1,242 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Get the covaridate data for a cohort table — getCovariateData • PatientLevelPrediction</title>
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script>
-
-<!-- sticky kit -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-<meta property="og:title" content="Get the covaridate data for a cohort table — getCovariateData" />
-
-<meta property="og:description" content="This function executes some SQL to extract covaraite data for a cohort table" />
-<meta name="twitter:card" content="summary" />
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-  </head>
-
-  <body>
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.5</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Get the covaridate data for a cohort table</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SaveLoadPlp.R'><code>R/SaveLoadPlp.R</code></a></small>
-    <div class="hidden name"><code>getCovariateData.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    
-    <p>This function executes some SQL to extract covaraite data for a cohort table</p>
-    
-    </div>
-
-    <pre class="usage"><span class='fu'>getCovariateData</span>(<span class='no'>connection</span>, <span class='no'>cdmDatabaseSchema</span>,
-  <span class='kw'>oracleTempSchema</span> <span class='kw'>=</span> <span class='no'>cdmDatabaseSchema</span>, <span class='kw'>cohortTable</span> <span class='kw'>=</span> <span class='st'>"#cohort_person"</span>,
-  <span class='kw'>cdmVersion</span> <span class='kw'>=</span> <span class='fl'>5</span>, <span class='no'>covariateSettings</span>)</pre>
-    
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>connection</th>
-      <td><p>Can also use an existing connection rather than the connectionDetails</p></td>
-    </tr>
-    <tr>
-      <th>cdmDatabaseSchema</th>
-      <td><p>The name of the database schema that contains the OMOP CDM
-instance.  Requires read permissions to this database. On SQL
-Server, this should specifiy both the database and the schema,
-so for example 'cdm_instance.dbo'.</p></td>
-    </tr>
-    <tr>
-      <th>oracleTempSchema</th>
-      <td><p>For Oracle only: the name of the database schema where you want
-all temporary tables to be managed. Requires create/insert
-permissions to this database.</p></td>
-    </tr>
-    <tr>
-      <th>cohortTable</th>
-      <td><p>The temp table containing the cohort of people</p></td>
-    </tr>
-    <tr>
-      <th>cdmVersion</th>
-      <td><p>The version of the CDM (default 5)</p></td>
-    </tr>
-    <tr>
-      <th>covariateSettings</th>
-      <td><p>An object of type <code>covariateSettings</code> as created using the
-<code>createCovariateSettings</code> function in the
-<code>FeatureExtraction</code> package.</p></td>
-    </tr>
-    </table>
-    
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>Returns the covariates for the people in the temp table</p>
-    
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    
-    
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-    <h2>Contents</h2>
-    <ul class="nav nav-pills nav-stacked">
-      <li><a href="#arguments">Arguments</a></li>
-      
-      <li><a href="#value">Value</a></li>
-
-      <li><a href="#details">Details</a></li>
-          </ul>
-
-  </div>
-</div>
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-   </div>
-
-  
-
-  </body>
-</html>
-
diff --git a/docs/reference/getDemographicSummary.html b/docs/reference/getDemographicSummary.html
index fc69358f0..a4ffe6cdc 100644
--- a/docs/reference/getDemographicSummary.html
+++ b/docs/reference/getDemographicSummary.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Get a calibration per age/gender groups — getDemographicSummary • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Get a calibration per age/gender groups — getDemographicSummary" />
-<meta property="og:description" content="Get a calibration per age/gender groups" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Get a calibration per age/gender groups — getDemographicSummary • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Get a calibration per age/gender groups — getDemographicSummary"><meta property="og:description" content="Get a calibration per age/gender groups"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Get a calibration per age/gender groups</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/DemographicSummary.R'><code>R/DemographicSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/DemographicSummary.R" class="external-link"><code>R/DemographicSummary.R</code></a></small>
     <div class="hidden name"><code>getDemographicSummary.Rd</code></div>
     </div>
 
@@ -183,57 +131,56 @@ <h1>Get a calibration per age/gender groups</h1>
     <p>Get a calibration per age/gender groups</p>
     </div>
 
-    <pre class="usage"><span class='fu'>getDemographicSummary</span><span class='op'>(</span><span class='va'>prediction</span>, <span class='va'>predictionType</span>, typeColumn <span class='op'>=</span> <span class='st'>"evaluation"</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>prediction</th>
-      <td><p>A prediction object</p></td>
-    </tr>
-    <tr>
-      <th>predictionType</th>
-      <td><p>The type of prediction (binary or survival)</p></td>
-    </tr>
-    <tr>
-      <th>typeColumn</th>
-      <td><p>A column that is used to stratify the results</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A dataframe with the calibration summary</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">getDemographicSummary</span><span class="op">(</span><span class="va">prediction</span>, <span class="va">predictionType</span>, typeColumn <span class="op">=</span> <span class="st">"evaluation"</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>prediction</dt>
+<dd><p>A prediction object</p></dd>
+
+
+<dt>predictionType</dt>
+<dd><p>The type of prediction (binary or survival)</p></dd>
+
 
+<dt>typeColumn</dt>
+<dd><p>A column that is used to stratify the results</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A dataframe with the calibration summary</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Generates a data.frame with the calibration per each 5 year age group and gender group</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/getModelDetails.html b/docs/reference/getModelDetails.html
deleted file mode 100644
index 6d2e4a0cc..000000000
--- a/docs/reference/getModelDetails.html
+++ /dev/null
@@ -1,215 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Get the predictive model details — getModelDetails • PatientLevelPrediction</title>
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script>
-
-<!-- sticky kit -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-<meta property="og:title" content="Get the predictive model details — getModelDetails" />
-
-<meta property="og:description" content="getModelDetails shows the full model, so showing the betas of all variables included in the
-model, along with the variable names" />
-<meta name="twitter:card" content="summary" />
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-  </head>
-
-  <body>
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.5</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Get the predictive model details</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/FitGLMModel.R'><code>R/FitGLMModel.R</code></a></small>
-    <div class="hidden name"><code>getModelDetails.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    
-    <p><code>getModelDetails</code> shows the full model, so showing the betas of all variables included in the
-model, along with the variable names</p>
-    
-    </div>
-
-    <pre class="usage"><span class='fu'>getModelDetails</span>(<span class='no'>predictiveModel</span>, <span class='no'>plpData</span>)</pre>
-    
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>predictiveModel</th>
-      <td><p>An object of type <code>predictiveModel</code> as generated using he
-<code><a href='fitPlp.html'>fitPlp</a></code> function.</p></td>
-    </tr>
-    <tr>
-      <th>plpData</th>
-      <td><p>An object of type <code>plpData</code> as generated using
-<code><a href='getPlpData.html'>getPlpData</a></code>.</p></td>
-    </tr>
-    </table>
-    
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Shows the coefficients and names of the covariates with non-zero coefficients.</p>
-    
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-    <h2>Contents</h2>
-    <ul class="nav nav-pills nav-stacked">
-      <li><a href="#arguments">Arguments</a></li>
-      
-      <li><a href="#details">Details</a></li>
-          </ul>
-
-  </div>
-</div>
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-   </div>
-
-  
-
-  </body>
-</html>
-
diff --git a/docs/reference/getPlpData.html b/docs/reference/getPlpData.html
index 9464df6ad..88c691ae6 100644
--- a/docs/reference/getPlpData.html
+++ b/docs/reference/getPlpData.html
@@ -1,68 +1,13 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Get the patient level prediction data from the server — getPlpData • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Get the patient level prediction data from the server — getPlpData" />
-<meta property="og:description" content="This function executes a large set of SQL statements against the database in OMOP CDM format to
-extract the data needed to perform the analysis." />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Get the patient level prediction data from the server — getPlpData • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Get the patient level prediction data from the server — getPlpData"><meta property="og:description" content="This function executes a large set of SQL statements against the database in OMOP CDM format to
+extract the data needed to perform the analysis."><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -73,13 +18,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -95,13 +39,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -114,10 +57,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -126,13 +69,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -140,43 +86,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Get the patient level prediction data from the server</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ExtractData.R'><code>R/ExtractData.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ExtractData.R" class="external-link"><code>R/ExtractData.R</code></a></small>
     <div class="hidden name"><code>getPlpData.Rd</code></div>
     </div>
 
@@ -185,44 +133,55 @@ <h1>Get the patient level prediction data from the server</h1>
 extract the data needed to perform the analysis.</p>
     </div>
 
-    <pre class="usage"><span class='fu'>getPlpData</span><span class='op'>(</span><span class='va'>databaseDetails</span>, <span class='va'>covariateSettings</span>, <span class='va'>restrictPlpDataSettings</span><span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">getPlpData</span><span class="op">(</span><span class="va">databaseDetails</span>, <span class="va">covariateSettings</span>, <span class="va">restrictPlpDataSettings</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>databaseDetails</dt>
+<dd><p>The cdm database details created using <code><a href="createDatabaseDetails.html">createDatabaseDetails()</a></code></p></dd>
+
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>databaseDetails</th>
-      <td><p>The cdm database details created using <code><a href='createDatabaseDetails.html'>createDatabaseDetails()</a></code></p></td>
-    </tr>
-    <tr>
-      <th>covariateSettings</th>
-      <td><p>An object of type <code>covariateSettings</code> as created using the
+<dt>covariateSettings</dt>
+<dd><p>An object of type <code>covariateSettings</code> as created using the
 <code>createCovariateSettings</code> function in the
-<code>FeatureExtraction</code> package.</p></td>
-    </tr>
-    <tr>
-      <th>restrictPlpDataSettings</th>
-      <td><p>Extra settings to apply to the target population while extracting data.  Created using <code><a href='createRestrictPlpDataSettings.html'>createRestrictPlpDataSettings()</a></code>.</p></td>
-    </tr>
-    </table>
+<code>FeatureExtraction</code> package.</p></dd>
+
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+<dt>restrictPlpDataSettings</dt>
+<dd><p>Extra settings to apply to the target population while extracting data.  Created using <code><a href="createRestrictPlpDataSettings.html">createRestrictPlpDataSettings()</a></code>.</p></dd>
 
-    <p>Returns an object of type <code>plpData</code>, containing information on the cohorts, their
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>Returns an object of type <code>plpData</code>, containing information on the cohorts, their
 outcomes, and baseline covariates. Information about multiple outcomes can be captured at once for
-efficiency reasons. This object is a list with the following components:</p><dl>
-<dt>outcomes</dt><dd><p>A data frame listing the outcomes per person, including the time to event, and
+efficiency reasons. This object is a list with the following components:</p><dl><dt>outcomes</dt>
+<dd><p>A data frame listing the outcomes per person, including the time to event, and
 the outcome id. Outcomes are not yet filtered based on risk window, since this is done at
-a later stage.</p></dd> <dt>cohorts</dt><dd><p>A data frame listing the persons in each cohort, listing their
+a later stage.</p></dd>
+ <dt>cohorts</dt>
+<dd><p>A data frame listing the persons in each cohort, listing their
 exposure status as well as the time to the end of the observation period and time to the end of the
-cohort (usually the end of the exposure era).</p></dd> <dt>covariates</dt><dd><p>An ffdf object listing the
+cohort (usually the end of the exposure era).</p></dd>
+ <dt>covariates</dt>
+<dd><p>An ffdf object listing the
 baseline covariates per person in the two cohorts. This is done using a sparse representation:
-covariates with a value of 0 are omitted to save space.</p></dd> <dt>covariateRef</dt><dd><p>An ffdf object describing the covariates that have been extracted.</p></dd>
-<dt>metaData</dt><dd><p>A list of objects with information on how the cohortMethodData object was
-constructed.</p></dd> 
-</dl><p>The generic <code>()</code> and <code><a href='https://rdrr.io/r/base/summary.html'>summary()</a></code> functions have been implemented for this object.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
+covariates with a value of 0 are omitted to save space.</p></dd>
+ <dt>covariateRef</dt>
+<dd><p>An ffdf object describing the covariates that have been extracted.</p></dd>
+
+<dt>metaData</dt>
+<dd><p>A list of objects with information on how the cohortMethodData object was
+constructed.</p></dd>
+ 
+</dl><p>The generic <code>()</code> and <code><a href="https://rdrr.io/r/base/summary.html" class="external-link">summary()</a></code> functions have been implemented for this object.</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Based on the arguments, the at risk cohort data is retrieved, as well as outcomes
 occurring in these subjects. The at risk cohort is identified  through
 user-defined cohorts in a cohort table either inside the CDM instance or in a separate schema.
@@ -232,32 +191,29 @@ <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details
 If you wish to exclude concepts from covariates you will need to
 manually add the concept_ids and descendants to the <code>excludedCovariateConceptIds</code> of the
 <code>covariateSettings</code> argument.</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/getPlpTable.html b/docs/reference/getPlpTable.html
deleted file mode 100644
index 92a51f7b7..000000000
--- a/docs/reference/getPlpTable.html
+++ /dev/null
@@ -1,259 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create a dataframe with the summary details of the population cohort for publications — getPlpTable • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create a dataframe with the summary details of the population cohort for publications — getPlpTable" />
-<meta property="og:description" content="Create a dataframe with the summary details of the population cohort for publications" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Create a dataframe with the summary details of the population cohort for publications</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/CreatePlpDocument.R'><code>R/CreatePlpDocument.R</code></a></small>
-    <div class="hidden name"><code>getPlpTable.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Create a dataframe with the summary details of the population cohort for publications</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>getPlpTable</span><span class='op'>(</span>
-  <span class='va'>cdmDatabaseSchema</span>,
-  <span class='va'>oracleTempSchema</span>,
-  <span class='va'>covariateSettings</span>,
-  longTermStartDays <span class='op'>=</span> <span class='op'>-</span><span class='fl'>365</span>,
-  <span class='va'>population</span>,
-  <span class='va'>connectionDetails</span>,
-  cohortTable <span class='op'>=</span> <span class='st'>"#temp_person"</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>cdmDatabaseSchema</th>
-      <td><p>The schema containing the OMOP CDM data</p></td>
-    </tr>
-    <tr>
-      <th>oracleTempSchema</th>
-      <td><p>The oracle schema if needed</p></td>
-    </tr>
-    <tr>
-      <th>covariateSettings</th>
-      <td><p>The covariateSettings if different from default</p></td>
-    </tr>
-    <tr>
-      <th>longTermStartDays</th>
-      <td><p>How far to look back when looking for the variables in the data</p></td>
-    </tr>
-    <tr>
-      <th>population</th>
-      <td><p>The population you want the summary table for</p></td>
-    </tr>
-    <tr>
-      <th>connectionDetails</th>
-      <td><p>The connection details used to connect to the CDM database</p></td>
-    </tr>
-    <tr>
-      <th>cohortTable</th>
-      <td><p>The name of the temp table that will store the popualtion cohort</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>This function is used to create a summary table for population to be inserted into publications</p>
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='fu'>getTable1</span> <span class='op'>(</span><span class='va'>plpData</span>, <span class='va'>population</span>, <span class='va'>connectionDetails</span><span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/getPredictionDistribution.html b/docs/reference/getPredictionDistribution.html
index e2a3887d6..f6d9eac2f 100644
--- a/docs/reference/getPredictionDistribution.html
+++ b/docs/reference/getPredictionDistribution.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculates the prediction distribution — getPredictionDistribution • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculates the prediction distribution — getPredictionDistribution" />
-<meta property="og:description" content="Calculates the prediction distribution" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculates the prediction distribution — getPredictionDistribution • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculates the prediction distribution — getPredictionDistribution"><meta property="og:description" content="Calculates the prediction distribution"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculates the prediction distribution</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/PredictionDistribution.R'><code>R/PredictionDistribution.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/PredictionDistribution.R" class="external-link"><code>R/PredictionDistribution.R</code></a></small>
     <div class="hidden name"><code>getPredictionDistribution.Rd</code></div>
     </div>
 
@@ -183,62 +131,61 @@ <h1>Calculates the prediction distribution</h1>
     <p>Calculates the prediction distribution</p>
     </div>
 
-    <pre class="usage"><span class='fu'>getPredictionDistribution</span><span class='op'>(</span>
-  <span class='va'>prediction</span>,
-  <span class='va'>predictionType</span>,
-  typeColumn <span class='op'>=</span> <span class='st'>"evaluation"</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>prediction</th>
-      <td><p>A prediction object</p></td>
-    </tr>
-    <tr>
-      <th>predictionType</th>
-      <td><p>The type of prediction (binary or survival)</p></td>
-    </tr>
-    <tr>
-      <th>typeColumn</th>
-      <td><p>A column that is used to stratify the results</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>The 0.00, 0.1, 0.25, 0.5, 0.75, 0.9, 1.00 quantile pf the prediction,
-the mean and standard deviation per class</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">getPredictionDistribution</span><span class="op">(</span></span>
+<span>  <span class="va">prediction</span>,</span>
+<span>  <span class="va">predictionType</span>,</span>
+<span>  typeColumn <span class="op">=</span> <span class="st">"evaluation"</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>prediction</dt>
+<dd><p>A prediction object</p></dd>
+
+
+<dt>predictionType</dt>
+<dd><p>The type of prediction (binary or survival)</p></dd>
+
+
+<dt>typeColumn</dt>
+<dd><p>A column that is used to stratify the results</p></dd>
 
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>The 0.00, 0.1, 0.25, 0.5, 0.75, 0.9, 1.00 quantile pf the prediction,
+the mean and standard deviation per class</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculates the quantiles from a predition object</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/getPredictionDistribution_binary.html b/docs/reference/getPredictionDistribution_binary.html
index 9d4ccbde5..25312a70d 100644
--- a/docs/reference/getPredictionDistribution_binary.html
+++ b/docs/reference/getPredictionDistribution_binary.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculates the prediction distribution — getPredictionDistribution_binary • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculates the prediction distribution — getPredictionDistribution_binary" />
-<meta property="og:description" content="Calculates the prediction distribution" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculates the prediction distribution — getPredictionDistribution_binary • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculates the prediction distribution — getPredictionDistribution_binary"><meta property="og:description" content="Calculates the prediction distribution"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculates the prediction distribution</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/PredictionDistribution.R'><code>R/PredictionDistribution.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/PredictionDistribution.R" class="external-link"><code>R/PredictionDistribution.R</code></a></small>
     <div class="hidden name"><code>getPredictionDistribution_binary.Rd</code></div>
     </div>
 
@@ -183,58 +131,57 @@ <h1>Calculates the prediction distribution</h1>
     <p>Calculates the prediction distribution</p>
     </div>
 
-    <pre class="usage"><span class='fu'>getPredictionDistribution_binary</span><span class='op'>(</span><span class='va'>prediction</span>, <span class='va'>evalColumn</span>, <span class='va'>...</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>prediction</th>
-      <td><p>A prediction object</p></td>
-    </tr>
-    <tr>
-      <th>evalColumn</th>
-      <td><p>A column that is used to stratify the results</p></td>
-    </tr>
-    <tr>
-      <th>...</th>
-      <td><p>Other inputs</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>The 0.00, 0.1, 0.25, 0.5, 0.75, 0.9, 1.00 quantile pf the prediction,
-the mean and standard deviation per class</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">getPredictionDistribution_binary</span><span class="op">(</span><span class="va">prediction</span>, <span class="va">evalColumn</span>, <span class="va">...</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>prediction</dt>
+<dd><p>A prediction object</p></dd>
+
+
+<dt>evalColumn</dt>
+<dd><p>A column that is used to stratify the results</p></dd>
+
+
+<dt>...</dt>
+<dd><p>Other inputs</p></dd>
 
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>The 0.00, 0.1, 0.25, 0.5, 0.75, 0.9, 1.00 quantile pf the prediction,
+the mean and standard deviation per class</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculates the quantiles from a predition object</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/getThresholdSummary.html b/docs/reference/getThresholdSummary.html
index 9a1f6cc64..743440c53 100644
--- a/docs/reference/getThresholdSummary.html
+++ b/docs/reference/getThresholdSummary.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculate all measures for sparse ROC — getThresholdSummary • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculate all measures for sparse ROC — getThresholdSummary" />
-<meta property="og:description" content="Calculate all measures for sparse ROC" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculate all measures for sparse ROC — getThresholdSummary • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculate all measures for sparse ROC — getThresholdSummary"><meta property="og:description" content="Calculate all measures for sparse ROC"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculate all measures for sparse ROC</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ThresholdSummary.R'><code>R/ThresholdSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ThresholdSummary.R" class="external-link"><code>R/ThresholdSummary.R</code></a></small>
     <div class="hidden name"><code>getThresholdSummary.Rd</code></div>
     </div>
 
@@ -183,58 +131,57 @@ <h1>Calculate all measures for sparse ROC</h1>
     <p>Calculate all measures for sparse ROC</p>
     </div>
 
-    <pre class="usage"><span class='fu'>getThresholdSummary</span><span class='op'>(</span><span class='va'>prediction</span>, <span class='va'>predictionType</span>, typeColumn <span class='op'>=</span> <span class='st'>"evaluation"</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>prediction</th>
-      <td><p>A prediction object</p></td>
-    </tr>
-    <tr>
-      <th>predictionType</th>
-      <td><p>The type of prediction (binary or survival)</p></td>
-    </tr>
-    <tr>
-      <th>typeColumn</th>
-      <td><p>A column that is used to stratify the results</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A data.frame with all the measures</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">getThresholdSummary</span><span class="op">(</span><span class="va">prediction</span>, <span class="va">predictionType</span>, typeColumn <span class="op">=</span> <span class="st">"evaluation"</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>prediction</dt>
+<dd><p>A prediction object</p></dd>
+
+
+<dt>predictionType</dt>
+<dd><p>The type of prediction (binary or survival)</p></dd>
+
 
+<dt>typeColumn</dt>
+<dd><p>A column that is used to stratify the results</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A data.frame with all the measures</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculates the TP, FP, TN, FN, TPR, FPR, accuracy, PPF, FOR and Fmeasure
 from a prediction object</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/getThresholdSummary_binary.html b/docs/reference/getThresholdSummary_binary.html
index b45c0a63c..dd31368b0 100644
--- a/docs/reference/getThresholdSummary_binary.html
+++ b/docs/reference/getThresholdSummary_binary.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculate all measures for sparse ROC when prediction is bianry classification — getThresholdSummary_binary • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculate all measures for sparse ROC when prediction is bianry classification — getThresholdSummary_binary" />
-<meta property="og:description" content="Calculate all measures for sparse ROC when prediction is bianry classification" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculate all measures for sparse ROC when prediction is bianry classification — getThresholdSummary_binary • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculate all measures for sparse ROC when prediction is bianry classification — getThresholdSummary_binary"><meta property="og:description" content="Calculate all measures for sparse ROC when prediction is bianry classification"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculate all measures for sparse ROC when prediction is bianry classification</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ThresholdSummary.R'><code>R/ThresholdSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ThresholdSummary.R" class="external-link"><code>R/ThresholdSummary.R</code></a></small>
     <div class="hidden name"><code>getThresholdSummary_binary.Rd</code></div>
     </div>
 
@@ -183,58 +131,57 @@ <h1>Calculate all measures for sparse ROC when prediction is bianry classificati
     <p>Calculate all measures for sparse ROC when prediction is bianry classification</p>
     </div>
 
-    <pre class="usage"><span class='fu'>getThresholdSummary_binary</span><span class='op'>(</span><span class='va'>prediction</span>, <span class='va'>evalColumn</span>, <span class='va'>...</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>prediction</th>
-      <td><p>A prediction object</p></td>
-    </tr>
-    <tr>
-      <th>evalColumn</th>
-      <td><p>A column that is used to stratify the results</p></td>
-    </tr>
-    <tr>
-      <th>...</th>
-      <td><p>Other inputs</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A data.frame with all the measures</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">getThresholdSummary_binary</span><span class="op">(</span><span class="va">prediction</span>, <span class="va">evalColumn</span>, <span class="va">...</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>prediction</dt>
+<dd><p>A prediction object</p></dd>
+
+
+<dt>evalColumn</dt>
+<dd><p>A column that is used to stratify the results</p></dd>
+
 
+<dt>...</dt>
+<dd><p>Other inputs</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A data.frame with all the measures</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculates the TP, FP, TN, FN, TPR, FPR, accuracy, PPF, FOR and Fmeasure
 from a prediction object</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/grepCovariateNames.html b/docs/reference/grepCovariateNames.html
deleted file mode 100644
index 8438e0454..000000000
--- a/docs/reference/grepCovariateNames.html
+++ /dev/null
@@ -1,223 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Extract covariate names — grepCovariateNames • PatientLevelPrediction</title>
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script>
-
-<!-- sticky kit -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-<meta property="og:title" content="Extract covariate names — grepCovariateNames" />
-
-<meta property="og:description" content="Extracts covariate names using a regular-expression." />
-<meta name="twitter:card" content="summary" />
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-  </head>
-
-  <body>
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.5</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Extract covariate names</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SaveLoadPlp.R'><code>R/SaveLoadPlp.R</code></a></small>
-    <div class="hidden name"><code>grepCovariateNames.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    
-    <p>Extracts covariate names using a regular-expression.</p>
-    
-    </div>
-
-    <pre class="usage"><span class='fu'>grepCovariateNames</span>(<span class='no'>pattern</span>, <span class='no'>object</span>)</pre>
-    
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>pattern</th>
-      <td><p>A regular expression with which to name covariate names</p></td>
-    </tr>
-    <tr>
-      <th>object</th>
-      <td><p>An R object of type <code>plpData</code> or <code>covariateData</code>.</p></td>
-    </tr>
-    </table>
-    
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>Returns a <code>data.frame</code> containing information about covariates that match a regular
-expression.  This <code>data.frame</code> has the following columns:</p><dl class='dl-horizontal'>
-<dt>covariateId</dt><dd><p>Numerical identifier for use in model fitting using these covariates</p></dd>
-<dt>covariateName</dt><dd><p>Text identifier</p></dd> <dt>analysisId</dt><dd><p>Analysis identifier</p></dd> <dt>conceptId</dt><dd><p>OMOP
-common data model concept identifier, or 0</p></dd> </dl>
-
-    
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>This function extracts covariate names that match a regular-expression for a
-<code>plpData</code> or <code>covariateData</code> object.</p>
-    
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-    <h2>Contents</h2>
-    <ul class="nav nav-pills nav-stacked">
-      <li><a href="#arguments">Arguments</a></li>
-      
-      <li><a href="#value">Value</a></li>
-
-      <li><a href="#details">Details</a></li>
-          </ul>
-
-  </div>
-</div>
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-   </div>
-
-  
-
-  </body>
-</html>
-
diff --git a/docs/reference/ici.html b/docs/reference/ici.html
index 449fcbe26..aefd353ec 100644
--- a/docs/reference/ici.html
+++ b/docs/reference/ici.html
@@ -1,70 +1,15 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculate the Integrated Calibration Information from Austin and Steyerberg
-https://onlinelibrary.wiley.com/doi/full/10.1002/sim.8281 — ici • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculate the Integrated Calibration Information from Austin and Steyerberg
-https://onlinelibrary.wiley.com/doi/full/10.1002/sim.8281 — ici" />
-<meta property="og:description" content="Calculate the Integrated Calibration Information from Austin and Steyerberg
-https://onlinelibrary.wiley.com/doi/full/10.1002/sim.8281" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculate the Integrated Calibration Information from Austin and Steyerberg
+https://onlinelibrary.wiley.com/doi/full/10.1002/sim.8281 — ici • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculate the Integrated Calibration Information from Austin and Steyerberg
+https://onlinelibrary.wiley.com/doi/full/10.1002/sim.8281 — ici"><meta property="og:description" content="Calculate the Integrated Calibration Information from Austin and Steyerberg
+https://onlinelibrary.wiley.com/doi/full/10.1002/sim.8281"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -75,13 +20,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -97,13 +41,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -116,10 +59,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -128,13 +71,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -142,44 +88,46 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculate the Integrated Calibration Information from Austin and Steyerberg
 https://onlinelibrary.wiley.com/doi/full/10.1002/sim.8281</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/EvaluationSummary.R'><code>R/EvaluationSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/EvaluationSummary.R" class="external-link"><code>R/EvaluationSummary.R</code></a></small>
     <div class="hidden name"><code>ici.Rd</code></div>
     </div>
 
@@ -188,49 +136,48 @@ <h1>Calculate the Integrated Calibration Information from Austin and Steyerberg
 https://onlinelibrary.wiley.com/doi/full/10.1002/sim.8281</p>
     </div>
 
-    <pre class="usage"><span class='fu'>ici</span><span class='op'>(</span><span class='va'>prediction</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>prediction</th>
-      <td><p>the prediction object found in the plpResult object</p></td>
-    </tr>
-    </table>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">ici</span><span class="op">(</span><span class="va">prediction</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>prediction</dt>
+<dd><p>the prediction object found in the plpResult object</p></dd>
 
-    <p>Integrated Calibration Information</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
+<p>Integrated Calibration Information</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculate the Integrated Calibration Information</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/index.html b/docs/reference/index.html
index 5c485e21f..652594c5b 100644
--- a/docs/reference/index.html
+++ b/docs/reference/index.html
@@ -1,66 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Function reference • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Function reference" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Function reference • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Function reference"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-index">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -71,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -93,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -112,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -124,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -138,564 +85,683 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="contents col-md-9">
     <div class="page-header">
       <h1>Reference</h1>
     </div>
 
-    <table class="ref-index">
-
-    <colgroup>
-      
-      <col class="alias" />
-      <col class="title" />
-    </colgroup>
-
-    <tbody>
-      <tr>
-        <th colspan="2">
-          <h2 id="section-extracting-data-from-the-omop-cdm-database" class="hasAnchor"><a href="#section-extracting-data-from-the-omop-cdm-database" class="anchor"></a>Extracting data from the OMOP CDM database</h2>
-          <p class="section-desc"><p>Functions for getting the necessary data from the database in Common Data Model and saving/loading.</p></p>
+    <table class="ref-index"><colgroup><col class="alias"><col class="title"></colgroup><tbody><tr><th colspan="2">
+          <h2 id="extracting-data-from-the-omop-cdm-database">Extracting data from the OMOP CDM database <a href="#extracting-data-from-the-omop-cdm-database" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p><p>Functions for getting the necessary data from the database in Common Data Model and saving/loading.</p>
         </th>
-      </tr>
-      
-      
-    </tbody><tbody>
-      
-      
-      <tr>
-        
-        <td>
+      </tr></tbody><tbody><tr><td>
           <p><code><a href="createDatabaseDetails.html">createDatabaseDetails()</a></code> </p>
         </td>
         <td><p>Create a setting that holds the details about the cdmDatabase connection for data extraction</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="createRestrictPlpDataSettings.html">createRestrictPlpDataSettings()</a></code> </p>
         </td>
         <td><p>createRestrictPlpDataSettings define extra restriction settings when calling getPlpData</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="getPlpData.html">getPlpData()</a></code> </p>
         </td>
         <td><p>Get the patient level prediction data from the server</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="savePlpData.html">savePlpData()</a></code> </p>
         </td>
         <td><p>Save the cohort data to folder</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="loadPlpData.html">loadPlpData()</a></code> </p>
         </td>
         <td><p>Load the cohort data from a folder</p></td>
-      </tr>
-    </tbody><tbody>
-      <tr>
-        <th colspan="2">
-          <h2 id="section-settings-for-designing-a-prediction-models" class="hasAnchor"><a href="#section-settings-for-designing-a-prediction-models" class="anchor"></a>Settings for designing a prediction models</h2>
-          <p class="section-desc"><p>Design settings required when developing a model.</p></p>
+      </tr><tr><td>
+          <p><code><a href="getCohortCovariateData.html">getCohortCovariateData()</a></code> </p>
+        </td>
+        <td><p>Extracts covariates based on cohorts</p></td>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="settings-for-designing-a-prediction-models">Settings for designing a prediction models <a href="#settings-for-designing-a-prediction-models" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p><p>Design settings required when developing a model.</p>
         </th>
-      </tr>
-      
-      
-    </tbody><tbody>
-      
-      
-      <tr>
-        
-        <td>
+      </tr></tbody><tbody><tr><td>
           <p><code><a href="createStudyPopulationSettings.html">createStudyPopulationSettings()</a></code> </p>
         </td>
         <td><p>create the study population settings</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="createDefaultSplitSetting.html">createDefaultSplitSetting()</a></code> </p>
         </td>
         <td><p>Create the settings for defining how the plpData are split into test/validation/train sets using 
-default splitting functions (either random stratified by outcome, time or subject splitting).</p></td>
-      </tr><tr>
-        
-        <td>
+default splitting functions (either random stratified by outcome, time or subject splitting)</p></td>
+      </tr><tr><td>
           <p><code><a href="createSampleSettings.html">createSampleSettings()</a></code> </p>
         </td>
         <td><p>Create the settings for defining how the trainData from <code>splitData</code> are sampled using 
 default sample functions.</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="createFeatureEngineeringSettings.html">createFeatureEngineeringSettings()</a></code> </p>
         </td>
         <td><p>Create the settings for defining any feature engineering that will be done</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="createPreprocessSettings.html">createPreprocessSettings()</a></code> </p>
         </td>
-        <td><p>Create the settings for preprocessing the trainData using <code> </code>.</p></td>
-      </tr>
-    </tbody><tbody>
-      <tr>
-        <th colspan="2">
-          <h2 id="section-execution-settings-when-developing-a-model" class="hasAnchor"><a href="#section-execution-settings-when-developing-a-model" class="anchor"></a>Execution settings when developing a model</h2>
-          <p class="section-desc"><p>Execution settings required when developing a model.</p></p>
+        <td><p>Create the settings for preprocessing the trainData.</p></td>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="optional-design-settings">Optional design settings <a href="#optional-design-settings" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p><p>Settings for optional steps that can be used in the PLP pipeline</p>
         </th>
-      </tr>
-      
-      
-    </tbody><tbody>
-      
-      
-      <tr>
-        
-        <td>
+      </tr></tbody><tbody><tr><td>
+          <p><code><a href="createCohortCovariateSettings.html">createCohortCovariateSettings()</a></code> </p>
+        </td>
+        <td><p>Extracts covariates based on cohorts</p></td>
+      </tr><tr><td>
+          <p><code><a href="createRandomForestFeatureSelection.html">createRandomForestFeatureSelection()</a></code> </p>
+        </td>
+        <td><p>Create the settings for random foreat based feature selection</p></td>
+      </tr><tr><td>
+          <p><code><a href="createUnivariateFeatureSelection.html">createUnivariateFeatureSelection()</a></code> </p>
+        </td>
+        <td><p>Create the settings for defining any feature selection that will be done</p></td>
+      </tr><tr><td>
+          <p><code><a href="createSplineSettings.html">createSplineSettings()</a></code> </p>
+        </td>
+        <td><p>Create the settings for adding a spline for continuous variables</p></td>
+      </tr><tr><td>
+          <p><code><a href="createStratifiedImputationSettings.html">createStratifiedImputationSettings()</a></code> </p>
+        </td>
+        <td><p>Create the settings for adding a spline for continuous variables</p></td>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="external-validation">External validation <a href="#external-validation" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p>
+        </th>
+      </tr></tbody><tbody><tr><td>
+          <p><code><a href="createValidationDesign.html">createValidationDesign()</a></code> </p>
+        </td>
+        <td><p>createValidationDesign - Define the validation design for external validation</p></td>
+      </tr><tr><td>
+          <p><code><a href="validateExternal.html">validateExternal()</a></code> </p>
+        </td>
+        <td><p>externalValidatePlp - Validate model performance on new data</p></td>
+      </tr><tr><td>
+          <p><code><a href="createValidationSettings.html">createValidationSettings()</a></code> </p>
+        </td>
+        <td><p>createValidationSettings define optional settings for performing external validation</p></td>
+      </tr><tr><td>
+          <p><code><a href="recalibratePlp.html">recalibratePlp()</a></code> </p>
+        </td>
+        <td><p>recalibratePlp</p></td>
+      </tr><tr><td>
+          <p><code><a href="recalibratePlpRefit.html">recalibratePlpRefit()</a></code> </p>
+        </td>
+        <td><p>recalibratePlpRefit</p></td>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="execution-settings-when-developing-a-model">Execution settings when developing a model <a href="#execution-settings-when-developing-a-model" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p><p>Execution settings required when developing a model.</p>
+        </th>
+      </tr></tbody><tbody><tr><td>
           <p><code><a href="createLogSettings.html">createLogSettings()</a></code> </p>
         </td>
         <td><p>Create the settings for logging the progression of the analysis</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="createExecuteSettings.html">createExecuteSettings()</a></code> </p>
         </td>
         <td><p>Creates list of settings specifying what parts of runPlp to execute</p></td>
-      </tr>
-    </tbody><tbody>
-      <tr>
-        <th colspan="2">
-          <h2 id="section-binary-classification-models" class="hasAnchor"><a href="#section-binary-classification-models" class="anchor"></a>Binary Classification Models</h2>
-          <p class="section-desc"><p>Functions for setting binary classifiers and their hyper-parameter search.</p></p>
+      </tr><tr><td>
+          <p><code><a href="createDefaultExecuteSettings.html">createDefaultExecuteSettings()</a></code> </p>
+        </td>
+        <td><p>Creates default list of settings specifying what parts of runPlp to execute</p></td>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="binary-classification-models">Binary Classification Models <a href="#binary-classification-models" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p><p>Functions for setting binary classifiers and their hyper-parameter search.</p>
         </th>
-      </tr>
-      
-      
-    </tbody><tbody>
-      
-      
-      <tr>
-        
-        <td>
+      </tr></tbody><tbody><tr><td>
           <p><code><a href="setAdaBoost.html">setAdaBoost()</a></code> </p>
         </td>
         <td><p>Create setting for AdaBoost with python DecisionTreeClassifier base estimator</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="setDecisionTree.html">setDecisionTree()</a></code> </p>
         </td>
         <td><p>Create setting for the scikit-learn 1.0.1 DecisionTree with python</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="setGradientBoostingMachine.html">setGradientBoostingMachine()</a></code> </p>
         </td>
         <td><p>Create setting for gradient boosting machine model using gbm_xgboost implementation</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="setKNN.html">setKNN()</a></code> </p>
         </td>
         <td><p>Create setting for knn model</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="setLassoLogisticRegression.html">setLassoLogisticRegression()</a></code> </p>
         </td>
         <td><p>Create setting for lasso logistic regression</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="setMLP.html">setMLP()</a></code> </p>
         </td>
         <td><p>Create setting for neural network model with python</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="setNaiveBayes.html">setNaiveBayes()</a></code> </p>
         </td>
         <td><p>Create setting for naive bayes model with python</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="setRandomForest.html">setRandomForest()</a></code> </p>
         </td>
         <td><p>Create setting for random forest model with python (very fast)</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="setSVM.html">setSVM()</a></code> </p>
         </td>
         <td><p>Create setting for the python sklearn SVM (SVC function)</p></td>
-      </tr>
-    </tbody><tbody>
-      <tr>
-        <th colspan="2">
-          <h2 id="section-survival-models" class="hasAnchor"><a href="#section-survival-models" class="anchor"></a>Survival Models</h2>
-          <p class="section-desc"><p>Functions for setting survival models and their hyper-parameter search.</p></p>
+      </tr><tr><td>
+          <p><code><a href="setIterativeHardThresholding.html">setIterativeHardThresholding()</a></code> </p>
+        </td>
+        <td><p>Create setting for lasso logistic regression</p></td>
+      </tr><tr><td>
+          <p><code><a href="setLightGBM.html">setLightGBM()</a></code> </p>
+        </td>
+        <td><p>Create setting for gradient boosting machine model using lightGBM (https://github.com/microsoft/LightGBM/tree/master/R-package).</p></td>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="survival-models">Survival Models <a href="#survival-models" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p><p>Functions for setting survival models and their hyper-parameter search.</p>
         </th>
-      </tr>
-      
-      
-    </tbody><tbody>
-      
-      
-      <tr>
-        
-        <td>
+      </tr></tbody><tbody><tr><td>
           <p><code><a href="setCoxModel.html">setCoxModel()</a></code> </p>
         </td>
         <td><p>Create setting for lasso Cox model</p></td>
-      </tr>
-    </tbody><tbody>
-      <tr>
-        <th colspan="2">
-          <h2 id="section-single-patient-level-prediction-model" class="hasAnchor"><a href="#section-single-patient-level-prediction-model" class="anchor"></a>Single Patient-Level Prediction Model</h2>
-          <p class="section-desc"><p>Functions for training/evaluating/applying a single patient-level-prediction model</p></p>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="single-patient-level-prediction-model">Single Patient-Level Prediction Model <a href="#single-patient-level-prediction-model" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p><p>Functions for training/evaluating/applying a single patient-level-prediction model</p>
         </th>
-      </tr>
-      
-      
-    </tbody><tbody>
-      
-      
-      <tr>
-        
-        <td>
+      </tr></tbody><tbody><tr><td>
           <p><code><a href="runPlp.html">runPlp()</a></code> </p>
         </td>
         <td><p>runPlp - Develop and internally evaluate a model using specified settings</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="externalValidateDbPlp.html">externalValidateDbPlp()</a></code> </p>
         </td>
         <td><p>externalValidateDbPlp - Validate a model on new databases</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="savePlpModel.html">savePlpModel()</a></code> </p>
         </td>
         <td><p>Saves the plp model</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="loadPlpModel.html">loadPlpModel()</a></code> </p>
         </td>
         <td><p>loads the plp model</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="savePlpResult.html">savePlpResult()</a></code> </p>
         </td>
         <td><p>Saves the result from runPlp into the location directory</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="loadPlpResult.html">loadPlpResult()</a></code> </p>
         </td>
         <td><p>Loads the evalaution dataframe</p></td>
-      </tr>
-    </tbody><tbody>
-      <tr>
-        <th colspan="2">
-          <h2 id="section-multiple-patient-level-prediction-models" class="hasAnchor"><a href="#section-multiple-patient-level-prediction-models" class="anchor"></a>Multiple Patient-Level Prediction Models</h2>
-          <p class="section-desc"><p>Functions for training mutliple patient-level-prediction model in an efficient way.</p></p>
+      </tr><tr><td>
+          <p><code><a href="diagnosePlp.html">diagnosePlp()</a></code> </p>
+        </td>
+        <td><p>diagnostic - Investigates the prediction problem settings - use before training a model</p></td>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="multiple-patient-level-prediction-models">Multiple Patient-Level Prediction Models <a href="#multiple-patient-level-prediction-models" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p><p>Functions for training mutliple patient-level-prediction model in an efficient way.</p>
         </th>
-      </tr>
-      
-      
-    </tbody><tbody>
-      
-      
-      <tr>
-        
-        <td>
+      </tr></tbody><tbody><tr><td>
           <p><code><a href="createModelDesign.html">createModelDesign()</a></code> </p>
         </td>
         <td><p>Specify settings for deceloping a single model</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="runMultiplePlp.html">runMultiplePlp()</a></code> </p>
         </td>
         <td><p>Run a list of predictions analyses</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="validateMultiplePlp.html">validateMultiplePlp()</a></code> </p>
         </td>
         <td><p>externally validate the multiple plp models across new datasets</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="savePlpAnalysesJson.html">savePlpAnalysesJson()</a></code> </p>
         </td>
         <td><p>Save the modelDesignList to a json file</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="loadPlpAnalysesJson.html">loadPlpAnalysesJson()</a></code> </p>
         </td>
         <td><p>Load the multiple prediction json settings from a file</p></td>
-      </tr>
-    </tbody><tbody>
-      <tr>
-        <th colspan="2">
-          <h2 id="section-saving-results-into-database" class="hasAnchor"><a href="#section-saving-results-into-database" class="anchor"></a>Saving results into database</h2>
-          <p class="section-desc"><p>Functions for saving the prediction model and performances into a database.</p></p>
+      </tr><tr><td>
+          <p><code><a href="diagnoseMultiplePlp.html">diagnoseMultiplePlp()</a></code> </p>
+        </td>
+        <td><p>Run a list of predictions diagnoses</p></td>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="individual-pipeline-functions">Individual pipeline functions <a href="#individual-pipeline-functions" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p><p>Functions for running parts of the PLP workflow</p>
         </th>
-      </tr>
-      
-      
-    </tbody><tbody>
-      
-      
-      <tr>
-        
-        <td>
+      </tr></tbody><tbody><tr><td>
+          <p><code><a href="createStudyPopulation.html">createStudyPopulation()</a></code> </p>
+        </td>
+        <td><p>Create a study population</p></td>
+      </tr><tr><td>
+          <p><code><a href="splitData.html">splitData()</a></code> </p>
+        </td>
+        <td><p>Split the plpData into test/train sets using a splitting settings of class <code>splitSettings</code></p></td>
+      </tr><tr><td>
+          <p><code><a href="preprocessData.html">preprocessData()</a></code> </p>
+        </td>
+        <td><p>A function that wraps around FeatureExtraction::tidyCovariateData to normalise the data
+and remove rare or redundant features</p></td>
+      </tr><tr><td>
+          <p><code><a href="fitPlp.html">fitPlp()</a></code> </p>
+        </td>
+        <td><p>fitPlp</p></td>
+      </tr><tr><td>
+          <p><code><a href="predictPlp.html">predictPlp()</a></code> </p>
+        </td>
+        <td><p>predictPlp</p></td>
+      </tr><tr><td>
+          <p><code><a href="evaluatePlp.html">evaluatePlp()</a></code> </p>
+        </td>
+        <td><p>evaluatePlp</p></td>
+      </tr><tr><td>
+          <p><code><a href="covariateSummary.html">covariateSummary()</a></code> </p>
+        </td>
+        <td><p>covariateSummary</p></td>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="saving-results-into-database">Saving results into database <a href="#saving-results-into-database" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p><p>Functions for saving the prediction model and performances into a database.</p>
+        </th>
+      </tr></tbody><tbody><tr><td>
+          <p><code><a href="insertResultsToSqlite.html">insertResultsToSqlite()</a></code> </p>
+        </td>
+        <td><p>Create sqlite database with the results</p></td>
+      </tr><tr><td>
           <p><code><a href="createPlpResultTables.html">createPlpResultTables()</a></code> </p>
         </td>
         <td><p>Create the results tables to store PatientLevelPrediction models and results into a database</p></td>
-      </tr><tr>
-        
-        <td>
-          <p><code><a href="populatePlpResultTables.html">populatePlpResultTables()</a></code> </p>
+      </tr><tr><td>
+          <p><code><a href="addMultipleRunPlpToDatabase.html">addMultipleRunPlpToDatabase()</a></code> </p>
         </td>
         <td><p>Populate the PatientLevelPrediction results tables</p></td>
-      </tr>
-    </tbody><tbody>
-      <tr>
-        <th colspan="2">
-          <h2 id="section-shiny-viewers" class="hasAnchor"><a href="#section-shiny-viewers" class="anchor"></a>Shiny Viewers</h2>
-          <p class="section-desc"><p>Functions for viewing results via a shiny app</p></p>
+      </tr><tr><td>
+          <p><code><a href="addRunPlpToDatabase.html">addRunPlpToDatabase()</a></code> </p>
+        </td>
+        <td><p>Function to add the run plp (development or validation) to database</p></td>
+      </tr><tr><td>
+          <p><code><a href="createDatabaseSchemaSettings.html">createDatabaseSchemaSettings()</a></code> </p>
+        </td>
+        <td><p>Create the PatientLevelPrediction database result schema settings</p></td>
+      </tr><tr><td>
+          <p><code><a href="createDatabaseList.html">createDatabaseList()</a></code> </p>
+        </td>
+        <td><p>Create a list with the database details and database meta data entries</p></td>
+      </tr><tr><td>
+          <p><code><a href="addDiagnosePlpToDatabase.html">addDiagnosePlpToDatabase()</a></code> </p>
+        </td>
+        <td><p>Insert a diagnostic result into a PLP result schema database</p></td>
+      </tr><tr><td>
+          <p><code><a href="addMultipleDiagnosePlpToDatabase.html">addMultipleDiagnosePlpToDatabase()</a></code> </p>
+        </td>
+        <td><p>Insert mutliple diagnosePlp results saved to a directory into a PLP result schema database</p></td>
+      </tr><tr><td>
+          <p><code><a href="extractDatabaseToCsv.html">extractDatabaseToCsv()</a></code> </p>
+        </td>
+        <td><p>Exports all the results from a database into csv files</p></td>
+      </tr><tr><td>
+          <p><code><a href="insertCsvToDatabase.html">insertCsvToDatabase()</a></code> </p>
+        </td>
+        <td><p>Function to insert results into a database from csvs</p></td>
+      </tr><tr><td>
+          <p><code><a href="insertModelDesignInDatabase.html">insertModelDesignInDatabase()</a></code> </p>
+        </td>
+        <td><p>Insert a model design into a PLP result schema database</p></td>
+      </tr><tr><td>
+          <p><code><a href="migrateDataModel.html">migrateDataModel()</a></code> </p>
+        </td>
+        <td><p>Migrate Data model</p></td>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="shiny-viewers">Shiny Viewers <a href="#shiny-viewers" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p><p>Functions for viewing results via a shiny app</p>
         </th>
-      </tr>
-      
-      
-    </tbody><tbody>
-      
-      
-      <tr>
-        
-        <td>
+      </tr></tbody><tbody><tr><td>
           <p><code><a href="viewPlp.html">viewPlp()</a></code> </p>
         </td>
         <td><p>viewPlp - Interactively view the performance and model settings</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="viewMultiplePlp.html">viewMultiplePlp()</a></code> </p>
         </td>
         <td><p>open a local shiny app for viewing the result of a multiple PLP analyses</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="viewDatabaseResultPlp.html">viewDatabaseResultPlp()</a></code> </p>
         </td>
         <td><p>open a local shiny app for viewing the result of a PLP analyses from a database</p></td>
-      </tr>
-    </tbody><tbody>
-      <tr>
-        <th colspan="2">
-          <h2 id="section-plotting" class="hasAnchor"><a href="#section-plotting" class="anchor"></a>Plotting</h2>
-          <p class="section-desc"><p>Functions for various performance plots</p></p>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="plotting">Plotting <a href="#plotting" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p><p>Functions for various performance plots</p>
         </th>
-      </tr>
-      
-      
-    </tbody><tbody>
-      
-      
-      <tr>
-        
-        <td>
+      </tr></tbody><tbody><tr><td>
           <p><code><a href="plotPlp.html">plotPlp()</a></code> </p>
         </td>
         <td><p>Plot all the PatientLevelPrediction plots</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="plotSparseRoc.html">plotSparseRoc()</a></code> </p>
         </td>
         <td><p>Plot the ROC curve using the sparse thresholdSummary data frame</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="plotSmoothCalibration.html">plotSmoothCalibration()</a></code> </p>
         </td>
         <td><p>Plot the smooth calibration as detailed in Calster et al. "A calibration heirarchy for risk models
 was defined: from utopia to empirical data" (2016)</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="plotSparseCalibration.html">plotSparseCalibration()</a></code> </p>
         </td>
         <td><p>Plot the calibration</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="plotSparseCalibration2.html">plotSparseCalibration2()</a></code> </p>
         </td>
         <td><p>Plot the conventional calibration</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="plotDemographicSummary.html">plotDemographicSummary()</a></code> </p>
         </td>
         <td><p>Plot the Observed vs. expected incidence, by age and gender</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="plotF1Measure.html">plotF1Measure()</a></code> </p>
         </td>
         <td><p>Plot the F1 measure efficiency frontier using the sparse thresholdSummary data frame</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="plotGeneralizability.html">plotGeneralizability()</a></code> </p>
         </td>
         <td><p>Plot the train/test generalizability diagnostic</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="plotPrecisionRecall.html">plotPrecisionRecall()</a></code> </p>
         </td>
         <td><p>Plot the precision-recall curve using the sparse thresholdSummary data frame</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="plotPredictedPDF.html">plotPredictedPDF()</a></code> </p>
         </td>
         <td><p>Plot the Predicted probability density function, showing prediction overlap between true and false cases</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="plotPreferencePDF.html">plotPreferencePDF()</a></code> </p>
         </td>
         <td><p>Plot the preference score probability density function, showing prediction overlap between true and false cases
 #'</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="plotPredictionDistribution.html">plotPredictionDistribution()</a></code> </p>
         </td>
         <td><p>Plot the side-by-side boxplots of prediction distribution, by class#'</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="plotVariableScatterplot.html">plotVariableScatterplot()</a></code> </p>
         </td>
         <td><p>Plot the variable importance scatterplot</p></td>
-      </tr>
-    </tbody><tbody>
-      <tr>
-        <th colspan="2">
-          <h2 id="section-learning-curves" class="hasAnchor"><a href="#section-learning-curves" class="anchor"></a>Learning Curves</h2>
-          <p class="section-desc"><p>Functions for creating and plotting learning curves</p></p>
+      </tr><tr><td>
+          <p><code><a href="outcomeSurvivalPlot.html">outcomeSurvivalPlot()</a></code> </p>
+        </td>
+        <td><p>Plot the outcome incidence over time</p></td>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="learning-curves">Learning Curves <a href="#learning-curves" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p><p>Functions for creating and plotting learning curves</p>
         </th>
-      </tr>
-      
-      
-    </tbody><tbody>
-      
-      
-      <tr>
-        
-        <td>
+      </tr></tbody><tbody><tr><td>
           <p><code><a href="createLearningCurve.html">createLearningCurve()</a></code> </p>
         </td>
         <td><p>createLearningCurve</p></td>
-      </tr><tr>
-        
-        <td>
+      </tr><tr><td>
           <p><code><a href="plotLearningCurve.html">plotLearningCurve()</a></code> </p>
         </td>
         <td><p>plotLearningCurve</p></td>
-      </tr>
-    </tbody><tbody>
-      <tr>
-        <th colspan="2">
-          <h2 id="section-simulation" class="hasAnchor"><a href="#section-simulation" class="anchor"></a>Simulation</h2>
-          <p class="section-desc"><p>Functions for simulating cohort method data objects.</p></p>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="simulation">Simulation <a href="#simulation" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p><p>Functions for simulating cohort method data objects.</p>
         </th>
-      </tr>
-      
-      
-    </tbody><tbody>
-      
-      
-      <tr>
-        
-        <td>
+      </tr></tbody><tbody><tr><td>
           <p><code><a href="simulatePlpData.html">simulatePlpData()</a></code> </p>
         </td>
         <td><p>Generate simulated data</p></td>
-      </tr>
-    </tbody>
-    </table>
-  </div>
+      </tr><tr><td>
+          <p><code><a href="plpDataSimulationProfile.html">plpDataSimulationProfile</a></code> </p>
+        </td>
+        <td><p>A simulation profile</p></td>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="data-manipulation-functions">Data manipulation functions <a href="#data-manipulation-functions" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p><p>Functions for manipulating data</p>
+        </th>
+      </tr></tbody><tbody><tr><td>
+          <p><code><a href="toSparseM.html">toSparseM()</a></code> </p>
+        </td>
+        <td><p>Convert the plpData in COO format into a sparse R matrix</p></td>
+      </tr><tr><td>
+          <p><code><a href="MapIds.html">MapIds()</a></code> </p>
+        </td>
+        <td><p>Map covariate and row Ids so they start from 1</p></td>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="helper-utility-functions">Helper/utility functions <a href="#helper-utility-functions" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p>
+        </th>
+      </tr></tbody><tbody><tr><td>
+          <p><code><a href="listAppend.html">listAppend()</a></code> </p>
+        </td>
+        <td><p>join two lists</p></td>
+      </tr><tr><td>
+          <p><code><a href="listCartesian.html">listCartesian()</a></code> </p>
+        </td>
+        <td><p>Cartesian product</p></td>
+      </tr><tr><td>
+          <p><code><a href="createTempModelLoc.html">createTempModelLoc()</a></code> </p>
+        </td>
+        <td><p>Create a temporary model location</p></td>
+      </tr><tr><td>
+          <p><code><a href="configurePython.html">configurePython()</a></code> </p>
+        </td>
+        <td><p>Sets up a virtual environment to use for PLP (can be conda or python)</p></td>
+      </tr><tr><td>
+          <p><code><a href="setPythonEnvironment.html">setPythonEnvironment()</a></code> </p>
+        </td>
+        <td><p>Use the virtual environment created using configurePython()</p></td>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="evaluation-measures">Evaluation measures <a href="#evaluation-measures" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p>
+        </th>
+      </tr></tbody><tbody><tr><td>
+          <p><code><a href="accuracy.html">accuracy()</a></code> </p>
+        </td>
+        <td><p>Calculate the accuracy</p></td>
+      </tr><tr><td>
+          <p><code><a href="averagePrecision.html">averagePrecision()</a></code> </p>
+        </td>
+        <td><p>Calculate the average precision</p></td>
+      </tr><tr><td>
+          <p><code><a href="brierScore.html">brierScore()</a></code> </p>
+        </td>
+        <td><p>brierScore</p></td>
+      </tr><tr><td>
+          <p><code><a href="calibrationLine.html">calibrationLine()</a></code> </p>
+        </td>
+        <td><p>calibrationLine</p></td>
+      </tr><tr><td>
+          <p><code><a href="computeAuc.html">computeAuc()</a></code> </p>
+        </td>
+        <td><p>Compute the area under the ROC curve</p></td>
+      </tr><tr><td>
+          <p><code><a href="f1Score.html">f1Score()</a></code> </p>
+        </td>
+        <td><p>Calculate the f1Score</p></td>
+      </tr><tr><td>
+          <p><code><a href="falseDiscoveryRate.html">falseDiscoveryRate()</a></code> </p>
+        </td>
+        <td><p>Calculate the falseDiscoveryRate</p></td>
+      </tr><tr><td>
+          <p><code><a href="falseNegativeRate.html">falseNegativeRate()</a></code> </p>
+        </td>
+        <td><p>Calculate the falseNegativeRate</p></td>
+      </tr><tr><td>
+          <p><code><a href="falseOmissionRate.html">falseOmissionRate()</a></code> </p>
+        </td>
+        <td><p>Calculate the falseOmissionRate</p></td>
+      </tr><tr><td>
+          <p><code><a href="falsePositiveRate.html">falsePositiveRate()</a></code> </p>
+        </td>
+        <td><p>Calculate the falsePositiveRate</p></td>
+      </tr><tr><td>
+          <p><code><a href="ici.html">ici()</a></code> </p>
+        </td>
+        <td><p>Calculate the Integrated Calibration Information from Austin and Steyerberg
+https://onlinelibrary.wiley.com/doi/full/10.1002/sim.8281</p></td>
+      </tr><tr><td>
+          <p><code><a href="modelBasedConcordance.html">modelBasedConcordance()</a></code> </p>
+        </td>
+        <td><p>Calculate the model-based concordance, which is a calculation of the expected discrimination performance of a model under the assumption the model predicts the "TRUE" outcome
+as detailed in van Klaveren et al. https://pubmed.ncbi.nlm.nih.gov/27251001/</p></td>
+      </tr><tr><td>
+          <p><code><a href="negativeLikelihoodRatio.html">negativeLikelihoodRatio()</a></code> </p>
+        </td>
+        <td><p>Calculate the negativeLikelihoodRatio</p></td>
+      </tr><tr><td>
+          <p><code><a href="negativePredictiveValue.html">negativePredictiveValue()</a></code> </p>
+        </td>
+        <td><p>Calculate the negativePredictiveValue</p></td>
+      </tr><tr><td>
+          <p><code><a href="positiveLikelihoodRatio.html">positiveLikelihoodRatio()</a></code> </p>
+        </td>
+        <td><p>Calculate the positiveLikelihoodRatio</p></td>
+      </tr><tr><td>
+          <p><code><a href="positivePredictiveValue.html">positivePredictiveValue()</a></code> </p>
+        </td>
+        <td><p>Calculate the positivePredictiveValue</p></td>
+      </tr><tr><td>
+          <p><code><a href="sensitivity.html">sensitivity()</a></code> </p>
+        </td>
+        <td><p>Calculate the sensitivity</p></td>
+      </tr><tr><td>
+          <p><code><a href="specificity.html">specificity()</a></code> </p>
+        </td>
+        <td><p>Calculate the specificity</p></td>
+      </tr><tr><td>
+          <p><code><a href="computeGridPerformance.html">computeGridPerformance()</a></code> </p>
+        </td>
+        <td><p>Computes grid performance with a specified performance function</p></td>
+      </tr><tr><td>
+          <p><code><a href="diagnosticOddsRatio.html">diagnosticOddsRatio()</a></code> </p>
+        </td>
+        <td><p>Calculate the diagnostic odds ratio</p></td>
+      </tr><tr><td>
+          <p><code><a href="getCalibrationSummary.html">getCalibrationSummary()</a></code> </p>
+        </td>
+        <td><p>Get a sparse summary of the calibration</p></td>
+      </tr><tr><td>
+          <p><code><a href="getDemographicSummary.html">getDemographicSummary()</a></code> </p>
+        </td>
+        <td><p>Get a calibration per age/gender groups</p></td>
+      </tr><tr><td>
+          <p><code><a href="getThresholdSummary.html">getThresholdSummary()</a></code> </p>
+        </td>
+        <td><p>Calculate all measures for sparse ROC</p></td>
+      </tr><tr><td>
+          <p><code><a href="getThresholdSummary_binary.html">getThresholdSummary_binary()</a></code> </p>
+        </td>
+        <td><p>Calculate all measures for sparse ROC when prediction is bianry classification</p></td>
+      </tr><tr><td>
+          <p><code><a href="getPredictionDistribution.html">getPredictionDistribution()</a></code> </p>
+        </td>
+        <td><p>Calculates the prediction distribution</p></td>
+      </tr><tr><td>
+          <p><code><a href="getPredictionDistribution_binary.html">getPredictionDistribution_binary()</a></code> </p>
+        </td>
+        <td><p>Calculates the prediction distribution</p></td>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="saving-loading-models-as-json">Saving/loading models as json <a href="#saving-loading-models-as-json" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p><p>Functions for saving or loading models as json</p>
+        </th>
+      </tr></tbody><tbody><tr><td>
+          <p><code><a href="sklearnFromJson.html">sklearnFromJson()</a></code> </p>
+        </td>
+        <td><p>Loads sklearn python model from json</p></td>
+      </tr><tr><td>
+          <p><code><a href="sklearnToJson.html">sklearnToJson()</a></code> </p>
+        </td>
+        <td><p>Saves sklearn python model object to json in path</p></td>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="load-save-for-sharing">Load/save for sharing <a href="#load-save-for-sharing" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p><p>Functions for loading/saving objects for sharing</p>
+        </th>
+      </tr></tbody><tbody><tr><td>
+          <p><code><a href="savePlpShareable.html">savePlpShareable()</a></code> </p>
+        </td>
+        <td><p>Save the plp result as json files and csv files for transparent sharing</p></td>
+      </tr><tr><td>
+          <p><code><a href="loadPlpShareable.html">loadPlpShareable()</a></code> </p>
+        </td>
+        <td><p>Loads the plp result saved as json/csv files for transparent sharing</p></td>
+      </tr><tr><td>
+          <p><code><a href="loadPrediction.html">loadPrediction()</a></code> </p>
+        </td>
+        <td><p>Loads the prediciton dataframe to csv</p></td>
+      </tr><tr><td>
+          <p><code><a href="savePrediction.html">savePrediction()</a></code> </p>
+        </td>
+        <td><p>Saves the prediction dataframe to RDS</p></td>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="feature-importance">Feature importance <a href="#feature-importance" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p>
+        </th>
+      </tr></tbody><tbody><tr><td>
+          <p><code><a href="pfi.html">pfi()</a></code> </p>
+        </td>
+        <td><p>pfi</p></td>
+      </tr></tbody><tbody><tr><th colspan="2">
+          <h2 id="other-functions">Other functions <a href="#other-functions" class="anchor" aria-hidden="true"></a></h2>
+          <p class="section-desc"></p>
+        </th>
+      </tr></tbody><tbody><tr><td>
+          <p><code><a href="predictCyclops.html">predictCyclops()</a></code> </p>
+        </td>
+        <td><p>Create predictive probabilities</p></td>
+      </tr></tbody></table></div>
 
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/insertCsvToDatabase.html b/docs/reference/insertCsvToDatabase.html
new file mode 100644
index 000000000..80fd8e6cf
--- /dev/null
+++ b/docs/reference/insertCsvToDatabase.html
@@ -0,0 +1,200 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Function to insert results into a database from csvs — insertCsvToDatabase • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Function to insert results into a database from csvs — insertCsvToDatabase"><meta property="og:description" content="This function converts a folder with csv results into plp objects and loads them into a plp result database"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Function to insert results into a database from csvs</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ImportFromCsv.R" class="external-link"><code>R/ImportFromCsv.R</code></a></small>
+    <div class="hidden name"><code>insertCsvToDatabase.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>This function converts a folder with csv results into plp objects and loads them into a plp result database</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">insertCsvToDatabase</span><span class="op">(</span></span>
+<span>  <span class="va">csvFolder</span>,</span>
+<span>  <span class="va">connectionDetails</span>,</span>
+<span>  <span class="va">databaseSchemaSettings</span>,</span>
+<span>  <span class="va">modelSaveLocation</span>,</span>
+<span>  csvTableAppend <span class="op">=</span> <span class="st">""</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>csvFolder</dt>
+<dd><p>The location to the csv folder with the plp results</p></dd>
+
+
+<dt>connectionDetails</dt>
+<dd><p>A connection details for the plp results database that the csv results will be inserted into</p></dd>
+
+
+<dt>databaseSchemaSettings</dt>
+<dd><p>A object created by <code>createDatabaseSchemaSettings</code> with all the settings specifying the result tables to insert the csv results into</p></dd>
+
+
+<dt>modelSaveLocation</dt>
+<dd><p>The location to save any models from the csv folder - this should be the same location you picked when inserting other models into the database</p></dd>
+
+
+<dt>csvTableAppend</dt>
+<dd><p>A string that appends the csv file names</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>Returns a data.frame indicating whether the results were inported into the database</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>The user needs to have plp csv results in a single folder and an existing plp result database</p>
+    </div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/insertDbPopulation.html b/docs/reference/insertDbPopulation.html
deleted file mode 100644
index c733a5724..000000000
--- a/docs/reference/insertDbPopulation.html
+++ /dev/null
@@ -1,245 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Insert a population into a database — insertDbPopulation • PatientLevelPrediction</title>
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script>
-
-<!-- sticky kit -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-<meta property="og:title" content="Insert a population into a database — insertDbPopulation" />
-
-<meta property="og:description" content="Insert a population into a database" />
-<meta name="twitter:card" content="summary" />
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-  </head>
-
-  <body>
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.5</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Insert a population into a database</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SaveLoadPlp.R'><code>R/SaveLoadPlp.R</code></a></small>
-    <div class="hidden name"><code>insertDbPopulation.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    
-    <p>Insert a population into a database</p>
-    
-    </div>
-
-    <pre class="usage"><span class='fu'>insertDbPopulation</span>(<span class='no'>population</span>, <span class='kw'>cohortIds</span> <span class='kw'>=</span> <span class='fl'>1</span>, <span class='no'>connectionDetails</span>,
-  <span class='no'>cohortDatabaseSchema</span>, <span class='kw'>cohortTable</span> <span class='kw'>=</span> <span class='st'>"cohort"</span>, <span class='kw'>createTable</span> <span class='kw'>=</span> <span class='fl'>FALSE</span>,
-  <span class='kw'>dropTableIfExists</span> <span class='kw'>=</span> <span class='fl'>TRUE</span>, <span class='kw'>cdmVersion</span> <span class='kw'>=</span> <span class='st'>"5"</span>)</pre>
-    
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>population</th>
-      <td><p>Either an object of type <code>plpData</code> or a population object generated by functions
-like <code>createStudyPopulation</code>.</p></td>
-    </tr>
-    <tr>
-      <th>cohortIds</th>
-      <td><p>The IDs to be used for the treated and comparator cohort, respectively.</p></td>
-    </tr>
-    <tr>
-      <th>connectionDetails</th>
-      <td><p>An R object of type<br /><code>connectionDetails</code> created using the
-function <code>createConnectionDetails</code> in the
-<code>DatabaseConnector</code> package.</p></td>
-    </tr>
-    <tr>
-      <th>cohortDatabaseSchema</th>
-      <td><p>The name of the database schema where the data will be written.
-Requires write permissions to this database. On SQL
-Server, this should specifiy both the database and the schema,
-so for example 'cdm_instance.dbo'.</p></td>
-    </tr>
-    <tr>
-      <th>cohortTable</th>
-      <td><p>The name of the table in the database schema  where the data will be written.</p></td>
-    </tr>
-    <tr>
-      <th>createTable</th>
-      <td><p>Should a new table be created? If not, the data will be inserted into an existing
-table.</p></td>
-    </tr>
-    <tr>
-      <th>dropTableIfExists</th>
-      <td><p>If <code>createTable = TRUE</code> and the table already exists it will be overwritten.</p></td>
-    </tr>
-    <tr>
-      <th>cdmVersion</th>
-      <td><p>Define the OMOP CDM version used: currently support "4" and "5".</p></td>
-    </tr>
-    </table>
-    
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Inserts a population table into a database. The table in the database will have the same structure as the
-'cohort' table in the Common Data Model.</p>
-    
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-    <h2>Contents</h2>
-    <ul class="nav nav-pills nav-stacked">
-      <li><a href="#arguments">Arguments</a></li>
-      
-      <li><a href="#details">Details</a></li>
-          </ul>
-
-  </div>
-</div>
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-   </div>
-
-  
-
-  </body>
-</html>
-
diff --git a/docs/reference/insertModelDesignInDatabase.html b/docs/reference/insertModelDesignInDatabase.html
new file mode 100644
index 000000000..a69c6aa60
--- /dev/null
+++ b/docs/reference/insertModelDesignInDatabase.html
@@ -0,0 +1,197 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Insert a model design into a PLP result schema database — insertModelDesignInDatabase • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Insert a model design into a PLP result schema database — insertModelDesignInDatabase"><meta property="og:description" content="This function inserts a model design and all the settings into the result schema"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Insert a model design into a PLP result schema database</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/uploadToDatabaseModelDesign.R" class="external-link"><code>R/uploadToDatabaseModelDesign.R</code></a></small>
+    <div class="hidden name"><code>insertModelDesignInDatabase.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>This function inserts a model design and all the settings into the result schema</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">insertModelDesignInDatabase</span><span class="op">(</span></span>
+<span>  <span class="va">object</span>,</span>
+<span>  <span class="va">conn</span>,</span>
+<span>  <span class="va">databaseSchemaSettings</span>,</span>
+<span>  <span class="va">cohortDefinitions</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>object</dt>
+<dd><p>An object of class modelDesign, runPlp or externalValidatePlp</p></dd>
+
+
+<dt>conn</dt>
+<dd><p>A connection to a database created by using the
+function <code>connect</code> in the
+<code>DatabaseConnector</code> package.</p></dd>
+
+
+<dt>databaseSchemaSettings</dt>
+<dd><p>A object created by <code>createDatabaseSchemaSettings</code> with all the settings specifying the result tables</p></dd>
+
+
+<dt>cohortDefinitions</dt>
+<dd><p>A set of one or more cohorts extracted using ROhdsiWebApi::exportCohortDefinitionSet()</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>Returns NULL but uploads the model design into the database schema specified in databaseSchemaSettings</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>This function can be used to upload a model design into a database</p>
+    </div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/insertResultsToSqlite.html b/docs/reference/insertResultsToSqlite.html
new file mode 100644
index 000000000..1fe355a82
--- /dev/null
+++ b/docs/reference/insertResultsToSqlite.html
@@ -0,0 +1,195 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create sqlite database with the results — insertResultsToSqlite • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create sqlite database with the results — insertResultsToSqlite"><meta property="og:description" content="This function create an sqlite database with the PLP result schema and inserts all results"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Create sqlite database with the results</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/uploadToDatabase.R" class="external-link"><code>R/uploadToDatabase.R</code></a></small>
+    <div class="hidden name"><code>insertResultsToSqlite.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>This function create an sqlite database with the PLP result schema and inserts all results</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">insertResultsToSqlite</span><span class="op">(</span></span>
+<span>  <span class="va">resultLocation</span>,</span>
+<span>  <span class="va">cohortDefinitions</span>,</span>
+<span>  databaseList <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  sqliteLocation <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html" class="external-link">file.path</a></span><span class="op">(</span><span class="va">resultLocation</span>, <span class="st">"sqlite"</span><span class="op">)</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>resultLocation</dt>
+<dd><p>(string) location of directory where the main package results were saved</p></dd>
+
+
+<dt>cohortDefinitions</dt>
+<dd><p>A set of one or more cohorts extracted using ROhdsiWebApi::exportCohortDefinitionSet()</p></dd>
+
+
+<dt>databaseList</dt>
+<dd><p>A list created by <code>createDatabaseList</code> to specify the databases</p></dd>
+
+
+<dt>sqliteLocation</dt>
+<dd><p>(string) location of directory where the sqlite database will be saved</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>Returns the location of the sqlite database file</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>This function can be used upload PatientLevelPrediction results into an sqlite database</p>
+    </div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/interpretInstallCode.html b/docs/reference/interpretInstallCode.html
deleted file mode 100644
index a8fcd6e16..000000000
--- a/docs/reference/interpretInstallCode.html
+++ /dev/null
@@ -1,222 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Tells you the package issue — interpretInstallCode • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Tells you the package issue — interpretInstallCode" />
-<meta property="og:description" content="Tells you the package issue" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Tells you the package issue</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/HelperFunctions.R'><code>R/HelperFunctions.R</code></a></small>
-    <div class="hidden name"><code>interpretInstallCode.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Tells you the package issue</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>interpretInstallCode</span><span class='op'>(</span><span class='va'>response</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>response</th>
-      <td><p>The response code from checkPlpInstallation()</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>This function prints any issues found during the checkPlpInstallation() call</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/launchDiagnosticsExplorer.html b/docs/reference/launchDiagnosticsExplorer.html
deleted file mode 100644
index 17135c991..000000000
--- a/docs/reference/launchDiagnosticsExplorer.html
+++ /dev/null
@@ -1,235 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Launch the Diagnostics Explorer Shiny app — launchDiagnosticsExplorer • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Launch the Diagnostics Explorer Shiny app — launchDiagnosticsExplorer" />
-<meta property="og:description" content="Launch the Diagnostics Explorer Shiny app" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/InstallationGuide.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
-    </li>
-    <li>
-      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
-    </li>
-    <li>
-      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Launch the Diagnostics Explorer Shiny app</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ViewShinyPlp.R'><code>R/ViewShinyPlp.R</code></a></small>
-    <div class="hidden name"><code>launchDiagnosticsExplorer.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Launch the Diagnostics Explorer Shiny app</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>launchDiagnosticsExplorer</span><span class='op'>(</span><span class='va'>dataFolder</span>, launch.browser <span class='op'>=</span> <span class='cn'>FALSE</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>dataFolder</th>
-      <td><p>A folder where the exported zip files with the results are stored.
-Zip files containing results from multiple databases can be placed in the same
-folder.</p></td>
-    </tr>
-    <tr>
-      <th>launch.browser</th>
-      <td><p>Should the app be launched in your default browser, or in a Shiny window.
-Note: copying to clipboard will not work in a Shiny window.</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Launches a Shiny app that allows the user to explore the diagnostics</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/listAppend.html b/docs/reference/listAppend.html
index d98a5a463..2f370ad63 100644
--- a/docs/reference/listAppend.html
+++ b/docs/reference/listAppend.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>join two lists — listAppend • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="join two lists — listAppend" />
-<meta property="og:description" content="join two lists" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>join two lists — listAppend • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="join two lists — listAppend"><meta property="og:description" content="join two lists"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>join two lists</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/HelperFunctions.R'><code>R/HelperFunctions.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/HelperFunctions.R" class="external-link"><code>R/HelperFunctions.R</code></a></small>
     <div class="hidden name"><code>listAppend.Rd</code></div>
     </div>
 
@@ -183,50 +131,46 @@ <h1>join two lists</h1>
     <p>join two lists</p>
     </div>
 
-    <pre class="usage"><span class='fu'>listAppend</span><span class='op'>(</span><span class='va'>a</span>, <span class='va'>b</span><span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">listAppend</span><span class="op">(</span><span class="va">a</span>, <span class="va">b</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>a</dt>
+<dd><p>A list</p></dd>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>a</th>
-      <td><p>A list</p></td>
-    </tr>
-    <tr>
-      <th>b</th>
-      <td><p>Another list</p></td>
-    </tr>
-    </table>
 
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<dt>b</dt>
+<dd><p>Another list</p></dd>
 
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>This function joins two lists</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/listCartesian.html b/docs/reference/listCartesian.html
new file mode 100644
index 000000000..5c5c573b7
--- /dev/null
+++ b/docs/reference/listCartesian.html
@@ -0,0 +1,174 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Cartesian product — listCartesian • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Cartesian product — listCartesian"><meta property="og:description" content="Computes the Cartesian product of all the combinations of elements in a list"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Cartesian product</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SklearnClassifierHelpers.R" class="external-link"><code>R/SklearnClassifierHelpers.R</code></a></small>
+    <div class="hidden name"><code>listCartesian.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>Computes the Cartesian product of all the combinations of elements in a list</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">listCartesian</span><span class="op">(</span><span class="va">allList</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>allList</dt>
+<dd><p>a list of lists</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A list with all possible combinations from the input list of lists</p>
+    </div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/loadEnsemblePlpModel.html b/docs/reference/loadEnsemblePlpModel.html
deleted file mode 100644
index 117c7a11c..000000000
--- a/docs/reference/loadEnsemblePlpModel.html
+++ /dev/null
@@ -1,222 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>loads the Ensmeble plp model and return a model list — loadEnsemblePlpModel • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="loads the Ensmeble plp model and return a model list — loadEnsemblePlpModel" />
-<meta property="og:description" content="loads the Ensmeble plp model and return a model list" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>loads the Ensmeble plp model and return a model list</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Ensemble.R'><code>R/Ensemble.R</code></a></small>
-    <div class="hidden name"><code>loadEnsemblePlpModel.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>loads the Ensmeble plp model and return a model list</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>loadEnsemblePlpModel</span><span class='op'>(</span><span class='va'>dirPath</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>dirPath</th>
-      <td><p>The location of the model</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Loads a plp model list that was saved using <code><a href='savePlpModel.html'>savePlpModel()</a></code></p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/loadEnsemblePlpResult.html b/docs/reference/loadEnsemblePlpResult.html
deleted file mode 100644
index 231ea5eb7..000000000
--- a/docs/reference/loadEnsemblePlpResult.html
+++ /dev/null
@@ -1,222 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>loads the Ensemble plp results — loadEnsemblePlpResult • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="loads the Ensemble plp results — loadEnsemblePlpResult" />
-<meta property="og:description" content="loads the Ensemble plp results" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>loads the Ensemble plp results</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Ensemble.R'><code>R/Ensemble.R</code></a></small>
-    <div class="hidden name"><code>loadEnsemblePlpResult.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>loads the Ensemble plp results</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>loadEnsemblePlpResult</span><span class='op'>(</span><span class='va'>dirPath</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>dirPath</th>
-      <td><p>The location of the model</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Loads a plp model list that was saved using <code>saveEnsemblePlpResults()</code></p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/loadPlpAnalysesJson.html b/docs/reference/loadPlpAnalysesJson.html
index 513a9a062..4d1a0a0cc 100644
--- a/docs/reference/loadPlpAnalysesJson.html
+++ b/docs/reference/loadPlpAnalysesJson.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Load the multiple prediction json settings from a file — loadPlpAnalysesJson • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Load the multiple prediction json settings from a file — loadPlpAnalysesJson" />
-<meta property="og:description" content="Load the multiple prediction json settings from a file" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Load the multiple prediction json settings from a file — loadPlpAnalysesJson • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Load the multiple prediction json settings from a file — loadPlpAnalysesJson"><meta property="og:description" content="Load the multiple prediction json settings from a file"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Load the multiple prediction json settings from a file</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/RunMultiplePlp.R'><code>R/RunMultiplePlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/RunMultiplePlp.R" class="external-link"><code>R/RunMultiplePlp.R</code></a></small>
     <div class="hidden name"><code>loadPlpAnalysesJson.Rd</code></div>
     </div>
 
@@ -183,53 +131,51 @@ <h1>Load the multiple prediction json settings from a file</h1>
     <p>Load the multiple prediction json settings from a file</p>
     </div>
 
-    <pre class="usage"><span class='fu'>loadPlpAnalysesJson</span><span class='op'>(</span><span class='va'>jsonFileLocation</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>jsonFileLocation</th>
-      <td><p>The location of the file 'predictionAnalysisList.json' with the modelDesignList</p></td>
-    </tr>
-    </table>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">loadPlpAnalysesJson</span><span class="op">(</span><span class="va">jsonFileLocation</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>jsonFileLocation</dt>
+<dd><p>The location of the file 'predictionAnalysisList.json' with the modelDesignList</p></dd>
 
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>This function interprets a json with the multiple prediction settings and creates a list 
 that can be combined with connection settings to run a multiple prediction study</p>
+    </div>
 
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>modelDesignList</span> <span class='op'>&lt;-</span> <span class='fu'>loadPlpAnalysesJson</span><span class='op'>(</span><span class='st'>'location of json settings'</span><span class='op'>)</span><span class='op'>$</span><span class='va'>analysis</span>
-<span class='op'>}</span>
-
-</div></pre>
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="kw">if</span> <span class="op">(</span><span class="cn">FALSE</span><span class="op">)</span> <span class="op">{</span></span></span>
+<span class="r-in"><span><span class="va">modelDesignList</span> <span class="op">&lt;-</span> <span class="fu">loadPlpAnalysesJson</span><span class="op">(</span><span class="st">'location of json settings'</span><span class="op">)</span><span class="op">$</span><span class="va">analysis</span></span></span>
+<span class="r-in"><span><span class="op">}</span></span></span>
+<span class="r-in"><span></span></span>
+</code></pre></div>
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/loadPlpData.html b/docs/reference/loadPlpData.html
index a8da4e124..ab34cac18 100644
--- a/docs/reference/loadPlpData.html
+++ b/docs/reference/loadPlpData.html
@@ -1,68 +1,13 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Load the cohort data from a folder — loadPlpData • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Load the cohort data from a folder — loadPlpData" />
-<meta property="og:description" content="loadPlpData loads an object of type plpData from a folder in the file
-system." />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Load the cohort data from a folder — loadPlpData • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Load the cohort data from a folder — loadPlpData"><meta property="og:description" content="loadPlpData loads an object of type plpData from a folder in the file
+system."><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -73,13 +18,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -95,13 +39,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -114,10 +57,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -126,13 +69,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -140,43 +86,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Load the cohort data from a folder</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SaveLoadPlp.R'><code>R/SaveLoadPlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SaveLoadPlp.R" class="external-link"><code>R/SaveLoadPlp.R</code></a></small>
     <div class="hidden name"><code>loadPlpData.Rd</code></div>
     </div>
 
@@ -185,57 +133,59 @@ <h1>Load the cohort data from a folder</h1>
 system.</p>
     </div>
 
-    <pre class="usage"><span class='fu'>loadPlpData</span><span class='op'>(</span><span class='va'>file</span>, readOnly <span class='op'>=</span> <span class='cn'>TRUE</span><span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">loadPlpData</span><span class="op">(</span><span class="va">file</span>, readOnly <span class="op">=</span> <span class="cn">TRUE</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>file</th>
-      <td><p>The name of the folder containing the data.</p></td>
-    </tr>
-    <tr>
-      <th>readOnly</th>
-      <td><p>If true, the data is opened read only.</p></td>
-    </tr>
-    </table>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>file</dt>
+<dd><p>The name of the folder containing the data.</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
 
-    <p>An object of class plpData.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<dt>readOnly</dt>
+<dd><p>If true, the data is opened read only.</p></dd>
 
-    <p>The data will be written to a set of files in the folder specified by the user.</p>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='co'># todo</span>
+<p>An object of class plpData.</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>The data will be written to a set of files in the folder specified by the user.</p>
+    </div>
 
-</div></pre>
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="co"># todo</span></span></span>
+<span class="r-in"><span></span></span>
+<span class="r-in"></span>
+</code></pre></div>
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/loadPlpFromCsv.html b/docs/reference/loadPlpFromCsv.html
deleted file mode 100644
index c51564912..000000000
--- a/docs/reference/loadPlpFromCsv.html
+++ /dev/null
@@ -1,222 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Loads parts of the plp result saved as csv files for transparent sharing — loadPlpFromCsv • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Loads parts of the plp result saved as csv files for transparent sharing — loadPlpFromCsv" />
-<meta property="og:description" content="Loads parts of the plp result saved as csv files for transparent sharing" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Loads parts of the plp result saved as csv files for transparent sharing</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SaveLoadPlp.R'><code>R/SaveLoadPlp.R</code></a></small>
-    <div class="hidden name"><code>loadPlpFromCsv.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Loads parts of the plp result saved as csv files for transparent sharing</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>loadPlpFromCsv</span><span class='op'>(</span><span class='va'>dirPath</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>dirPath</th>
-      <td><p>The directory with the results as csv files</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Load the main results from csv files into a runPlp object</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/loadPlpModel.html b/docs/reference/loadPlpModel.html
index e78445f92..3248a457e 100644
--- a/docs/reference/loadPlpModel.html
+++ b/docs/reference/loadPlpModel.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>loads the plp model — loadPlpModel • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="loads the plp model — loadPlpModel" />
-<meta property="og:description" content="loads the plp model" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>loads the plp model — loadPlpModel • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="loads the plp model — loadPlpModel"><meta property="og:description" content="loads the plp model"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>loads the plp model</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SaveLoadPlp.R'><code>R/SaveLoadPlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SaveLoadPlp.R" class="external-link"><code>R/SaveLoadPlp.R</code></a></small>
     <div class="hidden name"><code>loadPlpModel.Rd</code></div>
     </div>
 
@@ -183,46 +131,42 @@ <h1>loads the plp model</h1>
     <p>loads the plp model</p>
     </div>
 
-    <pre class="usage"><span class='fu'>loadPlpModel</span><span class='op'>(</span><span class='va'>dirPath</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>dirPath</th>
-      <td><p>The location of the model</p></td>
-    </tr>
-    </table>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">loadPlpModel</span><span class="op">(</span><span class="va">dirPath</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>dirPath</dt>
+<dd><p>The location of the model</p></dd>
 
-    <p>Loads a plp model that was saved using <code><a href='savePlpModel.html'>savePlpModel()</a></code></p>
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>Loads a plp model that was saved using <code><a href="savePlpModel.html">savePlpModel()</a></code></p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/loadPlpResult.html b/docs/reference/loadPlpResult.html
index 961c710ec..25327ba94 100644
--- a/docs/reference/loadPlpResult.html
+++ b/docs/reference/loadPlpResult.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Loads the evalaution dataframe — loadPlpResult • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Loads the evalaution dataframe — loadPlpResult" />
-<meta property="og:description" content="Loads the evalaution dataframe" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Loads the evalaution dataframe — loadPlpResult • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Loads the evalaution dataframe — loadPlpResult"><meta property="og:description" content="Loads the evalaution dataframe"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Loads the evalaution dataframe</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SaveLoadPlp.R'><code>R/SaveLoadPlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SaveLoadPlp.R" class="external-link"><code>R/SaveLoadPlp.R</code></a></small>
     <div class="hidden name"><code>loadPlpResult.Rd</code></div>
     </div>
 
@@ -183,46 +131,42 @@ <h1>Loads the evalaution dataframe</h1>
     <p>Loads the evalaution dataframe</p>
     </div>
 
-    <pre class="usage"><span class='fu'>loadPlpResult</span><span class='op'>(</span><span class='va'>dirPath</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>dirPath</th>
-      <td><p>The directory where the evaluation was saved</p></td>
-    </tr>
-    </table>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">loadPlpResult</span><span class="op">(</span><span class="va">dirPath</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>dirPath</dt>
+<dd><p>The directory where the evaluation was saved</p></dd>
 
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Loads the evaluation</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/loadPlpShareable.html b/docs/reference/loadPlpShareable.html
index 774b07d56..a69479f3d 100644
--- a/docs/reference/loadPlpShareable.html
+++ b/docs/reference/loadPlpShareable.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Loads the plp result saved as json/csv files for transparent sharing — loadPlpShareable • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Loads the plp result saved as json/csv files for transparent sharing — loadPlpShareable" />
-<meta property="og:description" content="Loads the plp result saved as json/csv files for transparent sharing" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Loads the plp result saved as json/csv files for transparent sharing — loadPlpShareable • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Loads the plp result saved as json/csv files for transparent sharing — loadPlpShareable"><meta property="og:description" content="Loads the plp result saved as json/csv files for transparent sharing"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Loads the plp result saved as json/csv files for transparent sharing</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SaveLoadPlp.R'><code>R/SaveLoadPlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SaveLoadPlp.R" class="external-link"><code>R/SaveLoadPlp.R</code></a></small>
     <div class="hidden name"><code>loadPlpShareable.Rd</code></div>
     </div>
 
@@ -183,46 +131,42 @@ <h1>Loads the plp result saved as json/csv files for transparent sharing</h1>
     <p>Loads the plp result saved as json/csv files for transparent sharing</p>
     </div>
 
-    <pre class="usage"><span class='fu'>loadPlpShareable</span><span class='op'>(</span><span class='va'>loadDirectory</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>loadDirectory</th>
-      <td><p>The directory with the results as json/csv files</p></td>
-    </tr>
-    </table>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">loadPlpShareable</span><span class="op">(</span><span class="va">loadDirectory</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>loadDirectory</dt>
+<dd><p>The directory with the results as json/csv files</p></dd>
 
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Load the main results from json/csv files into a runPlp object</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/loadPrediction.html b/docs/reference/loadPrediction.html
index f2ec47d45..a4715f9c5 100644
--- a/docs/reference/loadPrediction.html
+++ b/docs/reference/loadPrediction.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Loads the prediciton dataframe to csv — loadPrediction • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Loads the prediciton dataframe to csv — loadPrediction" />
-<meta property="og:description" content="Loads the prediciton dataframe to csv" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Loads the prediciton dataframe to csv — loadPrediction • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Loads the prediciton dataframe to csv — loadPrediction"><meta property="og:description" content="Loads the prediciton dataframe to csv"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Loads the prediciton dataframe to csv</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SaveLoadPlp.R'><code>R/SaveLoadPlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SaveLoadPlp.R" class="external-link"><code>R/SaveLoadPlp.R</code></a></small>
     <div class="hidden name"><code>loadPrediction.Rd</code></div>
     </div>
 
@@ -183,46 +131,42 @@ <h1>Loads the prediciton dataframe to csv</h1>
     <p>Loads the prediciton dataframe to csv</p>
     </div>
 
-    <pre class="usage"><span class='fu'>loadPrediction</span><span class='op'>(</span><span class='va'>fileLocation</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>fileLocation</th>
-      <td><p>The location with the saved prediction</p></td>
-    </tr>
-    </table>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">loadPrediction</span><span class="op">(</span><span class="va">fileLocation</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>fileLocation</dt>
+<dd><p>The location with the saved prediction</p></dd>
 
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Loads the prediciton  RDS file</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/loadPredictionAnalysisList.html b/docs/reference/loadPredictionAnalysisList.html
deleted file mode 100644
index 0ae126bb7..000000000
--- a/docs/reference/loadPredictionAnalysisList.html
+++ /dev/null
@@ -1,240 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Load the multiple prediction json settings from a file — loadPredictionAnalysisList • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Load the multiple prediction json settings from a file — loadPredictionAnalysisList" />
-<meta property="og:description" content="Load the multiple prediction json settings from a file" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Load the multiple prediction json settings from a file</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/RunMultiplePlp.R'><code>R/RunMultiplePlp.R</code></a></small>
-    <div class="hidden name"><code>loadPredictionAnalysisList.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Load the multiple prediction json settings from a file</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>loadPredictionAnalysisList</span><span class='op'>(</span><span class='va'>predictionAnalysisListFile</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>predictionAnalysisListFile</th>
-      <td><p>The prediciton specification json extracted from atlas.</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>This function interprets a json with the multiple prediction settings and creates a list 
-that can be combined with connection settings to run a multiple prediction study</p>
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>predictionAnalysisList</span> <span class='op'>&lt;-</span> <span class='fu'>loadPredictionAnalysisList</span><span class='op'>(</span><span class='st'>'./predictionStudyAnalyses.json'</span><span class='op'>)</span>
-<span class='va'>predictionAnalysisList</span><span class='op'>$</span><span class='va'>connectionDetails</span> <span class='op'>=</span> <span class='va'>connectionDetails</span>
-<span class='va'>predictionAnalysisList</span><span class='op'>$</span><span class='va'>cdmDatabaseSchema</span> <span class='op'>=</span> <span class='va'>cdmDatabaseSchema</span>
-<span class='va'>predictionAnalysisList</span><span class='op'>$</span><span class='va'>cdmDatabaseName</span> <span class='op'>=</span> <span class='va'>cdmDatabaseName</span>
-<span class='va'>predictionAnalysisList</span><span class='op'>$</span><span class='va'>oracleTempSchema</span> <span class='op'>=</span> <span class='va'>oracleTempSchema</span>
-<span class='va'>predictionAnalysisList</span><span class='op'>$</span><span class='va'>cohortDatabaseSchema</span> <span class='op'>=</span> <span class='va'>cohortDatabaseSchema</span>
-<span class='va'>predictionAnalysisList</span><span class='op'>$</span><span class='va'>cohortTable</span> <span class='op'>=</span> <span class='va'>cohortTable</span>
-<span class='va'>predictionAnalysisList</span><span class='op'>$</span><span class='va'>outcomeDatabaseSchema</span> <span class='op'>=</span> <span class='va'>outcomeDatabaseSchema</span>
-<span class='va'>predictionAnalysisList</span><span class='op'>$</span><span class='va'>outcomeTable</span> <span class='op'>=</span> <span class='va'>outcomeTable</span>
-<span class='va'>predictionAnalysisList</span><span class='op'>$</span><span class='va'>cdmVersion</span> <span class='op'>=</span> <span class='va'>cdmVersion</span>
-<span class='va'>predictionAnalysisList</span><span class='op'>$</span><span class='va'>outputFolder</span> <span class='op'>=</span> <span class='va'>outputFolder</span>
-<span class='va'>result</span> <span class='op'>&lt;-</span> <span class='fu'><a href='https://rdrr.io/r/base/do.call.html'>do.call</a></span><span class='op'>(</span><span class='va'>runPlpAnalyses</span>, <span class='va'>predictionAnalysisList</span><span class='op'>)</span>
-<span class='op'>}</span>
-
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/migrateDataModel.html b/docs/reference/migrateDataModel.html
new file mode 100644
index 000000000..5810fedca
--- /dev/null
+++ b/docs/reference/migrateDataModel.html
@@ -0,0 +1,180 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Migrate Data model — migrateDataModel • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Migrate Data model — migrateDataModel"><meta property="og:description" content="Migrate data from current state to next state
+It is strongly advised that you have a backup of all data (either sqlite files, a backup database (in the case you
+are using a postgres backend) or have kept the csv/zip files from your data generation."><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Migrate Data model</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/DatabaseMigration.R" class="external-link"><code>R/DatabaseMigration.R</code></a></small>
+    <div class="hidden name"><code>migrateDataModel.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>Migrate data from current state to next state</p>
+<p>It is strongly advised that you have a backup of all data (either sqlite files, a backup database (in the case you
+are using a postgres backend) or have kept the csv/zip files from your data generation.</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">migrateDataModel</span><span class="op">(</span><span class="va">connectionDetails</span>, <span class="va">databaseSchema</span>, tablePrefix <span class="op">=</span> <span class="st">""</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>connectionDetails</dt>
+<dd><p>DatabaseConnector connection details object</p></dd>
+
+
+<dt>databaseSchema</dt>
+<dd><p>String schema where database schema lives</p></dd>
+
+
+<dt>tablePrefix</dt>
+<dd><p>(Optional) Use if a table prefix is used before table names (e.g. "cd_")</p></dd>
+
+</dl></div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/modelBasedConcordance.html b/docs/reference/modelBasedConcordance.html
index 2665220c4..b18d502df 100644
--- a/docs/reference/modelBasedConcordance.html
+++ b/docs/reference/modelBasedConcordance.html
@@ -1,70 +1,14 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculate the model-based concordance, which is a calculation of the expected discrimination performance of a model under the assumption the model predicts the "TRUE" outcome
-as detailed in van Klaveren et al. https://pubmed.ncbi.nlm.nih.gov/27251001/ — modelBasedConcordance • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculate the model-based concordance, which is a calculation of the expected discrimination performance of a model under the assumption the model predicts the "TRUE" outcome
-as detailed in van Klaveren et al. https://pubmed.ncbi.nlm.nih.gov/27251001/ — modelBasedConcordance" />
-<meta property="og:description" content="Calculate the model-based concordance, which is a calculation of the expected discrimination performance of a model under the assumption the model predicts the &quot;TRUE&quot; outcome
-as detailed in van Klaveren et al. https://pubmed.ncbi.nlm.nih.gov/27251001/" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculate the model-based concordance, which is a calculation of the expected discrimination performance of a model under the assumption the model predicts the "TRUE" outcome
+as detailed in van Klaveren et al. https://pubmed.ncbi.nlm.nih.gov/27251001/ — modelBasedConcordance • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculate the model-based concordance, which is a calculation of the expected discrimination performance of a model under the assumption the model predicts the " true outcome as detailed in van klaveren et al. https: modelbasedconcordance><meta property="og:description" content='Calculate the model-based concordance, which is a calculation of the expected discrimination performance of a model under the assumption the model predicts the "TRUE" outcome
+as detailed in van Klaveren et al. https://pubmed.ncbi.nlm.nih.gov/27251001/'><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -75,13 +19,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -97,13 +40,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -116,10 +58,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -128,13 +70,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -142,44 +87,46 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculate the model-based concordance, which is a calculation of the expected discrimination performance of a model under the assumption the model predicts the "TRUE" outcome
 as detailed in van Klaveren et al. https://pubmed.ncbi.nlm.nih.gov/27251001/</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/EvaluatePlp.R'><code>R/EvaluatePlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/EvaluatePlp.R" class="external-link"><code>R/EvaluatePlp.R</code></a></small>
     <div class="hidden name"><code>modelBasedConcordance.Rd</code></div>
     </div>
 
@@ -188,49 +135,48 @@ <h1>Calculate the model-based concordance, which is a calculation of the expecte
 as detailed in van Klaveren et al. https://pubmed.ncbi.nlm.nih.gov/27251001/</p>
     </div>
 
-    <pre class="usage"><span class='fu'>modelBasedConcordance</span><span class='op'>(</span><span class='va'>prediction</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>prediction</th>
-      <td><p>the prediction object found in the plpResult object</p></td>
-    </tr>
-    </table>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">modelBasedConcordance</span><span class="op">(</span><span class="va">prediction</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>prediction</dt>
+<dd><p>the prediction object found in the plpResult object</p></dd>
 
-    <p>model-based concordance value</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
+<p>model-based concordance value</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculate the model-based concordance</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/negativeLikelihoodRatio.html b/docs/reference/negativeLikelihoodRatio.html
index 9082c2e50..a1fa5b4f2 100644
--- a/docs/reference/negativeLikelihoodRatio.html
+++ b/docs/reference/negativeLikelihoodRatio.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculate the negativeLikelihoodRatio — negativeLikelihoodRatio • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculate the negativeLikelihoodRatio — negativeLikelihoodRatio" />
-<meta property="og:description" content="Calculate the negativeLikelihoodRatio" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculate the negativeLikelihoodRatio — negativeLikelihoodRatio • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculate the negativeLikelihoodRatio — negativeLikelihoodRatio"><meta property="og:description" content="Calculate the negativeLikelihoodRatio"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculate the negativeLikelihoodRatio</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ThresholdSummary.R'><code>R/ThresholdSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ThresholdSummary.R" class="external-link"><code>R/ThresholdSummary.R</code></a></small>
     <div class="hidden name"><code>negativeLikelihoodRatio.Rd</code></div>
     </div>
 
@@ -183,61 +131,60 @@ <h1>Calculate the negativeLikelihoodRatio</h1>
     <p>Calculate the negativeLikelihoodRatio</p>
     </div>
 
-    <pre class="usage"><span class='fu'>negativeLikelihoodRatio</span><span class='op'>(</span><span class='va'>TP</span>, <span class='va'>TN</span>, <span class='va'>FN</span>, <span class='va'>FP</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>TP</th>
-      <td><p>Number of true positives</p></td>
-    </tr>
-    <tr>
-      <th>TN</th>
-      <td><p>Number of true negatives</p></td>
-    </tr>
-    <tr>
-      <th>FN</th>
-      <td><p>Number of false negatives</p></td>
-    </tr>
-    <tr>
-      <th>FP</th>
-      <td><p>Number of false positives</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>negativeLikelihoodRatio value</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">negativeLikelihoodRatio</span><span class="op">(</span><span class="va">TP</span>, <span class="va">TN</span>, <span class="va">FN</span>, <span class="va">FP</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>TP</dt>
+<dd><p>Number of true positives</p></dd>
+
+
+<dt>TN</dt>
+<dd><p>Number of true negatives</p></dd>
+
+
+<dt>FN</dt>
+<dd><p>Number of false negatives</p></dd>
+
+
+<dt>FP</dt>
+<dd><p>Number of false positives</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>negativeLikelihoodRatio value</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculate the negativeLikelihoodRatio</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/negativePredictiveValue.html b/docs/reference/negativePredictiveValue.html
index 5fb3da3af..816870bb3 100644
--- a/docs/reference/negativePredictiveValue.html
+++ b/docs/reference/negativePredictiveValue.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculate the negativePredictiveValue — negativePredictiveValue • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculate the negativePredictiveValue — negativePredictiveValue" />
-<meta property="og:description" content="Calculate the negativePredictiveValue" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculate the negativePredictiveValue — negativePredictiveValue • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculate the negativePredictiveValue — negativePredictiveValue"><meta property="og:description" content="Calculate the negativePredictiveValue"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculate the negativePredictiveValue</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ThresholdSummary.R'><code>R/ThresholdSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ThresholdSummary.R" class="external-link"><code>R/ThresholdSummary.R</code></a></small>
     <div class="hidden name"><code>negativePredictiveValue.Rd</code></div>
     </div>
 
@@ -183,61 +131,60 @@ <h1>Calculate the negativePredictiveValue</h1>
     <p>Calculate the negativePredictiveValue</p>
     </div>
 
-    <pre class="usage"><span class='fu'>negativePredictiveValue</span><span class='op'>(</span><span class='va'>TP</span>, <span class='va'>TN</span>, <span class='va'>FN</span>, <span class='va'>FP</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>TP</th>
-      <td><p>Number of true positives</p></td>
-    </tr>
-    <tr>
-      <th>TN</th>
-      <td><p>Number of true negatives</p></td>
-    </tr>
-    <tr>
-      <th>FN</th>
-      <td><p>Number of false negatives</p></td>
-    </tr>
-    <tr>
-      <th>FP</th>
-      <td><p>Number of false positives</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>negativePredictiveValue value</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">negativePredictiveValue</span><span class="op">(</span><span class="va">TP</span>, <span class="va">TN</span>, <span class="va">FN</span>, <span class="va">FP</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>TP</dt>
+<dd><p>Number of true positives</p></dd>
+
+
+<dt>TN</dt>
+<dd><p>Number of true negatives</p></dd>
+
+
+<dt>FN</dt>
+<dd><p>Number of false negatives</p></dd>
+
+
+<dt>FP</dt>
+<dd><p>Number of false positives</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>negativePredictiveValue value</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculate the negativePredictiveValue</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/outcomeSurvivalPlot.html b/docs/reference/outcomeSurvivalPlot.html
index c3fa6bfb5..fc364c7e5 100644
--- a/docs/reference/outcomeSurvivalPlot.html
+++ b/docs/reference/outcomeSurvivalPlot.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Plot the outcome incidence over time — outcomeSurvivalPlot • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Plot the outcome incidence over time — outcomeSurvivalPlot" />
-<meta property="og:description" content="Plot the outcome incidence over time" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Plot the outcome incidence over time — outcomeSurvivalPlot • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Plot the outcome incidence over time — outcomeSurvivalPlot"><meta property="og:description" content="Plot the outcome incidence over time"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Plot the outcome incidence over time</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Plotting.R'><code>R/Plotting.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Plotting.R" class="external-link"><code>R/Plotting.R</code></a></small>
     <div class="hidden name"><code>outcomeSurvivalPlot.Rd</code></div>
     </div>
 
@@ -183,79 +131,78 @@ <h1>Plot the outcome incidence over time</h1>
     <p>Plot the outcome incidence over time</p>
     </div>
 
-    <pre class="usage"><span class='fu'>outcomeSurvivalPlot</span><span class='op'>(</span>
-  <span class='va'>plpData</span>,
-  <span class='va'>outcomeId</span>,
-  populationSettings <span class='op'>=</span> <span class='fu'><a href='createStudyPopulationSettings.html'>createStudyPopulationSettings</a></span><span class='op'>(</span>binary <span class='op'>=</span> <span class='cn'>T</span>, includeAllOutcomes <span class='op'>=</span>
-    <span class='cn'>T</span>, firstExposureOnly <span class='op'>=</span> <span class='cn'>FALSE</span>, washoutPeriod <span class='op'>=</span> <span class='fl'>0</span>, removeSubjectsWithPriorOutcome <span class='op'>=</span>
-    <span class='cn'>TRUE</span>, priorOutcomeLookback <span class='op'>=</span> <span class='fl'>99999</span>, requireTimeAtRisk <span class='op'>=</span> <span class='cn'>F</span>, riskWindowStart <span class='op'>=</span> <span class='fl'>1</span>,
-    startAnchor <span class='op'>=</span> <span class='st'>"cohort start"</span>, riskWindowEnd <span class='op'>=</span> <span class='fl'>3650</span>, endAnchor <span class='op'>=</span> <span class='st'>"cohort start"</span><span class='op'>)</span>,
-  riskTable <span class='op'>=</span> <span class='cn'>T</span>,
-  confInt <span class='op'>=</span> <span class='cn'>T</span>,
-  yLabel <span class='op'>=</span> <span class='st'>"Fraction of those who are outcome free in target population"</span>
-<span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">outcomeSurvivalPlot</span><span class="op">(</span></span>
+<span>  <span class="va">plpData</span>,</span>
+<span>  <span class="va">outcomeId</span>,</span>
+<span>  populationSettings <span class="op">=</span> <span class="fu"><a href="createStudyPopulationSettings.html">createStudyPopulationSettings</a></span><span class="op">(</span>binary <span class="op">=</span> <span class="cn">T</span>, includeAllOutcomes <span class="op">=</span> <span class="cn">T</span>,</span>
+<span>    firstExposureOnly <span class="op">=</span> <span class="cn">FALSE</span>, washoutPeriod <span class="op">=</span> <span class="fl">0</span>, removeSubjectsWithPriorOutcome <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>    priorOutcomeLookback <span class="op">=</span> <span class="fl">99999</span>, requireTimeAtRisk <span class="op">=</span> <span class="cn">F</span>, riskWindowStart <span class="op">=</span> <span class="fl">1</span>, startAnchor</span>
+<span>    <span class="op">=</span> <span class="st">"cohort start"</span>, riskWindowEnd <span class="op">=</span> <span class="fl">3650</span>, endAnchor <span class="op">=</span> <span class="st">"cohort start"</span><span class="op">)</span>,</span>
+<span>  riskTable <span class="op">=</span> <span class="cn">T</span>,</span>
+<span>  confInt <span class="op">=</span> <span class="cn">T</span>,</span>
+<span>  yLabel <span class="op">=</span> <span class="st">"Fraction of those who are outcome free in target population"</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpData</dt>
+<dd><p>The plpData object returned by running getPlpData()</p></dd>
+
+
+<dt>outcomeId</dt>
+<dd><p>The cohort id corresponding to the outcome</p></dd>
+
+
+<dt>populationSettings</dt>
+<dd><p>The population settings created using <code>createStudyPopulationSettings</code></p></dd>
+
+
+<dt>riskTable</dt>
+<dd><p>(binary) Whether to include a table at the bottom  of the plot showing the number of people at risk over time</p></dd>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpData</th>
-      <td><p>The plpData object returned by running getPlpData()</p></td>
-    </tr>
-    <tr>
-      <th>outcomeId</th>
-      <td><p>The cohort id corresponding to the outcome</p></td>
-    </tr>
-    <tr>
-      <th>populationSettings</th>
-      <td><p>The population settings created using <code>createStudyPopulationSettings</code></p></td>
-    </tr>
-    <tr>
-      <th>riskTable</th>
-      <td><p>(binary) Whether to include a table at the bottom  of the plot showing the number of people at risk over time</p></td>
-    </tr>
-    <tr>
-      <th>confInt</th>
-      <td><p>(binary) Whether to include a confidence interval</p></td>
-    </tr>
-    <tr>
-      <th>yLabel</th>
-      <td><p>(string) The label for the y-axis</p></td>
-    </tr>
-    </table>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+<dt>confInt</dt>
+<dd><p>(binary) Whether to include a confidence interval</p></dd>
 
-    <p>TRUE if it ran</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
 
+<dt>yLabel</dt>
+<dd><p>(string) The label for the y-axis</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>TRUE if it ran</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>This creates a survival plot that can be used to pick a suitable time-at-risk period</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/personSplitter.html b/docs/reference/personSplitter.html
deleted file mode 100644
index 8b337e3dc..000000000
--- a/docs/reference/personSplitter.html
+++ /dev/null
@@ -1,244 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Split data into random subsets stratified by class — personSplitter • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Split data into random subsets stratified by class — personSplitter" />
-<meta property="og:description" content="Split data into random subsets stratified by class" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Split data into random subsets stratified by class</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/DataSplitting.R'><code>R/DataSplitting.R</code></a></small>
-    <div class="hidden name"><code>personSplitter.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Split data into random subsets stratified by class</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>personSplitter</span><span class='op'>(</span><span class='va'>population</span>, test <span class='op'>=</span> <span class='fl'>0.3</span>, train <span class='op'>=</span> <span class='cn'>NULL</span>, nfold <span class='op'>=</span> <span class='fl'>3</span>, seed <span class='op'>=</span> <span class='cn'>NULL</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>population</th>
-      <td><p>An object created using createStudyPopulation().</p></td>
-    </tr>
-    <tr>
-      <th>test</th>
-      <td><p>A real number between 0 and 1 indicating the test set fraction of the data</p></td>
-    </tr>
-    <tr>
-      <th>train</th>
-      <td><p>A real number between 0 and 1 indicating the train set fraction of the data.
-If not set train is equal to 1 - test</p></td>
-    </tr>
-    <tr>
-      <th>nfold</th>
-      <td><p>An integer &gt;= 1 specifying the number of folds used in cross validation</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>If set a fixed seed is used, otherwise a random split is performed</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A dataframe containing the columns: rowId and index</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Returns a dataframe of rowIds and indexes with a -1 index indicating the rowId belongs to the test
-set and a positive integer index value indicating the rowId's cross valiation fold within the train
-set.</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/pfi.html b/docs/reference/pfi.html
index 61a9d5dc6..8a039c50a 100644
--- a/docs/reference/pfi.html
+++ b/docs/reference/pfi.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>pfi — pfi • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="pfi — pfi" />
-<meta property="og:description" content="Calculate the permutation feature importance for a PLP model." />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>pfi — pfi • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="pfi — pfi"><meta property="og:description" content="Calculate the permutation feature importance for a PLP model."><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>pfi</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/FeatureImportance.R'><code>R/FeatureImportance.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/FeatureImportance.R" class="external-link"><code>R/FeatureImportance.R</code></a></small>
     <div class="hidden name"><code>pfi.Rd</code></div>
     </div>
 
@@ -183,87 +131,86 @@ <h1>pfi</h1>
     <p>Calculate the permutation feature importance for a PLP model.</p>
     </div>
 
-    <pre class="usage"><span class='fu'>pfi</span><span class='op'>(</span>
-  <span class='va'>plpResult</span>,
-  <span class='va'>population</span>,
-  <span class='va'>plpData</span>,
-  repeats <span class='op'>=</span> <span class='fl'>1</span>,
-  covariates <span class='op'>=</span> <span class='cn'>NULL</span>,
-  cores <span class='op'>=</span> <span class='cn'>NULL</span>,
-  log <span class='op'>=</span> <span class='cn'>NULL</span>,
-  logthreshold <span class='op'>=</span> <span class='st'>"INFO"</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpResult</th>
-      <td><p>An object of type <code>runPlp</code></p></td>
-    </tr>
-    <tr>
-      <th>population</th>
-      <td><p>The population created using createStudyPopulation() who will have their risks predicted</p></td>
-    </tr>
-    <tr>
-      <th>plpData</th>
-      <td><p>An object of type <code>plpData</code> - the patient level prediction
-data extracted from the CDM.</p></td>
-    </tr>
-    <tr>
-      <th>repeats</th>
-      <td><p>The number of times to permute each covariate</p></td>
-    </tr>
-    <tr>
-      <th>covariates</th>
-      <td><p>A vector of covariates to calculate the pfi for.  If NULL it uses all covariates included in the model.</p></td>
-    </tr>
-    <tr>
-      <th>cores</th>
-      <td><p>Number of cores to use when running this (it runs in parallel)</p></td>
-    </tr>
-    <tr>
-      <th>log</th>
-      <td><p>A location to save the log for running pfi</p></td>
-    </tr>
-    <tr>
-      <th>logthreshold</th>
-      <td><p>The log threshold (e.g., INFO, TRACE, ...)</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A dataframe with the covariateIds and the pfi (change in AUC caused by permuting the covariate) value</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">pfi</span><span class="op">(</span></span>
+<span>  <span class="va">plpResult</span>,</span>
+<span>  <span class="va">population</span>,</span>
+<span>  <span class="va">plpData</span>,</span>
+<span>  repeats <span class="op">=</span> <span class="fl">1</span>,</span>
+<span>  covariates <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  cores <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  log <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  logthreshold <span class="op">=</span> <span class="st">"INFO"</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpResult</dt>
+<dd><p>An object of type <code>runPlp</code></p></dd>
+
+
+<dt>population</dt>
+<dd><p>The population created using createStudyPopulation() who will have their risks predicted</p></dd>
+
+
+<dt>plpData</dt>
+<dd><p>An object of type <code>plpData</code> - the patient level prediction
+data extracted from the CDM.</p></dd>
+
 
+<dt>repeats</dt>
+<dd><p>The number of times to permute each covariate</p></dd>
+
+
+<dt>covariates</dt>
+<dd><p>A vector of covariates to calculate the pfi for.  If NULL it uses all covariates included in the model.</p></dd>
+
+
+<dt>cores</dt>
+<dd><p>Number of cores to use when running this (it runs in parallel)</p></dd>
+
+
+<dt>log</dt>
+<dd><p>A location to save the log for running pfi</p></dd>
+
+
+<dt>logthreshold</dt>
+<dd><p>The log threshold (e.g., INFO, TRACE, ...)</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A dataframe with the covariateIds and the pfi (change in AUC caused by permuting the covariate) value</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>The function permutes the each covariate/features &lt;repeats&gt; times and calculates the mean AUC change caused by the permutation.</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/plotDemographicSummary.html b/docs/reference/plotDemographicSummary.html
index 08466abab..8756040c8 100644
--- a/docs/reference/plotDemographicSummary.html
+++ b/docs/reference/plotDemographicSummary.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Plot the Observed vs. expected incidence, by age and gender — plotDemographicSummary • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Plot the Observed vs. expected incidence, by age and gender — plotDemographicSummary" />
-<meta property="og:description" content="Plot the Observed vs. expected incidence, by age and gender" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Plot the Observed vs. expected incidence, by age and gender — plotDemographicSummary • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Plot the Observed vs. expected incidence, by age and gender — plotDemographicSummary"><meta property="og:description" content="Plot the Observed vs. expected incidence, by age and gender"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Plot the Observed vs. expected incidence, by age and gender</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Plotting.R'><code>R/Plotting.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Plotting.R" class="external-link"><code>R/Plotting.R</code></a></small>
     <div class="hidden name"><code>plotDemographicSummary.Rd</code></div>
     </div>
 
@@ -183,70 +131,69 @@ <h1>Plot the Observed vs. expected incidence, by age and gender</h1>
     <p>Plot the Observed vs. expected incidence, by age and gender</p>
     </div>
 
-    <pre class="usage"><span class='fu'>plotDemographicSummary</span><span class='op'>(</span>
-  <span class='va'>plpResult</span>,
-  typeColumn <span class='op'>=</span> <span class='st'>"evaluation"</span>,
-  saveLocation <span class='op'>=</span> <span class='cn'>NULL</span>,
-  fileName <span class='op'>=</span> <span class='st'>"roc.png"</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpResult</th>
-      <td><p>A plp result object as generated using the <code><a href='runPlp.html'>runPlp</a></code> function.</p></td>
-    </tr>
-    <tr>
-      <th>typeColumn</th>
-      <td><p>The name of the column specifying the evaluation type</p></td>
-    </tr>
-    <tr>
-      <th>saveLocation</th>
-      <td><p>Directory to save plot (if NULL plot is not saved)</p></td>
-    </tr>
-    <tr>
-      <th>fileName</th>
-      <td><p>Name of the file to save to plot, for example
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">plotDemographicSummary</span><span class="op">(</span></span>
+<span>  <span class="va">plpResult</span>,</span>
+<span>  typeColumn <span class="op">=</span> <span class="st">"evaluation"</span>,</span>
+<span>  saveLocation <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  fileName <span class="op">=</span> <span class="st">"roc.png"</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpResult</dt>
+<dd><p>A plp result object as generated using the <code><a href="runPlp.html">runPlp</a></code> function.</p></dd>
+
+
+<dt>typeColumn</dt>
+<dd><p>The name of the column specifying the evaluation type</p></dd>
+
+
+<dt>saveLocation</dt>
+<dd><p>Directory to save plot (if NULL plot is not saved)</p></dd>
+
+
+<dt>fileName</dt>
+<dd><p>Name of the file to save to plot, for example
 'plot.png'. See the function <code>ggsave</code> in the ggplot2 package for
-supported file formats.</p></td>
-    </tr>
-    </table>
+supported file formats.</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
-    <p>A ggplot object. Use the <code><a href='https://ggplot2.tidyverse.org/reference/ggsave.html'>ggsave</a></code> function to save to file in a different
+<p>A ggplot object. Use the <code><a href="https://ggplot2.tidyverse.org/reference/ggsave.html" class="external-link">ggsave</a></code> function to save to file in a different
 format.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Create a plot showing the Observed vs. expected incidence, by age and gender
 #'</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/plotF1Measure.html b/docs/reference/plotF1Measure.html
index bbd38b250..aec0a7eae 100644
--- a/docs/reference/plotF1Measure.html
+++ b/docs/reference/plotF1Measure.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Plot the F1 measure efficiency frontier using the sparse thresholdSummary data frame — plotF1Measure • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Plot the F1 measure efficiency frontier using the sparse thresholdSummary data frame — plotF1Measure" />
-<meta property="og:description" content="Plot the F1 measure efficiency frontier using the sparse thresholdSummary data frame" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Plot the F1 measure efficiency frontier using the sparse thresholdSummary data frame — plotF1Measure • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Plot the F1 measure efficiency frontier using the sparse thresholdSummary data frame — plotF1Measure"><meta property="og:description" content="Plot the F1 measure efficiency frontier using the sparse thresholdSummary data frame"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Plot the F1 measure efficiency frontier using the sparse thresholdSummary data frame</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Plotting.R'><code>R/Plotting.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Plotting.R" class="external-link"><code>R/Plotting.R</code></a></small>
     <div class="hidden name"><code>plotF1Measure.Rd</code></div>
     </div>
 
@@ -183,69 +131,68 @@ <h1>Plot the F1 measure efficiency frontier using the sparse thresholdSummary da
     <p>Plot the F1 measure efficiency frontier using the sparse thresholdSummary data frame</p>
     </div>
 
-    <pre class="usage"><span class='fu'>plotF1Measure</span><span class='op'>(</span>
-  <span class='va'>plpResult</span>,
-  typeColumn <span class='op'>=</span> <span class='st'>"evaluation"</span>,
-  saveLocation <span class='op'>=</span> <span class='cn'>NULL</span>,
-  fileName <span class='op'>=</span> <span class='st'>"roc.png"</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpResult</th>
-      <td><p>A plp result object as generated using the <code><a href='runPlp.html'>runPlp</a></code> function.</p></td>
-    </tr>
-    <tr>
-      <th>typeColumn</th>
-      <td><p>The name of the column specifying the evaluation type</p></td>
-    </tr>
-    <tr>
-      <th>saveLocation</th>
-      <td><p>Directory to save plot (if NULL plot is not saved)</p></td>
-    </tr>
-    <tr>
-      <th>fileName</th>
-      <td><p>Name of the file to save to plot, for example
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">plotF1Measure</span><span class="op">(</span></span>
+<span>  <span class="va">plpResult</span>,</span>
+<span>  typeColumn <span class="op">=</span> <span class="st">"evaluation"</span>,</span>
+<span>  saveLocation <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  fileName <span class="op">=</span> <span class="st">"roc.png"</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpResult</dt>
+<dd><p>A plp result object as generated using the <code><a href="runPlp.html">runPlp</a></code> function.</p></dd>
+
+
+<dt>typeColumn</dt>
+<dd><p>The name of the column specifying the evaluation type</p></dd>
+
+
+<dt>saveLocation</dt>
+<dd><p>Directory to save plot (if NULL plot is not saved)</p></dd>
+
+
+<dt>fileName</dt>
+<dd><p>Name of the file to save to plot, for example
 'plot.png'. See the function <code>ggsave</code> in the ggplot2 package for
-supported file formats.</p></td>
-    </tr>
-    </table>
+supported file formats.</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
-    <p>A ggplot object. Use the <code><a href='https://ggplot2.tidyverse.org/reference/ggsave.html'>ggsave</a></code> function to save to file in a different
+<p>A ggplot object. Use the <code><a href="https://ggplot2.tidyverse.org/reference/ggsave.html" class="external-link">ggsave</a></code> function to save to file in a different
 format.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Create a plot showing the F1 measure efficiency frontier using the sparse thresholdSummary data frame</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/plotGeneralizability.html b/docs/reference/plotGeneralizability.html
index 806b1e2da..f91a77613 100644
--- a/docs/reference/plotGeneralizability.html
+++ b/docs/reference/plotGeneralizability.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Plot the train/test generalizability diagnostic — plotGeneralizability • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Plot the train/test generalizability diagnostic — plotGeneralizability" />
-<meta property="og:description" content="Plot the train/test generalizability diagnostic" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Plot the train/test generalizability diagnostic — plotGeneralizability • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Plot the train/test generalizability diagnostic — plotGeneralizability"><meta property="og:description" content="Plot the train/test generalizability diagnostic"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Plot the train/test generalizability diagnostic</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Plotting.R'><code>R/Plotting.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Plotting.R" class="external-link"><code>R/Plotting.R</code></a></small>
     <div class="hidden name"><code>plotGeneralizability.Rd</code></div>
     </div>
 
@@ -183,66 +131,65 @@ <h1>Plot the train/test generalizability diagnostic</h1>
     <p>Plot the train/test generalizability diagnostic</p>
     </div>
 
-    <pre class="usage"><span class='fu'>plotGeneralizability</span><span class='op'>(</span>
-  <span class='va'>covariateSummary</span>,
-  saveLocation <span class='op'>=</span> <span class='cn'>NULL</span>,
-  fileName <span class='op'>=</span> <span class='st'>"Generalizability.png"</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>covariateSummary</th>
-      <td><p>A prediction object as generated using the
-<code><a href='runPlp.html'>runPlp</a></code> function.</p></td>
-    </tr>
-    <tr>
-      <th>saveLocation</th>
-      <td><p>Directory to save plot (if NULL plot is not saved)</p></td>
-    </tr>
-    <tr>
-      <th>fileName</th>
-      <td><p>Name of the file to save to plot, for example
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">plotGeneralizability</span><span class="op">(</span></span>
+<span>  <span class="va">covariateSummary</span>,</span>
+<span>  saveLocation <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  fileName <span class="op">=</span> <span class="st">"Generalizability.png"</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>covariateSummary</dt>
+<dd><p>A prediction object as generated using the
+<code><a href="runPlp.html">runPlp</a></code> function.</p></dd>
+
+
+<dt>saveLocation</dt>
+<dd><p>Directory to save plot (if NULL plot is not saved)</p></dd>
+
+
+<dt>fileName</dt>
+<dd><p>Name of the file to save to plot, for example
 'plot.png'. See the function <code>ggsave</code> in the ggplot2 package for
-supported file formats.</p></td>
-    </tr>
-    </table>
+supported file formats.</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
-    <p>A ggplot object. Use the <code><a href='https://ggplot2.tidyverse.org/reference/ggsave.html'>ggsave</a></code> function to save to file in a different
+<p>A ggplot object. Use the <code><a href="https://ggplot2.tidyverse.org/reference/ggsave.html" class="external-link">ggsave</a></code> function to save to file in a different
 format.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Create a plot showing the train/test generalizability diagnostic
 #'</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/plotLearningCurve.html b/docs/reference/plotLearningCurve.html
index a3a514041..803ab6e4f 100644
--- a/docs/reference/plotLearningCurve.html
+++ b/docs/reference/plotLearningCurve.html
@@ -1,68 +1,13 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>plotLearningCurve — plotLearningCurve • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="plotLearningCurve — plotLearningCurve" />
-<meta property="og:description" content="Create a plot of the learning curve using the object returned
-from createLearningCurve." />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>plotLearningCurve — plotLearningCurve • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="plotLearningCurve — plotLearningCurve"><meta property="og:description" content="Create a plot of the learning curve using the object returned
+from createLearningCurve."><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -73,13 +18,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -95,13 +39,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -114,10 +57,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -126,13 +69,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -140,43 +86,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>plotLearningCurve</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/LearningCurve.R'><code>R/LearningCurve.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/LearningCurve.R" class="external-link"><code>R/LearningCurve.R</code></a></small>
     <div class="hidden name"><code>plotLearningCurve.Rd</code></div>
     </div>
 
@@ -185,96 +133,94 @@ <h1>plotLearningCurve</h1>
 from <code>createLearningCurve</code>.</p>
     </div>
 
-    <pre class="usage"><span class='fu'>plotLearningCurve</span><span class='op'>(</span>
-  <span class='va'>learningCurve</span>,
-  metric <span class='op'>=</span> <span class='st'>"AUROC"</span>,
-  abscissa <span class='op'>=</span> <span class='st'>"events"</span>,
-  plotTitle <span class='op'>=</span> <span class='st'>"Learning Curve"</span>,
-  plotSubtitle <span class='op'>=</span> <span class='cn'>NULL</span>,
-  fileName <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">plotLearningCurve</span><span class="op">(</span></span>
+<span>  <span class="va">learningCurve</span>,</span>
+<span>  metric <span class="op">=</span> <span class="st">"AUROC"</span>,</span>
+<span>  abscissa <span class="op">=</span> <span class="st">"events"</span>,</span>
+<span>  plotTitle <span class="op">=</span> <span class="st">"Learning Curve"</span>,</span>
+<span>  plotSubtitle <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  fileName <span class="op">=</span> <span class="cn">NULL</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>learningCurve</dt>
+<dd><p>An object returned by <code><a href="createLearningCurve.html">createLearningCurve</a></code>
+function.</p></dd>
+
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>learningCurve</th>
-      <td><p>An object returned by <code><a href='createLearningCurve.html'>createLearningCurve</a></code>
-function.</p></td>
-    </tr>
-    <tr>
-      <th>metric</th>
-      <td><p>Specifies the metric to be plotted:</p><ul>
-<li><p><code>'AUROC'</code> - use the area under the Receiver Operating
+<dt>metric</dt>
+<dd><p>Specifies the metric to be plotted:</p><ul><li><p><code>'AUROC'</code> - use the area under the Receiver Operating
     Characteristic curve</p></li>
 <li><p><code>'AUPRC'</code> - use the area under the Precision-Recall curve</p></li>
 <li><p><code>'sBrier'</code> - use the scaled Brier score</p></li>
-</ul></td>
-    </tr>
-    <tr>
-      <th>abscissa</th>
-      <td><p>Specify the abscissa metric to be plotted:</p><ul>
-<li><p><code>'events'</code> - use number of events</p></li>
+</ul></dd>
+
+
+<dt>abscissa</dt>
+<dd><p>Specify the abscissa metric to be plotted:</p><ul><li><p><code>'events'</code> - use number of events</p></li>
 <li><p><code>'observations'</code> - use number of observations</p></li>
-</ul></td>
-    </tr>
-    <tr>
-      <th>plotTitle</th>
-      <td><p>Title of the learning curve plot.</p></td>
-    </tr>
-    <tr>
-      <th>plotSubtitle</th>
-      <td><p>Subtitle of the learning curve plot.</p></td>
-    </tr>
-    <tr>
-      <th>fileName</th>
-      <td><p>Filename of plot to be saved, for example <code>'plot.png'</code>.
+</ul></dd>
+
+
+<dt>plotTitle</dt>
+<dd><p>Title of the learning curve plot.</p></dd>
+
+
+<dt>plotSubtitle</dt>
+<dd><p>Subtitle of the learning curve plot.</p></dd>
+
+
+<dt>fileName</dt>
+<dd><p>Filename of plot to be saved, for example <code>'plot.png'</code>.
 See the function <code>ggsave</code> in the ggplot2 package for supported file 
-formats.</p></td>
-    </tr>
-    </table>
+formats.</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
-    <p>A ggplot object. Use the <code><a href='https://ggplot2.tidyverse.org/reference/ggsave.html'>ggsave</a></code> function to save to 
+<p>A ggplot object. Use the <code><a href="https://ggplot2.tidyverse.org/reference/ggsave.html" class="external-link">ggsave</a></code> function to save to 
 file in a different format.</p>
+    </div>
 
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='co'># create learning curve object</span>
-<span class='va'>learningCurve</span> <span class='op'>&lt;-</span> <span class='fu'><a href='createLearningCurve.html'>createLearningCurve</a></span><span class='op'>(</span><span class='va'>population</span>,
-                                     <span class='va'>plpData</span>,
-                                     <span class='va'>modelSettings</span><span class='op'>)</span>
-<span class='co'># plot the learning curve</span>
-<span class='fu'>plotLearningCurve</span><span class='op'>(</span><span class='va'>learningCurve</span><span class='op'>)</span>
-<span class='op'>}</span>
-
-</div></pre>
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="kw">if</span> <span class="op">(</span><span class="cn">FALSE</span><span class="op">)</span> <span class="op">{</span></span></span>
+<span class="r-in"><span><span class="co"># create learning curve object</span></span></span>
+<span class="r-in"><span><span class="va">learningCurve</span> <span class="op">&lt;-</span> <span class="fu"><a href="createLearningCurve.html">createLearningCurve</a></span><span class="op">(</span><span class="va">population</span>,</span></span>
+<span class="r-in"><span>                                     <span class="va">plpData</span>,</span></span>
+<span class="r-in"><span>                                     <span class="va">modelSettings</span><span class="op">)</span></span></span>
+<span class="r-in"><span><span class="co"># plot the learning curve</span></span></span>
+<span class="r-in"><span><span class="fu">plotLearningCurve</span><span class="op">(</span><span class="va">learningCurve</span><span class="op">)</span></span></span>
+<span class="r-in"><span><span class="op">}</span></span></span>
+<span class="r-in"><span></span></span>
+</code></pre></div>
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/plotPlp.html b/docs/reference/plotPlp.html
index 199ff8253..0ae53ad83 100644
--- a/docs/reference/plotPlp.html
+++ b/docs/reference/plotPlp.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Plot all the PatientLevelPrediction plots — plotPlp • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Plot all the PatientLevelPrediction plots — plotPlp" />
-<meta property="og:description" content="Plot all the PatientLevelPrediction plots" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Plot all the PatientLevelPrediction plots — plotPlp • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Plot all the PatientLevelPrediction plots — plotPlp"><meta property="og:description" content="Plot all the PatientLevelPrediction plots"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Plot all the PatientLevelPrediction plots</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Plotting.R'><code>R/Plotting.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Plotting.R" class="external-link"><code>R/Plotting.R</code></a></small>
     <div class="hidden name"><code>plotPlp.Rd</code></div>
     </div>
 
@@ -183,58 +131,57 @@ <h1>Plot all the PatientLevelPrediction plots</h1>
     <p>Plot all the PatientLevelPrediction plots</p>
     </div>
 
-    <pre class="usage"><span class='fu'>plotPlp</span><span class='op'>(</span><span class='va'>plpResult</span>, saveLocation <span class='op'>=</span> <span class='cn'>NULL</span>, typeColumn <span class='op'>=</span> <span class='st'>"evaluation"</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpResult</th>
-      <td><p>Object returned by the runPlp() function</p></td>
-    </tr>
-    <tr>
-      <th>saveLocation</th>
-      <td><p>Name of the directory where the plots should be saved (NULL means no saving)</p></td>
-    </tr>
-    <tr>
-      <th>typeColumn</th>
-      <td><p>The name of the column specifying the evaluation type 
-(to stratify the plots)</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>TRUE if it ran</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">plotPlp</span><span class="op">(</span><span class="va">plpResult</span>, saveLocation <span class="op">=</span> <span class="cn">NULL</span>, typeColumn <span class="op">=</span> <span class="st">"evaluation"</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpResult</dt>
+<dd><p>Object returned by the runPlp() function</p></dd>
+
+
+<dt>saveLocation</dt>
+<dd><p>Name of the directory where the plots should be saved (NULL means no saving)</p></dd>
+
 
+<dt>typeColumn</dt>
+<dd><p>The name of the column specifying the evaluation type 
+(to stratify the plots)</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>TRUE if it ran</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Create a directory with all the plots</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/plotPrecisionRecall.html b/docs/reference/plotPrecisionRecall.html
index fba15177f..9296f567e 100644
--- a/docs/reference/plotPrecisionRecall.html
+++ b/docs/reference/plotPrecisionRecall.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Plot the precision-recall curve using the sparse thresholdSummary data frame — plotPrecisionRecall • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Plot the precision-recall curve using the sparse thresholdSummary data frame — plotPrecisionRecall" />
-<meta property="og:description" content="Plot the precision-recall curve using the sparse thresholdSummary data frame" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Plot the precision-recall curve using the sparse thresholdSummary data frame — plotPrecisionRecall • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Plot the precision-recall curve using the sparse thresholdSummary data frame — plotPrecisionRecall"><meta property="og:description" content="Plot the precision-recall curve using the sparse thresholdSummary data frame"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Plot the precision-recall curve using the sparse thresholdSummary data frame</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Plotting.R'><code>R/Plotting.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Plotting.R" class="external-link"><code>R/Plotting.R</code></a></small>
     <div class="hidden name"><code>plotPrecisionRecall.Rd</code></div>
     </div>
 
@@ -183,69 +131,68 @@ <h1>Plot the precision-recall curve using the sparse thresholdSummary data frame
     <p>Plot the precision-recall curve using the sparse thresholdSummary data frame</p>
     </div>
 
-    <pre class="usage"><span class='fu'>plotPrecisionRecall</span><span class='op'>(</span>
-  <span class='va'>plpResult</span>,
-  typeColumn <span class='op'>=</span> <span class='st'>"evaluation"</span>,
-  saveLocation <span class='op'>=</span> <span class='cn'>NULL</span>,
-  fileName <span class='op'>=</span> <span class='st'>"roc.png"</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpResult</th>
-      <td><p>A plp result object as generated using the <code><a href='runPlp.html'>runPlp</a></code> function.</p></td>
-    </tr>
-    <tr>
-      <th>typeColumn</th>
-      <td><p>The name of the column specifying the evaluation type</p></td>
-    </tr>
-    <tr>
-      <th>saveLocation</th>
-      <td><p>Directory to save plot (if NULL plot is not saved)</p></td>
-    </tr>
-    <tr>
-      <th>fileName</th>
-      <td><p>Name of the file to save to plot, for example
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">plotPrecisionRecall</span><span class="op">(</span></span>
+<span>  <span class="va">plpResult</span>,</span>
+<span>  typeColumn <span class="op">=</span> <span class="st">"evaluation"</span>,</span>
+<span>  saveLocation <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  fileName <span class="op">=</span> <span class="st">"roc.png"</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpResult</dt>
+<dd><p>A plp result object as generated using the <code><a href="runPlp.html">runPlp</a></code> function.</p></dd>
+
+
+<dt>typeColumn</dt>
+<dd><p>The name of the column specifying the evaluation type</p></dd>
+
+
+<dt>saveLocation</dt>
+<dd><p>Directory to save plot (if NULL plot is not saved)</p></dd>
+
+
+<dt>fileName</dt>
+<dd><p>Name of the file to save to plot, for example
 'plot.png'. See the function <code>ggsave</code> in the ggplot2 package for
-supported file formats.</p></td>
-    </tr>
-    </table>
+supported file formats.</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
-    <p>A ggplot object. Use the <code><a href='https://ggplot2.tidyverse.org/reference/ggsave.html'>ggsave</a></code> function to save to file in a different
+<p>A ggplot object. Use the <code><a href="https://ggplot2.tidyverse.org/reference/ggsave.html" class="external-link">ggsave</a></code> function to save to file in a different
 format.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Create a plot showing the precision-recall curve using the sparse thresholdSummary data frame</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/plotPredictedPDF.html b/docs/reference/plotPredictedPDF.html
index 9e51a26f6..f95ad0bc3 100644
--- a/docs/reference/plotPredictedPDF.html
+++ b/docs/reference/plotPredictedPDF.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Plot the Predicted probability density function, showing prediction overlap between true and false cases — plotPredictedPDF • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Plot the Predicted probability density function, showing prediction overlap between true and false cases — plotPredictedPDF" />
-<meta property="og:description" content="Plot the Predicted probability density function, showing prediction overlap between true and false cases" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Plot the Predicted probability density function, showing prediction overlap between true and false cases — plotPredictedPDF • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Plot the Predicted probability density function, showing prediction overlap between true and false cases — plotPredictedPDF"><meta property="og:description" content="Plot the Predicted probability density function, showing prediction overlap between true and false cases"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Plot the Predicted probability density function, showing prediction overlap between true and false cases</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Plotting.R'><code>R/Plotting.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Plotting.R" class="external-link"><code>R/Plotting.R</code></a></small>
     <div class="hidden name"><code>plotPredictedPDF.Rd</code></div>
     </div>
 
@@ -183,69 +131,68 @@ <h1>Plot the Predicted probability density function, showing prediction overlap
     <p>Plot the Predicted probability density function, showing prediction overlap between true and false cases</p>
     </div>
 
-    <pre class="usage"><span class='fu'>plotPredictedPDF</span><span class='op'>(</span>
-  <span class='va'>plpResult</span>,
-  typeColumn <span class='op'>=</span> <span class='st'>"evaluation"</span>,
-  saveLocation <span class='op'>=</span> <span class='cn'>NULL</span>,
-  fileName <span class='op'>=</span> <span class='st'>"PredictedPDF.png"</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpResult</th>
-      <td><p>A plp result object as generated using the <code><a href='runPlp.html'>runPlp</a></code> function.</p></td>
-    </tr>
-    <tr>
-      <th>typeColumn</th>
-      <td><p>The name of the column specifying the evaluation type</p></td>
-    </tr>
-    <tr>
-      <th>saveLocation</th>
-      <td><p>Directory to save plot (if NULL plot is not saved)</p></td>
-    </tr>
-    <tr>
-      <th>fileName</th>
-      <td><p>Name of the file to save to plot, for example
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">plotPredictedPDF</span><span class="op">(</span></span>
+<span>  <span class="va">plpResult</span>,</span>
+<span>  typeColumn <span class="op">=</span> <span class="st">"evaluation"</span>,</span>
+<span>  saveLocation <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  fileName <span class="op">=</span> <span class="st">"PredictedPDF.png"</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpResult</dt>
+<dd><p>A plp result object as generated using the <code><a href="runPlp.html">runPlp</a></code> function.</p></dd>
+
+
+<dt>typeColumn</dt>
+<dd><p>The name of the column specifying the evaluation type</p></dd>
+
+
+<dt>saveLocation</dt>
+<dd><p>Directory to save plot (if NULL plot is not saved)</p></dd>
+
+
+<dt>fileName</dt>
+<dd><p>Name of the file to save to plot, for example
 'plot.png'. See the function <code>ggsave</code> in the ggplot2 package for
-supported file formats.</p></td>
-    </tr>
-    </table>
+supported file formats.</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
-    <p>A ggplot object. Use the <code><a href='https://ggplot2.tidyverse.org/reference/ggsave.html'>ggsave</a></code> function to save to file in a different
+<p>A ggplot object. Use the <code><a href="https://ggplot2.tidyverse.org/reference/ggsave.html" class="external-link">ggsave</a></code> function to save to file in a different
 format.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Create a plot showing the predicted probability density function, showing prediction overlap between true and false cases</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/plotPredictionDistribution.html b/docs/reference/plotPredictionDistribution.html
index a2e751e3e..f39445b5c 100644
--- a/docs/reference/plotPredictionDistribution.html
+++ b/docs/reference/plotPredictionDistribution.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Plot the side-by-side boxplots of prediction distribution, by class#' — plotPredictionDistribution • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Plot the side-by-side boxplots of prediction distribution, by class#' — plotPredictionDistribution" />
-<meta property="og:description" content="Plot the side-by-side boxplots of prediction distribution, by class#'" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Plot the side-by-side boxplots of prediction distribution, by class#' — plotPredictionDistribution • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Plot the side-by-side boxplots of prediction distribution, by class#' — plotPredictionDistribution"><meta property="og:description" content="Plot the side-by-side boxplots of prediction distribution, by class#'"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Plot the side-by-side boxplots of prediction distribution, by class#'</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Plotting.R'><code>R/Plotting.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Plotting.R" class="external-link"><code>R/Plotting.R</code></a></small>
     <div class="hidden name"><code>plotPredictionDistribution.Rd</code></div>
     </div>
 
@@ -183,70 +131,69 @@ <h1>Plot the side-by-side boxplots of prediction distribution, by class#'</h1>
     <p>Plot the side-by-side boxplots of prediction distribution, by class#'</p>
     </div>
 
-    <pre class="usage"><span class='fu'>plotPredictionDistribution</span><span class='op'>(</span>
-  <span class='va'>plpResult</span>,
-  typeColumn <span class='op'>=</span> <span class='st'>"evaluation"</span>,
-  saveLocation <span class='op'>=</span> <span class='cn'>NULL</span>,
-  fileName <span class='op'>=</span> <span class='st'>"PredictionDistribution.png"</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpResult</th>
-      <td><p>A plp result object as generated using the <code><a href='runPlp.html'>runPlp</a></code> function.</p></td>
-    </tr>
-    <tr>
-      <th>typeColumn</th>
-      <td><p>The name of the column specifying the evaluation type</p></td>
-    </tr>
-    <tr>
-      <th>saveLocation</th>
-      <td><p>Directory to save plot (if NULL plot is not saved)</p></td>
-    </tr>
-    <tr>
-      <th>fileName</th>
-      <td><p>Name of the file to save to plot, for example
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">plotPredictionDistribution</span><span class="op">(</span></span>
+<span>  <span class="va">plpResult</span>,</span>
+<span>  typeColumn <span class="op">=</span> <span class="st">"evaluation"</span>,</span>
+<span>  saveLocation <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  fileName <span class="op">=</span> <span class="st">"PredictionDistribution.png"</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpResult</dt>
+<dd><p>A plp result object as generated using the <code><a href="runPlp.html">runPlp</a></code> function.</p></dd>
+
+
+<dt>typeColumn</dt>
+<dd><p>The name of the column specifying the evaluation type</p></dd>
+
+
+<dt>saveLocation</dt>
+<dd><p>Directory to save plot (if NULL plot is not saved)</p></dd>
+
+
+<dt>fileName</dt>
+<dd><p>Name of the file to save to plot, for example
 'plot.png'. See the function <code>ggsave</code> in the ggplot2 package for
-supported file formats.</p></td>
-    </tr>
-    </table>
+supported file formats.</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
-    <p>A ggplot object. Use the <code><a href='https://ggplot2.tidyverse.org/reference/ggsave.html'>ggsave</a></code> function to save to file in a different
+<p>A ggplot object. Use the <code><a href="https://ggplot2.tidyverse.org/reference/ggsave.html" class="external-link">ggsave</a></code> function to save to file in a different
 format.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Create a plot showing the side-by-side boxplots of prediction distribution, by class
 #'</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/plotPreferencePDF.html b/docs/reference/plotPreferencePDF.html
index 2b7105906..8b26d68a7 100644
--- a/docs/reference/plotPreferencePDF.html
+++ b/docs/reference/plotPreferencePDF.html
@@ -1,70 +1,15 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Plot the preference score probability density function, showing prediction overlap between true and false cases
-#' — plotPreferencePDF • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Plot the preference score probability density function, showing prediction overlap between true and false cases
-#' — plotPreferencePDF" />
-<meta property="og:description" content="Plot the preference score probability density function, showing prediction overlap between true and false cases
-#'" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Plot the preference score probability density function, showing prediction overlap between true and false cases
+#' — plotPreferencePDF • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Plot the preference score probability density function, showing prediction overlap between true and false cases
+#' — plotPreferencePDF"><meta property="og:description" content="Plot the preference score probability density function, showing prediction overlap between true and false cases
+#'"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -75,13 +20,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -97,13 +41,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -116,10 +59,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -128,13 +71,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -142,44 +88,46 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Plot the preference score probability density function, showing prediction overlap between true and false cases
 #'</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Plotting.R'><code>R/Plotting.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Plotting.R" class="external-link"><code>R/Plotting.R</code></a></small>
     <div class="hidden name"><code>plotPreferencePDF.Rd</code></div>
     </div>
 
@@ -188,70 +136,69 @@ <h1>Plot the preference score probability density function, showing prediction o
 #'</p>
     </div>
 
-    <pre class="usage"><span class='fu'>plotPreferencePDF</span><span class='op'>(</span>
-  <span class='va'>plpResult</span>,
-  typeColumn <span class='op'>=</span> <span class='st'>"evaluation"</span>,
-  saveLocation <span class='op'>=</span> <span class='cn'>NULL</span>,
-  fileName <span class='op'>=</span> <span class='st'>"plotPreferencePDF.png"</span>
-<span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">plotPreferencePDF</span><span class="op">(</span></span>
+<span>  <span class="va">plpResult</span>,</span>
+<span>  typeColumn <span class="op">=</span> <span class="st">"evaluation"</span>,</span>
+<span>  saveLocation <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  fileName <span class="op">=</span> <span class="st">"plotPreferencePDF.png"</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpResult</dt>
+<dd><p>A plp result object as generated using the <code><a href="runPlp.html">runPlp</a></code> function.</p></dd>
+
+
+<dt>typeColumn</dt>
+<dd><p>The name of the column specifying the evaluation type</p></dd>
+
+
+<dt>saveLocation</dt>
+<dd><p>Directory to save plot (if NULL plot is not saved)</p></dd>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpResult</th>
-      <td><p>A plp result object as generated using the <code><a href='runPlp.html'>runPlp</a></code> function.</p></td>
-    </tr>
-    <tr>
-      <th>typeColumn</th>
-      <td><p>The name of the column specifying the evaluation type</p></td>
-    </tr>
-    <tr>
-      <th>saveLocation</th>
-      <td><p>Directory to save plot (if NULL plot is not saved)</p></td>
-    </tr>
-    <tr>
-      <th>fileName</th>
-      <td><p>Name of the file to save to plot, for example
+
+<dt>fileName</dt>
+<dd><p>Name of the file to save to plot, for example
 'plot.png'. See the function <code>ggsave</code> in the ggplot2 package for
-supported file formats.</p></td>
-    </tr>
-    </table>
+supported file formats.</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
-    <p>A ggplot object. Use the <code><a href='https://ggplot2.tidyverse.org/reference/ggsave.html'>ggsave</a></code> function to save to file in a different
+<p>A ggplot object. Use the <code><a href="https://ggplot2.tidyverse.org/reference/ggsave.html" class="external-link">ggsave</a></code> function to save to file in a different
 format.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Create a plot showing the preference score probability density function, showing prediction overlap between true and false cases
 #'</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/plotRoc.html b/docs/reference/plotRoc.html
deleted file mode 100644
index 247e2f260..000000000
--- a/docs/reference/plotRoc.html
+++ /dev/null
@@ -1,233 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Plot the ROC curve — plotRoc • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Plot the ROC curve — plotRoc" />
-<meta property="og:description" content="Plot the ROC curve" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Plot the ROC curve</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Plotting.R'><code>R/Plotting.R</code></a></small>
-    <div class="hidden name"><code>plotRoc.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Plot the ROC curve</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>plotRoc</span><span class='op'>(</span><span class='va'>prediction</span>, fileName <span class='op'>=</span> <span class='cn'>NULL</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>prediction</th>
-      <td><p>A prediction object as generated using the
-<code><a href='predictProbabilities.html'>predictProbabilities</a></code> function.</p></td>
-    </tr>
-    <tr>
-      <th>fileName</th>
-      <td><p>Name of the file where the plot should be saved, for example
-'plot.png'. See the function <code>ggsave</code> in the ggplot2 package for
-supported file formats.</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A ggplot object. Use the <code><a href='https://ggplot2.tidyverse.org/reference/ggsave.html'>ggsave</a></code> function to save to file in a different
-format.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Create a plot showing the Receiver Operator Characteristics (ROC) curve.</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/plotSmoothCalibration.html b/docs/reference/plotSmoothCalibration.html
index 05215025e..bb6bd7cd4 100644
--- a/docs/reference/plotSmoothCalibration.html
+++ b/docs/reference/plotSmoothCalibration.html
@@ -1,70 +1,14 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Plot the smooth calibration as detailed in Calster et al. "A calibration heirarchy for risk models
-was defined: from utopia to empirical data" (2016) — plotSmoothCalibration • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Plot the smooth calibration as detailed in Calster et al. "A calibration heirarchy for risk models
-was defined: from utopia to empirical data" (2016) — plotSmoothCalibration" />
-<meta property="og:description" content="Plot the smooth calibration as detailed in Calster et al. &quot;A calibration heirarchy for risk models
-was defined: from utopia to empirical data&quot; (2016)" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Plot the smooth calibration as detailed in Calster et al. "A calibration heirarchy for risk models
+was defined: from utopia to empirical data" (2016) — plotSmoothCalibration • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Plot the smooth calibration as detailed in Calster et al. " a calibration heirarchy for risk models was defined: from utopia to empirical data plotsmoothcalibration><meta property="og:description" content='Plot the smooth calibration as detailed in Calster et al. "A calibration heirarchy for risk models
+was defined: from utopia to empirical data" (2016)'><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -75,13 +19,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -97,13 +40,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -116,10 +58,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -128,13 +70,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -142,44 +87,46 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Plot the smooth calibration as detailed in Calster et al. "A calibration heirarchy for risk models
 was defined: from utopia to empirical data" (2016)</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Plotting.R'><code>R/Plotting.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Plotting.R" class="external-link"><code>R/Plotting.R</code></a></small>
     <div class="hidden name"><code>plotSmoothCalibration.Rd</code></div>
     </div>
 
@@ -188,102 +135,101 @@ <h1>Plot the smooth calibration as detailed in Calster et al. "A calibration hei
 was defined: from utopia to empirical data" (2016)</p>
     </div>
 
-    <pre class="usage"><span class='fu'>plotSmoothCalibration</span><span class='op'>(</span>
-  <span class='va'>plpResult</span>,
-  smooth <span class='op'>=</span> <span class='st'>"loess"</span>,
-  span <span class='op'>=</span> <span class='fl'>0.75</span>,
-  nKnots <span class='op'>=</span> <span class='fl'>5</span>,
-  scatter <span class='op'>=</span> <span class='cn'>FALSE</span>,
-  bins <span class='op'>=</span> <span class='fl'>20</span>,
-  sample <span class='op'>=</span> <span class='cn'>TRUE</span>,
-  typeColumn <span class='op'>=</span> <span class='st'>"evaluation"</span>,
-  saveLocation <span class='op'>=</span> <span class='cn'>NULL</span>,
-  fileName <span class='op'>=</span> <span class='st'>"smoothCalibration.pdf"</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpResult</th>
-      <td><p>The result of running <code><a href='runPlp.html'>runPlp</a></code> function. An object containing the
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">plotSmoothCalibration</span><span class="op">(</span></span>
+<span>  <span class="va">plpResult</span>,</span>
+<span>  smooth <span class="op">=</span> <span class="st">"loess"</span>,</span>
+<span>  span <span class="op">=</span> <span class="fl">0.75</span>,</span>
+<span>  nKnots <span class="op">=</span> <span class="fl">5</span>,</span>
+<span>  scatter <span class="op">=</span> <span class="cn">FALSE</span>,</span>
+<span>  bins <span class="op">=</span> <span class="fl">20</span>,</span>
+<span>  sample <span class="op">=</span> <span class="cn">TRUE</span>,</span>
+<span>  typeColumn <span class="op">=</span> <span class="st">"evaluation"</span>,</span>
+<span>  saveLocation <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  fileName <span class="op">=</span> <span class="st">"smoothCalibration.pdf"</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpResult</dt>
+<dd><p>The result of running <code><a href="runPlp.html">runPlp</a></code> function. An object containing the
 model or location where the model is save, the data selection settings, the
 preprocessing and training settings as well as various performance measures
-obtained by the model.</p></td>
-    </tr>
-    <tr>
-      <th>smooth</th>
-      <td><p>options: 'loess' or 'rcs'</p></td>
-    </tr>
-    <tr>
-      <th>span</th>
-      <td><p>This specifies the width of span used for loess. This will allow for faster
-computing and lower memory usage.</p></td>
-    </tr>
-    <tr>
-      <th>nKnots</th>
-      <td><p>The number of knots to be used by the rcs evaluation. Default is 5</p></td>
-    </tr>
-    <tr>
-      <th>scatter</th>
-      <td><p>plot the decile calibrations as points on the graph. Default is False</p></td>
-    </tr>
-    <tr>
-      <th>bins</th>
-      <td><p>The number of bins for the histogram. Default is 20.</p></td>
-    </tr>
-    <tr>
-      <th>sample</th>
-      <td><p>If using loess then by default 20,000 patients will be sampled to save time</p></td>
-    </tr>
-    <tr>
-      <th>typeColumn</th>
-      <td><p>The name of the column specifying the evaluation type</p></td>
-    </tr>
-    <tr>
-      <th>saveLocation</th>
-      <td><p>Directory to save plot (if NULL plot is not saved)</p></td>
-    </tr>
-    <tr>
-      <th>fileName</th>
-      <td><p>Name of the file to save to plot, for example
-'plot.png'. See the function <code>ggsave</code> in the ggplot2 package for
-supported file formats.</p></td>
-    </tr>
-    </table>
+obtained by the model.</p></dd>
+
+
+<dt>smooth</dt>
+<dd><p>options: 'loess' or 'rcs'</p></dd>
+
+
+<dt>span</dt>
+<dd><p>This specifies the width of span used for loess. This will allow for faster
+computing and lower memory usage.</p></dd>
+
+
+<dt>nKnots</dt>
+<dd><p>The number of knots to be used by the rcs evaluation. Default is 5</p></dd>
+
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+<dt>scatter</dt>
+<dd><p>plot the decile calibrations as points on the graph. Default is False</p></dd>
 
-    <p>A ggplot object.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
 
+<dt>bins</dt>
+<dd><p>The number of bins for the histogram. Default is 20.</p></dd>
+
+
+<dt>sample</dt>
+<dd><p>If using loess then by default 20,000 patients will be sampled to save time</p></dd>
+
+
+<dt>typeColumn</dt>
+<dd><p>The name of the column specifying the evaluation type</p></dd>
+
+
+<dt>saveLocation</dt>
+<dd><p>Directory to save plot (if NULL plot is not saved)</p></dd>
+
+
+<dt>fileName</dt>
+<dd><p>Name of the file to save to plot, for example
+'plot.png'. See the function <code>ggsave</code> in the ggplot2 package for
+supported file formats.</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A ggplot object.</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Create a plot showing the smoothed calibration #'</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/plotSparseCalibration.html b/docs/reference/plotSparseCalibration.html
index fbe0f0372..cb1bc73d3 100644
--- a/docs/reference/plotSparseCalibration.html
+++ b/docs/reference/plotSparseCalibration.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Plot the calibration — plotSparseCalibration • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Plot the calibration — plotSparseCalibration" />
-<meta property="og:description" content="Plot the calibration" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Plot the calibration — plotSparseCalibration • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Plot the calibration — plotSparseCalibration"><meta property="og:description" content="Plot the calibration"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Plot the calibration</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Plotting.R'><code>R/Plotting.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Plotting.R" class="external-link"><code>R/Plotting.R</code></a></small>
     <div class="hidden name"><code>plotSparseCalibration.Rd</code></div>
     </div>
 
@@ -183,70 +131,69 @@ <h1>Plot the calibration</h1>
     <p>Plot the calibration</p>
     </div>
 
-    <pre class="usage"><span class='fu'>plotSparseCalibration</span><span class='op'>(</span>
-  <span class='va'>plpResult</span>,
-  typeColumn <span class='op'>=</span> <span class='st'>"evaluation"</span>,
-  saveLocation <span class='op'>=</span> <span class='cn'>NULL</span>,
-  fileName <span class='op'>=</span> <span class='st'>"roc.png"</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpResult</th>
-      <td><p>A plp result object as generated using the <code><a href='runPlp.html'>runPlp</a></code> function.</p></td>
-    </tr>
-    <tr>
-      <th>typeColumn</th>
-      <td><p>The name of the column specifying the evaluation type</p></td>
-    </tr>
-    <tr>
-      <th>saveLocation</th>
-      <td><p>Directory to save plot (if NULL plot is not saved)</p></td>
-    </tr>
-    <tr>
-      <th>fileName</th>
-      <td><p>Name of the file to save to plot, for example
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">plotSparseCalibration</span><span class="op">(</span></span>
+<span>  <span class="va">plpResult</span>,</span>
+<span>  typeColumn <span class="op">=</span> <span class="st">"evaluation"</span>,</span>
+<span>  saveLocation <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  fileName <span class="op">=</span> <span class="st">"roc.png"</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpResult</dt>
+<dd><p>A plp result object as generated using the <code><a href="runPlp.html">runPlp</a></code> function.</p></dd>
+
+
+<dt>typeColumn</dt>
+<dd><p>The name of the column specifying the evaluation type</p></dd>
+
+
+<dt>saveLocation</dt>
+<dd><p>Directory to save plot (if NULL plot is not saved)</p></dd>
+
+
+<dt>fileName</dt>
+<dd><p>Name of the file to save to plot, for example
 'plot.png'. See the function <code>ggsave</code> in the ggplot2 package for
-supported file formats.</p></td>
-    </tr>
-    </table>
+supported file formats.</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
-    <p>A ggplot object. Use the <code><a href='https://ggplot2.tidyverse.org/reference/ggsave.html'>ggsave</a></code> function to save to file in a different
+<p>A ggplot object. Use the <code><a href="https://ggplot2.tidyverse.org/reference/ggsave.html" class="external-link">ggsave</a></code> function to save to file in a different
 format.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Create a plot showing the calibration
 #'</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/plotSparseCalibration2.html b/docs/reference/plotSparseCalibration2.html
index 128e137a7..34929c48a 100644
--- a/docs/reference/plotSparseCalibration2.html
+++ b/docs/reference/plotSparseCalibration2.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Plot the conventional calibration — plotSparseCalibration2 • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Plot the conventional calibration — plotSparseCalibration2" />
-<meta property="og:description" content="Plot the conventional calibration" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Plot the conventional calibration — plotSparseCalibration2 • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Plot the conventional calibration — plotSparseCalibration2"><meta property="og:description" content="Plot the conventional calibration"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Plot the conventional calibration</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Plotting.R'><code>R/Plotting.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Plotting.R" class="external-link"><code>R/Plotting.R</code></a></small>
     <div class="hidden name"><code>plotSparseCalibration2.Rd</code></div>
     </div>
 
@@ -183,70 +131,69 @@ <h1>Plot the conventional calibration</h1>
     <p>Plot the conventional calibration</p>
     </div>
 
-    <pre class="usage"><span class='fu'>plotSparseCalibration2</span><span class='op'>(</span>
-  <span class='va'>plpResult</span>,
-  typeColumn <span class='op'>=</span> <span class='st'>"evaluation"</span>,
-  saveLocation <span class='op'>=</span> <span class='cn'>NULL</span>,
-  fileName <span class='op'>=</span> <span class='st'>"roc.png"</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpResult</th>
-      <td><p>A plp result object as generated using the <code><a href='runPlp.html'>runPlp</a></code> function.</p></td>
-    </tr>
-    <tr>
-      <th>typeColumn</th>
-      <td><p>The name of the column specifying the evaluation type</p></td>
-    </tr>
-    <tr>
-      <th>saveLocation</th>
-      <td><p>Directory to save plot (if NULL plot is not saved)</p></td>
-    </tr>
-    <tr>
-      <th>fileName</th>
-      <td><p>Name of the file to save to plot, for example
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">plotSparseCalibration2</span><span class="op">(</span></span>
+<span>  <span class="va">plpResult</span>,</span>
+<span>  typeColumn <span class="op">=</span> <span class="st">"evaluation"</span>,</span>
+<span>  saveLocation <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  fileName <span class="op">=</span> <span class="st">"roc.png"</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpResult</dt>
+<dd><p>A plp result object as generated using the <code><a href="runPlp.html">runPlp</a></code> function.</p></dd>
+
+
+<dt>typeColumn</dt>
+<dd><p>The name of the column specifying the evaluation type</p></dd>
+
+
+<dt>saveLocation</dt>
+<dd><p>Directory to save plot (if NULL plot is not saved)</p></dd>
+
+
+<dt>fileName</dt>
+<dd><p>Name of the file to save to plot, for example
 'plot.png'. See the function <code>ggsave</code> in the ggplot2 package for
-supported file formats.</p></td>
-    </tr>
-    </table>
+supported file formats.</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
-    <p>A ggplot object. Use the <code><a href='https://ggplot2.tidyverse.org/reference/ggsave.html'>ggsave</a></code> function to save to file in a different
+<p>A ggplot object. Use the <code><a href="https://ggplot2.tidyverse.org/reference/ggsave.html" class="external-link">ggsave</a></code> function to save to file in a different
 format.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Create a plot showing the calibration
 #'</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/plotSparseRoc.html b/docs/reference/plotSparseRoc.html
index 1f1f410e8..37ef3c135 100644
--- a/docs/reference/plotSparseRoc.html
+++ b/docs/reference/plotSparseRoc.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Plot the ROC curve using the sparse thresholdSummary data frame — plotSparseRoc • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Plot the ROC curve using the sparse thresholdSummary data frame — plotSparseRoc" />
-<meta property="og:description" content="Plot the ROC curve using the sparse thresholdSummary data frame" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Plot the ROC curve using the sparse thresholdSummary data frame — plotSparseRoc • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Plot the ROC curve using the sparse thresholdSummary data frame — plotSparseRoc"><meta property="og:description" content="Plot the ROC curve using the sparse thresholdSummary data frame"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Plot the ROC curve using the sparse thresholdSummary data frame</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Plotting.R'><code>R/Plotting.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Plotting.R" class="external-link"><code>R/Plotting.R</code></a></small>
     <div class="hidden name"><code>plotSparseRoc.Rd</code></div>
     </div>
 
@@ -183,69 +131,68 @@ <h1>Plot the ROC curve using the sparse thresholdSummary data frame</h1>
     <p>Plot the ROC curve using the sparse thresholdSummary data frame</p>
     </div>
 
-    <pre class="usage"><span class='fu'>plotSparseRoc</span><span class='op'>(</span>
-  <span class='va'>plpResult</span>,
-  typeColumn <span class='op'>=</span> <span class='st'>"evaluation"</span>,
-  saveLocation <span class='op'>=</span> <span class='cn'>NULL</span>,
-  fileName <span class='op'>=</span> <span class='st'>"roc.png"</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpResult</th>
-      <td><p>A plp result object as generated using the <code><a href='runPlp.html'>runPlp</a></code> function.</p></td>
-    </tr>
-    <tr>
-      <th>typeColumn</th>
-      <td><p>The name of the column specifying the evaluation type</p></td>
-    </tr>
-    <tr>
-      <th>saveLocation</th>
-      <td><p>Directory to save plot (if NULL plot is not saved)</p></td>
-    </tr>
-    <tr>
-      <th>fileName</th>
-      <td><p>Name of the file to save to plot, for example
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">plotSparseRoc</span><span class="op">(</span></span>
+<span>  <span class="va">plpResult</span>,</span>
+<span>  typeColumn <span class="op">=</span> <span class="st">"evaluation"</span>,</span>
+<span>  saveLocation <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  fileName <span class="op">=</span> <span class="st">"roc.png"</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpResult</dt>
+<dd><p>A plp result object as generated using the <code><a href="runPlp.html">runPlp</a></code> function.</p></dd>
+
+
+<dt>typeColumn</dt>
+<dd><p>The name of the column specifying the evaluation type</p></dd>
+
+
+<dt>saveLocation</dt>
+<dd><p>Directory to save plot (if NULL plot is not saved)</p></dd>
+
+
+<dt>fileName</dt>
+<dd><p>Name of the file to save to plot, for example
 'plot.png'. See the function <code>ggsave</code> in the ggplot2 package for
-supported file formats.</p></td>
-    </tr>
-    </table>
+supported file formats.</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
-    <p>A ggplot object. Use the <code><a href='https://ggplot2.tidyverse.org/reference/ggsave.html'>ggsave</a></code> function to save to file in a different
+<p>A ggplot object. Use the <code><a href="https://ggplot2.tidyverse.org/reference/ggsave.html" class="external-link">ggsave</a></code> function to save to file in a different
 format.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Create a plot showing the Receiver Operator Characteristics (ROC) curve.</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/plotVariableScatterplot.html b/docs/reference/plotVariableScatterplot.html
index a2934945a..752c4e840 100644
--- a/docs/reference/plotVariableScatterplot.html
+++ b/docs/reference/plotVariableScatterplot.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Plot the variable importance scatterplot — plotVariableScatterplot • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Plot the variable importance scatterplot — plotVariableScatterplot" />
-<meta property="og:description" content="Plot the variable importance scatterplot" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Plot the variable importance scatterplot — plotVariableScatterplot • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Plot the variable importance scatterplot — plotVariableScatterplot"><meta property="og:description" content="Plot the variable importance scatterplot"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Plot the variable importance scatterplot</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Plotting.R'><code>R/Plotting.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Plotting.R" class="external-link"><code>R/Plotting.R</code></a></small>
     <div class="hidden name"><code>plotVariableScatterplot.Rd</code></div>
     </div>
 
@@ -183,66 +131,65 @@ <h1>Plot the variable importance scatterplot</h1>
     <p>Plot the variable importance scatterplot</p>
     </div>
 
-    <pre class="usage"><span class='fu'>plotVariableScatterplot</span><span class='op'>(</span>
-  <span class='va'>covariateSummary</span>,
-  saveLocation <span class='op'>=</span> <span class='cn'>NULL</span>,
-  fileName <span class='op'>=</span> <span class='st'>"VariableScatterplot.png"</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>covariateSummary</th>
-      <td><p>A prediction object as generated using the
-<code><a href='runPlp.html'>runPlp</a></code> function.</p></td>
-    </tr>
-    <tr>
-      <th>saveLocation</th>
-      <td><p>Directory to save plot (if NULL plot is not saved)</p></td>
-    </tr>
-    <tr>
-      <th>fileName</th>
-      <td><p>Name of the file to save to plot, for example
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">plotVariableScatterplot</span><span class="op">(</span></span>
+<span>  <span class="va">covariateSummary</span>,</span>
+<span>  saveLocation <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  fileName <span class="op">=</span> <span class="st">"VariableScatterplot.png"</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>covariateSummary</dt>
+<dd><p>A prediction object as generated using the
+<code><a href="runPlp.html">runPlp</a></code> function.</p></dd>
+
+
+<dt>saveLocation</dt>
+<dd><p>Directory to save plot (if NULL plot is not saved)</p></dd>
+
+
+<dt>fileName</dt>
+<dd><p>Name of the file to save to plot, for example
 'plot.png'. See the function <code>ggsave</code> in the ggplot2 package for
-supported file formats.</p></td>
-    </tr>
-    </table>
+supported file formats.</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
 
-    <p>A ggplot object. Use the <code><a href='https://ggplot2.tidyverse.org/reference/ggsave.html'>ggsave</a></code> function to save to file in a different
+<p>A ggplot object. Use the <code><a href="https://ggplot2.tidyverse.org/reference/ggsave.html" class="external-link">ggsave</a></code> function to save to file in a different
 format.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Create a plot showing the variable importance scatterplot
 #'</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/plpDataSimulationProfile.html b/docs/reference/plpDataSimulationProfile.html
index fe6c92245..cdf677259 100644
--- a/docs/reference/plpDataSimulationProfile.html
+++ b/docs/reference/plpDataSimulationProfile.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>A simulation profile — plpDataSimulationProfile • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="A simulation profile — plpDataSimulationProfile" />
-<meta property="og:description" content="A simulation profile" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>A simulation profile — plpDataSimulationProfile • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="A simulation profile — plpDataSimulationProfile"><meta property="og:description" content="A simulation profile"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>A simulation profile</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/PatientLevelPrediction.R'><code>R/PatientLevelPrediction.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/PatientLevelPrediction.R" class="external-link"><code>R/PatientLevelPrediction.R</code></a></small>
     <div class="hidden name"><code>plpDataSimulationProfile.Rd</code></div>
     </div>
 
@@ -183,47 +131,54 @@ <h1>A simulation profile</h1>
     <p>A simulation profile</p>
     </div>
 
-    <pre class="usage"><span class='fu'><a href='https://rdrr.io/r/utils/data.html'>data</a></span><span class='op'>(</span><span class='va'>plpDataSimulationProfile</span><span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu"><a href="https://rdrr.io/r/utils/data.html" class="external-link">data</a></span><span class="op">(</span><span class="va">plpDataSimulationProfile</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="format">
+    <h2>Format</h2>
+    <p>A data frame containing the following elements:</p><dl><dt>covariatePrevalence</dt>
+<dd><p>prevalence of all covariates</p></dd>
 
-    <h2 class="hasAnchor" id="format"><a class="anchor" href="#format"></a>Format</h2>
+  <dt>outcomeModels</dt>
+<dd><p>regression model parameters to simulate outcomes</p></dd>
 
-    <p>A data frame containing the following elements:</p><dl>
-  <dt>covariatePrevalence</dt><dd><p>prevalence of all covariates</p></dd>
-  <dt>outcomeModels</dt><dd><p>regression model parameters to simulate outcomes</p></dd>
-  <dt>metaData</dt><dd><p>settings used to simulate the profile</p></dd>
-  <dt>covariateRef</dt><dd><p>covariateIds and covariateNames</p></dd>
-  <dt>timePrevalence</dt><dd><p>time window</p></dd>
-  <dt>exclusionPrevalence</dt><dd><p>prevalence of exclusion of covariates</p></dd>
+  <dt>metaData</dt>
+<dd><p>settings used to simulate the profile</p></dd>
 
-</dl>
+  <dt>covariateRef</dt>
+<dd><p>covariateIds and covariateNames</p></dd>
 
+  <dt>timePrevalence</dt>
+<dd><p>time window</p></dd>
+
+  <dt>exclusionPrevalence</dt>
+<dd><p>prevalence of exclusion of covariates</p></dd>
+
+
+</dl></div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/populatePlpResultTables.html b/docs/reference/populatePlpResultTables.html
deleted file mode 100644
index 09b0689e3..000000000
--- a/docs/reference/populatePlpResultTables.html
+++ /dev/null
@@ -1,356 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Populate the PatientLevelPrediction results tables — populatePlpResultTables • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Populate the PatientLevelPrediction results tables — populatePlpResultTables" />
-<meta property="og:description" content="This function formats and uploads results that have been generated via an ATLAS prediction package into a database" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/InstallationGuide.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
-    </li>
-    <li>
-      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
-    </li>
-    <li>
-      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Populate the PatientLevelPrediction results tables</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/uploadPlpDbResults.R'><code>R/uploadPlpDbResults.R</code></a></small>
-    <div class="hidden name"><code>populatePlpResultTables.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>This function formats and uploads results that have been generated via an ATLAS prediction package into a database</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>populatePlpResultTables</span><span class='op'>(</span>
-  <span class='va'>conn</span>,
-  <span class='va'>resultSchema</span>,
-  stringAppendToTables <span class='op'>=</span> <span class='st'>""</span>,
-  targetDialect <span class='op'>=</span> <span class='st'>"postgresql"</span>,
-  tempEmulationSchema <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/options.html'>getOption</a></span><span class='op'>(</span><span class='st'>"sqlRenderTempEmulationSchema"</span><span class='op'>)</span>,
-  <span class='va'>packageName</span>,
-  <span class='va'>studyJsonList</span>,
-  studyName <span class='op'>=</span> <span class='st'>""</span>,
-  studyDescription <span class='op'>=</span> <span class='st'>""</span>,
-  researcherName <span class='op'>=</span> <span class='st'>""</span>,
-  researcherEmail <span class='op'>=</span> <span class='st'>""</span>,
-  researcherOrg <span class='op'>=</span> <span class='st'>""</span>,
-  databaseName <span class='op'>=</span> <span class='cn'>NULL</span>,
-  databaseAcronym <span class='op'>=</span> <span class='cn'>NULL</span>,
-  databaseVersion <span class='op'>=</span> <span class='fl'>1</span>,
-  databaseDescription <span class='op'>=</span> <span class='cn'>NULL</span>,
-  databaseType <span class='op'>=</span> <span class='cn'>NULL</span>,
-  valDatabases <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span>ccae <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span>name <span class='op'>=</span> <span class='st'>"CCAE"</span>, description <span class='op'>=</span> <span class='st'>""</span>, version <span class='op'>=</span> <span class='fl'>1</span>, type <span class='op'>=</span>
-    <span class='st'>"US Claims"</span><span class='op'>)</span><span class='op'>)</span>,
-  resultLocation <span class='op'>=</span> <span class='cn'>NULL</span>,
-  resultPattern <span class='op'>=</span> <span class='st'>""</span>,
-  validationLocation <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/file.path.html'>file.path</a></span><span class='op'>(</span><span class='va'>resultLocation</span>, <span class='st'>"Validation"</span><span class='op'>)</span>,
-  addInternalValidation <span class='op'>=</span> <span class='cn'>T</span>,
-  addExternalValidation <span class='op'>=</span> <span class='cn'>T</span>,
-  gsubVal <span class='op'>=</span> <span class='cn'>NULL</span>,
-  removePattern <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>conn</th>
-      <td><p>A connection to a database created by using the
-function <code>connect</code> in the
-<code>DatabaseConnector</code> package.</p></td>
-    </tr>
-    <tr>
-      <th>resultSchema</th>
-      <td><p>(string) The name of the database schema that the result tables will be created.</p></td>
-    </tr>
-    <tr>
-      <th>stringAppendToTables</th>
-      <td><p>(string) A string that appends to the PatientLevelPrediction result tables</p></td>
-    </tr>
-    <tr>
-      <th>targetDialect</th>
-      <td><p>(string) The database management system being used</p></td>
-    </tr>
-    <tr>
-      <th>tempEmulationSchema</th>
-      <td><p>(string) The temp schema used when the database management system is oracle</p></td>
-    </tr>
-    <tr>
-      <th>packageName</th>
-      <td><p>(string) The name of the ATLAS R package used to generate the results (this is used to extract cohort jsons)</p></td>
-    </tr>
-    <tr>
-      <th>studyJsonList</th>
-      <td><p>(list) A list of lists per cohort with the cohort_name, cohort_id and cohort_json</p></td>
-    </tr>
-    <tr>
-      <th>studyName</th>
-      <td><p>(string) A reference study name</p></td>
-    </tr>
-    <tr>
-      <th>studyDescription</th>
-      <td><p>(string) A description of the study</p></td>
-    </tr>
-    <tr>
-      <th>researcherName</th>
-      <td><p>(string) Name of the researcher who developed the study</p></td>
-    </tr>
-    <tr>
-      <th>researcherEmail</th>
-      <td><p>(string) Email of the researcher who developed the study</p></td>
-    </tr>
-    <tr>
-      <th>researcherOrg</th>
-      <td><p>(string) Organisation of the researcher who developed the study</p></td>
-    </tr>
-    <tr>
-      <th>databaseName</th>
-      <td><p>(string) name of the database used to develop the model/s</p></td>
-    </tr>
-    <tr>
-      <th>databaseAcronym</th>
-      <td><p>(string) acronym of the database used to develop the model/s</p></td>
-    </tr>
-    <tr>
-      <th>databaseVersion</th>
-      <td><p>(int) Version of the database used to develop the model/s</p></td>
-    </tr>
-    <tr>
-      <th>databaseDescription</th>
-      <td><p>(string) Description of the database used to develop the model/s</p></td>
-    </tr>
-    <tr>
-      <th>databaseType</th>
-      <td><p>(string) Type of the database used to develop the model/s (e.g., claims)</p></td>
-    </tr>
-    <tr>
-      <th>valDatabases</th>
-      <td><p>(list) A named list with details of the external validation databases.  Needs to contain: name, description, version, type.</p></td>
-    </tr>
-    <tr>
-      <th>resultLocation</th>
-      <td><p>(string) location of directory where the main package results were saved</p></td>
-    </tr>
-    <tr>
-      <th>resultPattern</th>
-      <td><p>(string) A string to match to select models of interest</p></td>
-    </tr>
-    <tr>
-      <th>validationLocation</th>
-      <td><p>(string) location of directory where the validation package results were saved</p></td>
-    </tr>
-    <tr>
-      <th>addInternalValidation</th>
-      <td><p>(boolean) Whether the internval validation results should be uploaded</p></td>
-    </tr>
-    <tr>
-      <th>addExternalValidation</th>
-      <td><p>(boolean) Whether the externval validation results should be uploaded</p></td>
-    </tr>
-    <tr>
-      <th>gsubVal</th>
-      <td><p>(string) Remove patterns from the result name</p></td>
-    </tr>
-    <tr>
-      <th>removePattern</th>
-      <td><p>(string) Restrict to result names with this pattern</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>Returns NULL but uploads all the results in resultLocation to the PatientLevelPrediction result tables in resultSchema</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>This function can be used upload PatientLevelPrediction results into a database</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/positiveLikelihoodRatio.html b/docs/reference/positiveLikelihoodRatio.html
index 0c19aa720..15d6fb14f 100644
--- a/docs/reference/positiveLikelihoodRatio.html
+++ b/docs/reference/positiveLikelihoodRatio.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculate the positiveLikelihoodRatio — positiveLikelihoodRatio • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculate the positiveLikelihoodRatio — positiveLikelihoodRatio" />
-<meta property="og:description" content="Calculate the positiveLikelihoodRatio" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculate the positiveLikelihoodRatio — positiveLikelihoodRatio • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculate the positiveLikelihoodRatio — positiveLikelihoodRatio"><meta property="og:description" content="Calculate the positiveLikelihoodRatio"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculate the positiveLikelihoodRatio</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ThresholdSummary.R'><code>R/ThresholdSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ThresholdSummary.R" class="external-link"><code>R/ThresholdSummary.R</code></a></small>
     <div class="hidden name"><code>positiveLikelihoodRatio.Rd</code></div>
     </div>
 
@@ -183,61 +131,60 @@ <h1>Calculate the positiveLikelihoodRatio</h1>
     <p>Calculate the positiveLikelihoodRatio</p>
     </div>
 
-    <pre class="usage"><span class='fu'>positiveLikelihoodRatio</span><span class='op'>(</span><span class='va'>TP</span>, <span class='va'>TN</span>, <span class='va'>FN</span>, <span class='va'>FP</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>TP</th>
-      <td><p>Number of true positives</p></td>
-    </tr>
-    <tr>
-      <th>TN</th>
-      <td><p>Number of true negatives</p></td>
-    </tr>
-    <tr>
-      <th>FN</th>
-      <td><p>Number of false negatives</p></td>
-    </tr>
-    <tr>
-      <th>FP</th>
-      <td><p>Number of false positives</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>positiveLikelihoodRatio value</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">positiveLikelihoodRatio</span><span class="op">(</span><span class="va">TP</span>, <span class="va">TN</span>, <span class="va">FN</span>, <span class="va">FP</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>TP</dt>
+<dd><p>Number of true positives</p></dd>
+
+
+<dt>TN</dt>
+<dd><p>Number of true negatives</p></dd>
+
+
+<dt>FN</dt>
+<dd><p>Number of false negatives</p></dd>
+
+
+<dt>FP</dt>
+<dd><p>Number of false positives</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>positiveLikelihoodRatio value</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculate the positiveLikelihoodRatio</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/positivePredictiveValue.html b/docs/reference/positivePredictiveValue.html
index 940084bea..8e32622a2 100644
--- a/docs/reference/positivePredictiveValue.html
+++ b/docs/reference/positivePredictiveValue.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculate the positivePredictiveValue — positivePredictiveValue • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculate the positivePredictiveValue — positivePredictiveValue" />
-<meta property="og:description" content="Calculate the positivePredictiveValue" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculate the positivePredictiveValue — positivePredictiveValue • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculate the positivePredictiveValue — positivePredictiveValue"><meta property="og:description" content="Calculate the positivePredictiveValue"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculate the positivePredictiveValue</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ThresholdSummary.R'><code>R/ThresholdSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ThresholdSummary.R" class="external-link"><code>R/ThresholdSummary.R</code></a></small>
     <div class="hidden name"><code>positivePredictiveValue.Rd</code></div>
     </div>
 
@@ -183,61 +131,60 @@ <h1>Calculate the positivePredictiveValue</h1>
     <p>Calculate the positivePredictiveValue</p>
     </div>
 
-    <pre class="usage"><span class='fu'>positivePredictiveValue</span><span class='op'>(</span><span class='va'>TP</span>, <span class='va'>TN</span>, <span class='va'>FN</span>, <span class='va'>FP</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>TP</th>
-      <td><p>Number of true positives</p></td>
-    </tr>
-    <tr>
-      <th>TN</th>
-      <td><p>Number of true negatives</p></td>
-    </tr>
-    <tr>
-      <th>FN</th>
-      <td><p>Number of false negatives</p></td>
-    </tr>
-    <tr>
-      <th>FP</th>
-      <td><p>Number of false positives</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>positivePredictiveValue value</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">positivePredictiveValue</span><span class="op">(</span><span class="va">TP</span>, <span class="va">TN</span>, <span class="va">FN</span>, <span class="va">FP</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>TP</dt>
+<dd><p>Number of true positives</p></dd>
+
+
+<dt>TN</dt>
+<dd><p>Number of true negatives</p></dd>
+
+
+<dt>FN</dt>
+<dd><p>Number of false negatives</p></dd>
+
+
+<dt>FP</dt>
+<dd><p>Number of false positives</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>positivePredictiveValue value</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculate the positivePredictiveValue</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/predictAndromeda.html b/docs/reference/predictAndromeda.html
deleted file mode 100644
index 96470b65b..000000000
--- a/docs/reference/predictAndromeda.html
+++ /dev/null
@@ -1,253 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Generated predictions from a regression model — predictAndromeda • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Generated predictions from a regression model — predictAndromeda" />
-<meta property="og:description" content="Generated predictions from a regression model" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Generated predictions from a regression model</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Predict.R'><code>R/Predict.R</code></a></small>
-    <div class="hidden name"><code>predictAndromeda.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Generated predictions from a regression model</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>predictAndromeda</span><span class='op'>(</span>
-  <span class='va'>coefficients</span>,
-  <span class='va'>population</span>,
-  <span class='va'>covariateData</span>,
-  modelType <span class='op'>=</span> <span class='st'>"logistic"</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>coefficients</th>
-      <td><p>A names numeric vector where the names are the covariateIds, except for the
-first value which is expected to be the intercept.</p></td>
-    </tr>
-    <tr>
-      <th>population</th>
-      <td><p>A data frame containing the population to do the prediction for</p></td>
-    </tr>
-    <tr>
-      <th>covariateData</th>
-      <td><p>An andromeda object containing the covariateData with predefined columns
-(see below).</p></td>
-    </tr>
-    <tr>
-      <th>modelType</th>
-      <td><p>Current supported types are "logistic", "poisson", "cox" or "survival".</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>These columns are expected in the outcome object:</p><table class='table'>
-<tr><td><code>rowId</code></td><td>(integer)</td><td>Row ID is used to link multiple covariates (x) to a single outcome (y)</td></tr>
-<tr><td><code>time</code></td><td>(real)</td><td>For models that use time (e.g. Poisson or Cox regression) this contains time</td></tr>
-<tr><td></td><td></td><td>(e.g. number of days)</td></tr>
-</table>
-<p>These columns are expected in the covariates object:</p><table class='table'>
-<tr><td><code>rowId</code></td><td>(integer)</td><td>Row ID is used to link multiple covariates (x) to a single outcome
-(y)</td></tr>
-<tr><td><code>covariateId</code></td><td>(integer)</td><td>A numeric identifier of a covariate</td></tr>
-<tr><td><code>covariateValue</code></td><td>(real)</td><td>The value of the specified covariate</td></tr>
-</table>
-
-
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/predictCyclops.html b/docs/reference/predictCyclops.html
index 3a21d5630..8482299c3 100644
--- a/docs/reference/predictCyclops.html
+++ b/docs/reference/predictCyclops.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create predictive probabilities — predictCyclops • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create predictive probabilities — predictCyclops" />
-<meta property="og:description" content="Create predictive probabilities" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create predictive probabilities — predictCyclops • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create predictive probabilities — predictCyclops"><meta property="og:description" content="Create predictive probabilities"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create predictive probabilities</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/CyclopsModels.R'><code>R/CyclopsModels.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/CyclopsModels.R" class="external-link"><code>R/CyclopsModels.R</code></a></small>
     <div class="hidden name"><code>predictCyclops.Rd</code></div>
     </div>
 
@@ -183,59 +131,58 @@ <h1>Create predictive probabilities</h1>
     <p>Create predictive probabilities</p>
     </div>
 
-    <pre class="usage"><span class='fu'>predictCyclops</span><span class='op'>(</span><span class='va'>plpModel</span>, <span class='va'>data</span>, <span class='va'>cohort</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpModel</th>
-      <td><p>An object of type <code>predictiveModel</code> as generated using
-<code><a href='fitPlp.html'>fitPlp</a></code>.</p></td>
-    </tr>
-    <tr>
-      <th>data</th>
-      <td><p>The new plpData containing the covariateData for the new population</p></td>
-    </tr>
-    <tr>
-      <th>cohort</th>
-      <td><p>The cohort to calculate the prediction for</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>The value column in the result data.frame is: logistic: probabilities of the outcome, poisson:
-Poisson rate (per day) of the outome, survival: hazard rate (per day) of the outcome.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">predictCyclops</span><span class="op">(</span><span class="va">plpModel</span>, <span class="va">data</span>, <span class="va">cohort</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpModel</dt>
+<dd><p>An object of type <code>predictiveModel</code> as generated using
+<code><a href="fitPlp.html">fitPlp</a></code>.</p></dd>
+
+
+<dt>data</dt>
+<dd><p>The new plpData containing the covariateData for the new population</p></dd>
+
+
+<dt>cohort</dt>
+<dd><p>The cohort to calculate the prediction for</p></dd>
 
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>The value column in the result data.frame is: logistic: probabilities of the outcome, poisson:
+Poisson rate (per day) of the outome, survival: hazard rate (per day) of the outcome.</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Generates predictions for the population specified in plpData given the model.</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/predictFfdf.html b/docs/reference/predictFfdf.html
deleted file mode 100644
index 2affbc06c..000000000
--- a/docs/reference/predictFfdf.html
+++ /dev/null
@@ -1,235 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Generated predictions from a regression model — predictFfdf • PatientLevelPrediction</title>
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script>
-
-<!-- sticky kit -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-<meta property="og:title" content="Generated predictions from a regression model — predictFfdf" />
-
-<meta property="og:description" content="Generated predictions from a regression model" />
-<meta name="twitter:card" content="summary" />
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-  </head>
-
-  <body>
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.5</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Generated predictions from a regression model</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Predict.R'><code>R/Predict.R</code></a></small>
-    <div class="hidden name"><code>predictFfdf.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    
-    <p>Generated predictions from a regression model</p>
-    
-    </div>
-
-    <pre class="usage"><span class='fu'>predictFfdf</span>(<span class='no'>coefficients</span>, <span class='no'>population</span>, <span class='no'>covariates</span>, <span class='kw'>modelType</span> <span class='kw'>=</span> <span class='st'>"logistic"</span>)</pre>
-    
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>coefficients</th>
-      <td><p>A names numeric vector where the names are the covariateIds, except for the
-first value which is expected to be the intercept.</p></td>
-    </tr>
-    <tr>
-      <th>population</th>
-      <td><p>A data frame containing the population to do the prediction for</p></td>
-    </tr>
-    <tr>
-      <th>covariates</th>
-      <td><p>A data frame or ffdf object containing the covariates with predefined columns
-(see below).</p></td>
-    </tr>
-    <tr>
-      <th>modelType</th>
-      <td><p>Current supported types are "logistic", "poisson", "cox" or "survival".</p></td>
-    </tr>
-    </table>
-    
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>These columns are expected in the outcome object:</p><table class='table'>
-<tr><td></td><td><code>rowId</code></td><td></td></tr>
-<tr><td>(integer)</td><td></td><td>Row ID is used to link multiple covariates (x) to a single outcome (y)</td></tr>
-<tr><td></td><td><code>time</code></td><td></td></tr>
-<tr><td>(real)</td><td>For models that use time (e.g. Poisson or Cox regression) this contains time</td><td></td></tr>
-<tr><td></td><td>(e.g. number of days)</td><td></td></tr>
-</table>
-<p>These columns are expected in the covariates object:</p><table class='table'>
-<tr><td></td><td><code>rowId</code></td><td></td></tr>
-<tr><td>(integer)</td><td>Row ID is used to link multiple covariates (x) to a single outcome</td><td>(y)</td></tr>
-<tr><td></td><td><code>covariateId</code></td><td></td></tr>
-<tr><td>(integer)</td><td>A numeric identifier of a covariate</td><td></td></tr>
-<tr><td><code>covariateValue</code></td><td></td><td>(real)</td></tr>
-<tr><td>The value of the specified covariate</td><td></td><td></td></tr>
-</table>
-    
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-    <h2>Contents</h2>
-    <ul class="nav nav-pills nav-stacked">
-      <li><a href="#arguments">Arguments</a></li>
-      
-      <li><a href="#details">Details</a></li>
-          </ul>
-
-  </div>
-</div>
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-   </div>
-
-  
-
-  </body>
-</html>
-
diff --git a/docs/reference/predictPlp.html b/docs/reference/predictPlp.html
index dc81d5d0a..1765cf0ec 100644
--- a/docs/reference/predictPlp.html
+++ b/docs/reference/predictPlp.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>predictPlp — predictPlp • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="predictPlp — predictPlp" />
-<meta property="og:description" content="Predict the risk of the outcome using the input plpModel for the input plpData" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>predictPlp — predictPlp • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="predictPlp — predictPlp"><meta property="og:description" content="Predict the risk of the outcome using the input plpModel for the input plpData"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>predictPlp</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Predict.R'><code>R/Predict.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Predict.R" class="external-link"><code>R/Predict.R</code></a></small>
     <div class="hidden name"><code>predictPlp.Rd</code></div>
     </div>
 
@@ -183,62 +131,61 @@ <h1>predictPlp</h1>
     <p>Predict the risk of the outcome using the input plpModel for the input plpData</p>
     </div>
 
-    <pre class="usage"><span class='fu'>predictPlp</span><span class='op'>(</span><span class='va'>plpModel</span>, <span class='va'>plpData</span>, <span class='va'>population</span>, <span class='va'>timepoint</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpModel</th>
-      <td><p>An object of type <code>plpModel</code> - a patient level prediction model</p></td>
-    </tr>
-    <tr>
-      <th>plpData</th>
-      <td><p>An object of type <code>plpData</code> - the patient level prediction
-data extracted from the CDM.</p></td>
-    </tr>
-    <tr>
-      <th>population</th>
-      <td><p>The population created using createStudyPopulation() who will have their risks predicted or a cohort without the outcome known</p></td>
-    </tr>
-    <tr>
-      <th>timepoint</th>
-      <td><p>The timepoint to predict risk (survival models only)</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A dataframe containing the prediction for each person in the population with an attribute metaData containing prediction details.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">predictPlp</span><span class="op">(</span><span class="va">plpModel</span>, <span class="va">plpData</span>, <span class="va">population</span>, <span class="va">timepoint</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpModel</dt>
+<dd><p>An object of type <code>plpModel</code> - a patient level prediction model</p></dd>
+
+
+<dt>plpData</dt>
+<dd><p>An object of type <code>plpData</code> - the patient level prediction
+data extracted from the CDM.</p></dd>
+
+
+<dt>population</dt>
+<dd><p>The population created using createStudyPopulation() who will have their risks predicted or a cohort without the outcome known</p></dd>
+
+
+<dt>timepoint</dt>
+<dd><p>The timepoint to predict risk (survival models only)</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A dataframe containing the prediction for each person in the population with an attribute metaData containing prediction details.</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>The function applied the trained model on the plpData to make predictions</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/predictProbabilities.html b/docs/reference/predictProbabilities.html
deleted file mode 100644
index 6fbd0a8c7..000000000
--- a/docs/reference/predictProbabilities.html
+++ /dev/null
@@ -1,235 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create predictive probabilities — predictProbabilities • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create predictive probabilities — predictProbabilities" />
-<meta property="og:description" content="Create predictive probabilities" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Create predictive probabilities</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Predict.R'><code>R/Predict.R</code></a></small>
-    <div class="hidden name"><code>predictProbabilities.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Create predictive probabilities</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>predictProbabilities</span><span class='op'>(</span><span class='va'>predictiveModel</span>, <span class='va'>population</span>, <span class='va'>covariateData</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>predictiveModel</th>
-      <td><p>An object of type <code>predictiveModel</code> as generated using
-<code><a href='fitPlp.html'>fitPlp</a></code>.</p></td>
-    </tr>
-    <tr>
-      <th>population</th>
-      <td><p>The population to calculate the prediction for</p></td>
-    </tr>
-    <tr>
-      <th>covariateData</th>
-      <td><p>The covariateData containing the covariates for the population</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>The value column in the result data.frame is: logistic: probabilities of the outcome, poisson:
-Poisson rate (per day) of the outome, survival: hazard rate (per day) of the outcome.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Generates predictions for the population specified in plpData given the model.</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/preprocessData.html b/docs/reference/preprocessData.html
index c6e79957d..c8c042505 100644
--- a/docs/reference/preprocessData.html
+++ b/docs/reference/preprocessData.html
@@ -1,70 +1,15 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>A function that wraps around FeatureExtraction::tidyCovariateData to normalise the data
-and remove rare or redundant features — preprocessData • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="A function that wraps around FeatureExtraction::tidyCovariateData to normalise the data
-and remove rare or redundant features — preprocessData" />
-<meta property="og:description" content="A function that wraps around FeatureExtraction::tidyCovariateData to normalise the data
-and remove rare or redundant features" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>A function that wraps around FeatureExtraction::tidyCovariateData to normalise the data
+and remove rare or redundant features — preprocessData • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="A function that wraps around FeatureExtraction::tidyCovariateData to normalise the data
+and remove rare or redundant features — preprocessData"><meta property="og:description" content="A function that wraps around FeatureExtraction::tidyCovariateData to normalise the data
+and remove rare or redundant features"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -75,13 +20,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -97,13 +41,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -116,10 +59,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -128,13 +71,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -142,44 +88,46 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>A function that wraps around FeatureExtraction::tidyCovariateData to normalise the data
 and remove rare or redundant features</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/PreprocessingData.R'><code>R/PreprocessingData.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/PreprocessingData.R" class="external-link"><code>R/PreprocessingData.R</code></a></small>
     <div class="hidden name"><code>preprocessData.Rd</code></div>
     </div>
 
@@ -188,54 +136,53 @@ <h1>A function that wraps around FeatureExtraction::tidyCovariateData to normali
 and remove rare or redundant features</p>
     </div>
 
-    <pre class="usage"><span class='fu'>preprocessData</span><span class='op'>(</span><span class='va'>covariateData</span>, <span class='va'>preprocessSettings</span><span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">preprocessData</span><span class="op">(</span><span class="va">covariateData</span>, <span class="va">preprocessSettings</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>covariateData</th>
-      <td><p>The covariate part of the training data created by <code>splitData</code> after being sampled and having 
-any required feature engineering</p></td>
-    </tr>
-    <tr>
-      <th>preprocessSettings</th>
-      <td><p>The settings for the preprocessing created by <code>createPreprocessSettings</code></p></td>
-    </tr>
-    </table>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>covariateData</dt>
+<dd><p>The covariate part of the training data created by <code>splitData</code> after being sampled and having 
+any required feature engineering</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
 
-    <p>The data processed</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<dt>preprocessSettings</dt>
+<dd><p>The settings for the preprocessing created by <code>createPreprocessSettings</code></p></dd>
 
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>The data processed</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Returns an object of class <code>covariateData</code> that has been processed</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/randomSplitter.html b/docs/reference/randomSplitter.html
deleted file mode 100644
index 3f91292a3..000000000
--- a/docs/reference/randomSplitter.html
+++ /dev/null
@@ -1,244 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Split data into random subsets stratified by class — randomSplitter • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Split data into random subsets stratified by class — randomSplitter" />
-<meta property="og:description" content="Split data into random subsets stratified by class" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Split data into random subsets stratified by class</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/DataSplitting.R'><code>R/DataSplitting.R</code></a></small>
-    <div class="hidden name"><code>randomSplitter.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Split data into random subsets stratified by class</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>randomSplitter</span><span class='op'>(</span><span class='va'>population</span>, test <span class='op'>=</span> <span class='fl'>0.3</span>, train <span class='op'>=</span> <span class='cn'>NULL</span>, nfold <span class='op'>=</span> <span class='fl'>3</span>, seed <span class='op'>=</span> <span class='cn'>NULL</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>population</th>
-      <td><p>An object created using createStudyPopulation().</p></td>
-    </tr>
-    <tr>
-      <th>test</th>
-      <td><p>A real number between 0 and 1 indicating the test set fraction of the data</p></td>
-    </tr>
-    <tr>
-      <th>train</th>
-      <td><p>A real number between 0 and 1 indicating the train set fraction of the data.
-If not set train is equal to 1 - test</p></td>
-    </tr>
-    <tr>
-      <th>nfold</th>
-      <td><p>An integer &gt;= 1 specifying the number of folds used in cross validation</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>If set a fixed seed is used, otherwise a random split is performed</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A dataframe containing the columns: rowId and index</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Returns a dataframe of rowIds and indexes with a -1 index indicating the rowId belongs to the test
-set and a positive integer index value indicating the rowId's cross valiation fold within the train
-set.</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/recalibratePlp.html b/docs/reference/recalibratePlp.html
index d789c3085..d9fb6af46 100644
--- a/docs/reference/recalibratePlp.html
+++ b/docs/reference/recalibratePlp.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>recalibratePlp — recalibratePlp • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="recalibratePlp — recalibratePlp" />
-<meta property="og:description" content="Train various models using a default parameter gird search or user specified parameters" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>recalibratePlp — recalibratePlp • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="recalibratePlp — recalibratePlp"><meta property="og:description" content="Train various models using a default parameter gird search or user specified parameters"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>recalibratePlp</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Recalibration.R'><code>R/Recalibration.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Recalibration.R" class="external-link"><code>R/Recalibration.R</code></a></small>
     <div class="hidden name"><code>recalibratePlp.Rd</code></div>
     </div>
 
@@ -183,67 +131,66 @@ <h1>recalibratePlp</h1>
     <p>Train various models using a default parameter gird search or user specified parameters</p>
     </div>
 
-    <pre class="usage"><span class='fu'>recalibratePlp</span><span class='op'>(</span>
-  <span class='va'>prediction</span>,
-  <span class='va'>analysisId</span>,
-  typeColumn <span class='op'>=</span> <span class='st'>"evaluationType"</span>,
-  method <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='st'>"recalibrationInTheLarge"</span>, <span class='st'>"weakRecalibration"</span><span class='op'>)</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>prediction</th>
-      <td><p>A prediction dataframe</p></td>
-    </tr>
-    <tr>
-      <th>analysisId</th>
-      <td><p>The model analysisId</p></td>
-    </tr>
-    <tr>
-      <th>typeColumn</th>
-      <td><p>The column name where the strata types are specified</p></td>
-    </tr>
-    <tr>
-      <th>method</th>
-      <td><p>Method used to recalibrate ('recalibrationInTheLarge' or 'weakRecalibration' )</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>An object of class <code>runPlp</code> that is recalibrated on the new data</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">recalibratePlp</span><span class="op">(</span></span>
+<span>  <span class="va">prediction</span>,</span>
+<span>  <span class="va">analysisId</span>,</span>
+<span>  typeColumn <span class="op">=</span> <span class="st">"evaluationType"</span>,</span>
+<span>  method <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"recalibrationInTheLarge"</span>, <span class="st">"weakRecalibration"</span><span class="op">)</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>prediction</dt>
+<dd><p>A prediction dataframe</p></dd>
+
+
+<dt>analysisId</dt>
+<dd><p>The model analysisId</p></dd>
+
+
+<dt>typeColumn</dt>
+<dd><p>The column name where the strata types are specified</p></dd>
+
+
+<dt>method</dt>
+<dd><p>Method used to recalibrate ('recalibrationInTheLarge' or 'weakRecalibration' )</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>An object of class <code>runPlp</code> that is recalibrated on the new data</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>The user can define the machine learning model to train (regularised logistic regression, random forest,
 gradient boosting machine, neural network and )</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/recalibratePlpRefit.html b/docs/reference/recalibratePlpRefit.html
index 2af353ae8..f9338fe9b 100644
--- a/docs/reference/recalibratePlpRefit.html
+++ b/docs/reference/recalibratePlpRefit.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>recalibratePlpRefit — recalibratePlpRefit • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="recalibratePlpRefit — recalibratePlpRefit" />
-<meta property="og:description" content="Train various models using a default parameter gird search or user specified parameters" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>recalibratePlpRefit — recalibratePlpRefit • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="recalibratePlpRefit — recalibratePlpRefit"><meta property="og:description" content="Train various models using a default parameter gird search or user specified parameters"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>recalibratePlpRefit</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Recalibration.R'><code>R/Recalibration.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Recalibration.R" class="external-link"><code>R/Recalibration.R</code></a></small>
     <div class="hidden name"><code>recalibratePlpRefit.Rd</code></div>
     </div>
 
@@ -183,59 +131,58 @@ <h1>recalibratePlpRefit</h1>
     <p>Train various models using a default parameter gird search or user specified parameters</p>
     </div>
 
-    <pre class="usage"><span class='fu'>recalibratePlpRefit</span><span class='op'>(</span><span class='va'>plpModel</span>, <span class='va'>newPopulation</span>, <span class='va'>newData</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpModel</th>
-      <td><p>The trained plpModel (runPlp$model)</p></td>
-    </tr>
-    <tr>
-      <th>newPopulation</th>
-      <td><p>The population created using createStudyPopulation() who will have their risks predicted</p></td>
-    </tr>
-    <tr>
-      <th>newData</th>
-      <td><p>An object of type <code>plpData</code> - the patient level prediction
-data extracted from the CDM.</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>An object of class <code>runPlp</code> that is recalibrated on the new data</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">recalibratePlpRefit</span><span class="op">(</span><span class="va">plpModel</span>, <span class="va">newPopulation</span>, <span class="va">newData</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpModel</dt>
+<dd><p>The trained plpModel (runPlp$model)</p></dd>
+
+
+<dt>newPopulation</dt>
+<dd><p>The population created using createStudyPopulation() who will have their risks predicted</p></dd>
+
 
+<dt>newData</dt>
+<dd><p>An object of type <code>plpData</code> - the patient level prediction
+data extracted from the CDM.</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>An object of class <code>runPlp</code> that is recalibrated on the new data</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>The user can define the machine learning model to train (regularised logistic regression, random forest,
 gradient boosting machine, neural network and )</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/registerParallelBackend.html b/docs/reference/registerParallelBackend.html
deleted file mode 100644
index 4d84ea497..000000000
--- a/docs/reference/registerParallelBackend.html
+++ /dev/null
@@ -1,228 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>registerParallelBackend — registerParallelBackend • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="registerParallelBackend — registerParallelBackend" />
-<meta property="og:description" content="Registers a parallel backend for multi core processing. The
-number of cores will be detected automatically, unless specified otherwise." />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.0.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>registerParallelBackend</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Parallel.R'><code>R/Parallel.R</code></a></small>
-    <div class="hidden name"><code>registerParallelBackend.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Registers a parallel backend for multi core processing. The
-number of cores will be detected automatically, unless specified otherwise.</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>registerParallelBackend</span>(<span class='kw'>cores</span> <span class='kw'>=</span> <span class='kw'>NULL</span>, <span class='kw'>logical</span> <span class='kw'>=</span> <span class='fl'>TRUE</span>)</pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>cores</th>
-      <td><p>the number of cores to use for multi core processing</p></td>
-    </tr>
-    <tr>
-      <th>logical</th>
-      <td><p>whether to consider logical or physical cores</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> (<span class='fl'>FALSE</span>) {
-<span class='co'># detect logical cores automatically</span>
-<span class='fu'>registerParallelBackend</span>()
-
-<span class='co'># use four physical cores</span>
-<span class='no'>numCores</span> <span class='kw'>&lt;-</span> <span class='fl'>4</span>
-<span class='fu'>registerParallelBackend</span>(<span class='no'>numCores</span>, <span class='kw'>logical</span> <span class='kw'>=</span> <span class='fl'>FALSE</span>)
-}</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.5.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/registerSequentialBackend.html b/docs/reference/registerSequentialBackend.html
deleted file mode 100644
index 946d45c40..000000000
--- a/docs/reference/registerSequentialBackend.html
+++ /dev/null
@@ -1,212 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>registerSequentialBackend — registerSequentialBackend • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="registerSequentialBackend — registerSequentialBackend" />
-<meta property="og:description" content="registerSequentialBackend registers a sequential backend for
-single core processing." />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.0.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>registerSequentialBackend</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Parallel.R'><code>R/Parallel.R</code></a></small>
-    <div class="hidden name"><code>registerSequentialBackend.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>registerSequentialBackend registers a sequential backend for
-single core processing.</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>registerSequentialBackend</span>()</pre>
-
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> (<span class='fl'>FALSE</span>) {
-<span class='co'># register a sequential backend</span>
-<span class='fu'>registerSequentialBackend</span>()
-}</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.5.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/runEnsembleModel.html b/docs/reference/runEnsembleModel.html
deleted file mode 100644
index 5085dccb2..000000000
--- a/docs/reference/runEnsembleModel.html
+++ /dev/null
@@ -1,333 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>ensemble - Create an ensembling model using different models — runEnsembleModel • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="ensemble - Create an ensembling model using different models — runEnsembleModel" />
-<meta property="og:description" content="#'" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>ensemble - Create an ensembling model using different models</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Ensemble.R'><code>R/Ensemble.R</code></a></small>
-    <div class="hidden name"><code>runEnsembleModel.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>#'</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>runEnsembleModel</span><span class='op'>(</span>
-  <span class='va'>population</span>,
-  <span class='va'>dataList</span>,
-  <span class='va'>modelList</span>,
-  testSplit <span class='op'>=</span> <span class='st'>"time"</span>,
-  testFraction <span class='op'>=</span> <span class='fl'>0.2</span>,
-  stackerUseCV <span class='op'>=</span> <span class='cn'>TRUE</span>,
-  splitSeed <span class='op'>=</span> <span class='cn'>NULL</span>,
-  nfold <span class='op'>=</span> <span class='fl'>3</span>,
-  saveDirectory <span class='op'>=</span> <span class='cn'>NULL</span>,
-  saveEnsemble <span class='op'>=</span> <span class='cn'>F</span>,
-  savePlpData <span class='op'>=</span> <span class='cn'>F</span>,
-  savePlpResult <span class='op'>=</span> <span class='cn'>F</span>,
-  savePlpPlots <span class='op'>=</span> <span class='cn'>F</span>,
-  saveEvaluation <span class='op'>=</span> <span class='cn'>F</span>,
-  analysisId <span class='op'>=</span> <span class='cn'>NULL</span>,
-  verbosity <span class='op'>=</span> <span class='st'>"INFO"</span>,
-  ensembleStrategy <span class='op'>=</span> <span class='st'>"mean"</span>,
-  cores <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>population</th>
-      <td><p>The population created using createStudyPopulation() who will be used to
-develop the model</p></td>
-    </tr>
-    <tr>
-      <th>dataList</th>
-      <td><p>An list of object of type <code>plpData</code> - the patient level prediction
-data extracted from the CDM.</p></td>
-    </tr>
-    <tr>
-      <th>modelList</th>
-      <td><p>An list of type of base model created using one of the function in final
-ensembling model, the base model can be any model implemented in this
-package.</p></td>
-    </tr>
-    <tr>
-      <th>testSplit</th>
-      <td><p>Either 'person' or 'time' specifying the type of evaluation used. 'time'
-find the date where testFraction of patients had an index after the date
-and assigns patients with an index prior to this date into the training
-set and post the date into the test set 'person' splits the data into test
-(1-testFraction of the data) and train (validationFraction of the data)
-sets.  The split is stratified by the class label.</p></td>
-    </tr>
-    <tr>
-      <th>testFraction</th>
-      <td><p>The fraction of the data to be used as the test set in the patient split
-evaluation.</p></td>
-    </tr>
-    <tr>
-      <th>stackerUseCV</th>
-      <td><p>When doing stacking you can either use the train CV predictions to train the stacker (TRUE) or leave 20 percent of the data to train the stacker</p></td>
-    </tr>
-    <tr>
-      <th>splitSeed</th>
-      <td><p>The seed used to split the test/train set when using a person type
-testSplit</p></td>
-    </tr>
-    <tr>
-      <th>nfold</th>
-      <td><p>The number of folds used in the cross validation (default 3)</p></td>
-    </tr>
-    <tr>
-      <th>saveDirectory</th>
-      <td><p>The path to the directory where the results will be saved (if NULL uses working directory)</p></td>
-    </tr>
-    <tr>
-      <th>saveEnsemble</th>
-      <td><p>Binary indicating whether to save the ensemble</p></td>
-    </tr>
-    <tr>
-      <th>savePlpData</th>
-      <td><p>Binary indicating whether to save the plpData object (default is F)</p></td>
-    </tr>
-    <tr>
-      <th>savePlpResult</th>
-      <td><p>Binary indicating whether to save the object returned by runPlp (default is F)</p></td>
-    </tr>
-    <tr>
-      <th>savePlpPlots</th>
-      <td><p>Binary indicating whether to save the performance plots as pdf files (default is F)</p></td>
-    </tr>
-    <tr>
-      <th>saveEvaluation</th>
-      <td><p>Binary indicating whether to save the oerformance as csv files (default is T)</p></td>
-    </tr>
-    <tr>
-      <th>analysisId</th>
-      <td><p>The analysis ID</p></td>
-    </tr>
-    <tr>
-      <th>verbosity</th>
-      <td><p>Sets the level of the verbosity. If the log level is at or higher in
-priority than the logger threshold, a message will print. The levels are:</p><ul>
-<li><p>DEBUGHighest verbosity showing all debug statements</p></li>
-<li><p>TRACEShowing information about start and end of steps</p></li>
-<li><p>INFOShow informative information (Default)</p></li>
-<li><p>WARNShow warning messages</p></li>
-<li><p>ERRORShow error messages</p></li>
-<li><p>FATALBe silent except for fatal errors</p></li>
-</ul></td>
-    </tr>
-    <tr>
-      <th>ensembleStrategy</th>
-      <td><p>The strategy used for ensembling the outputs from different models, it can
-be 'mean', 'product', 'weighted' and 'stacked' 'mean' the average
-probability from differnt models 'product' the product rule 'weighted' the
-weighted average probability from different models using train AUC as
-weights. 'stacked' the stakced ensemble trains a logistics regression on
-different models.</p></td>
-    </tr>
-    <tr>
-      <th>cores</th>
-      <td><p>The number of cores to use when training the ensemble</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>This function applied a list of models and combines them into an ensemble model</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/runMultiplePlp.html b/docs/reference/runMultiplePlp.html
index 2ed6463a2..5e36a1c9f 100644
--- a/docs/reference/runMultiplePlp.html
+++ b/docs/reference/runMultiplePlp.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Run a list of predictions analyses — runMultiplePlp • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Run a list of predictions analyses — runMultiplePlp" />
-<meta property="og:description" content="Run a list of predictions analyses" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Run a list of predictions analyses — runMultiplePlp • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Run a list of predictions analyses — runMultiplePlp"><meta property="og:description" content="Run a list of predictions analyses"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Run a list of predictions analyses</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/RunMultiplePlp.R'><code>R/RunMultiplePlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/RunMultiplePlp.R" class="external-link"><code>R/RunMultiplePlp.R</code></a></small>
     <div class="hidden name"><code>runMultiplePlp.Rd</code></div>
     </div>
 
@@ -183,95 +131,83 @@ <h1>Run a list of predictions analyses</h1>
     <p>Run a list of predictions analyses</p>
     </div>
 
-    <pre class="usage"><span class='fu'>runMultiplePlp</span><span class='op'>(</span>
-  databaseDetails <span class='op'>=</span> <span class='fu'><a href='createDatabaseDetails.html'>createDatabaseDetails</a></span><span class='op'>(</span><span class='op'>)</span>,
-  modelDesignList <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fu'><a href='createModelDesign.html'>createModelDesign</a></span><span class='op'>(</span>targetId <span class='op'>=</span> <span class='fl'>1</span>, outcomeId <span class='op'>=</span> <span class='fl'>2</span>, modelSettings <span class='op'>=</span>
-    <span class='fu'><a href='setLassoLogisticRegression.html'>setLassoLogisticRegression</a></span><span class='op'>(</span><span class='op'>)</span><span class='op'>)</span>, <span class='fu'><a href='createModelDesign.html'>createModelDesign</a></span><span class='op'>(</span>targetId <span class='op'>=</span> <span class='fl'>1</span>, outcomeId <span class='op'>=</span> <span class='fl'>3</span>,
-    modelSettings <span class='op'>=</span> <span class='fu'><a href='setLassoLogisticRegression.html'>setLassoLogisticRegression</a></span><span class='op'>(</span><span class='op'>)</span><span class='op'>)</span><span class='op'>)</span>,
-  onlyFetchData <span class='op'>=</span> <span class='cn'>F</span>,
-  splitSettings <span class='op'>=</span> <span class='fu'><a href='createDefaultSplitSetting.html'>createDefaultSplitSetting</a></span><span class='op'>(</span>type <span class='op'>=</span> <span class='st'>"stratified"</span>, testFraction <span class='op'>=</span> <span class='fl'>0.25</span>,
-    trainFraction <span class='op'>=</span> <span class='fl'>0.75</span>, splitSeed <span class='op'>=</span> <span class='fl'>123</span>, nfold <span class='op'>=</span> <span class='fl'>3</span><span class='op'>)</span>,
-  cohortDefinitions <span class='op'>=</span> <span class='cn'>NULL</span>,
-  logSettings <span class='op'>=</span> <span class='fu'><a href='createLogSettings.html'>createLogSettings</a></span><span class='op'>(</span>verbosity <span class='op'>=</span> <span class='st'>"DEBUG"</span>, timeStamp <span class='op'>=</span> <span class='cn'>T</span>, logName <span class='op'>=</span>
-    <span class='st'>"runPlp Log"</span><span class='op'>)</span>,
-  saveDirectory <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/getwd.html'>getwd</a></span><span class='op'>(</span><span class='op'>)</span>
-<span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">runMultiplePlp</span><span class="op">(</span></span>
+<span>  databaseDetails <span class="op">=</span> <span class="fu"><a href="createDatabaseDetails.html">createDatabaseDetails</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  modelDesignList <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fu"><a href="createModelDesign.html">createModelDesign</a></span><span class="op">(</span>targetId <span class="op">=</span> <span class="fl">1</span>, outcomeId <span class="op">=</span> <span class="fl">2</span>, modelSettings <span class="op">=</span></span>
+<span>    <span class="fu"><a href="setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span><span class="op">)</span>, <span class="fu"><a href="createModelDesign.html">createModelDesign</a></span><span class="op">(</span>targetId <span class="op">=</span> <span class="fl">1</span>, outcomeId <span class="op">=</span> <span class="fl">3</span>,</span>
+<span>    modelSettings <span class="op">=</span> <span class="fu"><a href="setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>,</span>
+<span>  onlyFetchData <span class="op">=</span> <span class="cn">F</span>,</span>
+<span>  cohortDefinitions <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  logSettings <span class="op">=</span> <span class="fu"><a href="createLogSettings.html">createLogSettings</a></span><span class="op">(</span>verbosity <span class="op">=</span> <span class="st">"DEBUG"</span>, timeStamp <span class="op">=</span> <span class="cn">T</span>, logName <span class="op">=</span></span>
+<span>    <span class="st">"runPlp Log"</span><span class="op">)</span>,</span>
+<span>  saveDirectory <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  sqliteLocation <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html" class="external-link">file.path</a></span><span class="op">(</span><span class="va">saveDirectory</span>, <span class="st">"sqlite"</span><span class="op">)</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>databaseDetails</th>
-      <td><p>The database settings created using <code><a href='createDatabaseDetails.html'>createDatabaseDetails()</a></code></p></td>
-    </tr>
-    <tr>
-      <th>modelDesignList</th>
-      <td><p>A list of model designs created using <code><a href='createModelDesign.html'>createModelDesign()</a></code></p></td>
-    </tr>
-    <tr>
-      <th>onlyFetchData</th>
-      <td><p>Only fetches and saves the data object to the output folder without running the analysis.</p></td>
-    </tr>
-    <tr>
-      <th>splitSettings</th>
-      <td><p>The train/validation/test splitting used by all analyses created using <code><a href='createDefaultSplitSetting.html'>createDefaultSplitSetting()</a></code></p></td>
-    </tr>
-    <tr>
-      <th>cohortDefinitions</th>
-      <td><p>A list of cohort definitions for the target and outcome cohorts</p></td>
-    </tr>
-    <tr>
-      <th>logSettings</th>
-      <td><p>The setting spexcifying the logging for the analyses created using <code><a href='createLogSettings.html'>createLogSettings()</a></code></p></td>
-    </tr>
-    <tr>
-      <th>saveDirectory</th>
-      <td><p>Name of the folder where all the outputs will written to.</p></td>
-    </tr>
-    </table>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>databaseDetails</dt>
+<dd><p>The database settings created using <code><a href="createDatabaseDetails.html">createDatabaseDetails()</a></code></p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
 
-    <p>A data frame with the following columns:</p><table class='table'>
-<tr><td><code>analysisId</code></td><td>The unique identifier
-for a set of analysis choices.</td></tr>
-<tr><td><code>cohortId</code></td><td>The ID of the target cohort populations.</td></tr>
-<tr><td><code>outcomeId</code></td><td>The ID of the outcomeId.</td></tr>
-<tr><td><code>dataLocation</code></td><td>The location where the plpData was saved</td></tr>
-<tr><td><code>evaluationFolder</code></td><td>The name of file containing the evaluation saved as a csv</td></tr>
-<tr><td><code>the settings ids</code></td><td>The ids for all other settings used for model development.</td></tr>
-</table>
+<dt>modelDesignList</dt>
+<dd><p>A list of model designs created using <code><a href="createModelDesign.html">createModelDesign()</a></code></p></dd>
 
 
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<dt>onlyFetchData</dt>
+<dd><p>Only fetches and saves the data object to the output folder without running the analysis.</p></dd>
 
+
+<dt>cohortDefinitions</dt>
+<dd><p>A list of cohort definitions for the target and outcome cohorts</p></dd>
+
+
+<dt>logSettings</dt>
+<dd><p>The setting specifying the logging for the analyses created using <code><a href="createLogSettings.html">createLogSettings()</a></code></p></dd>
+
+
+<dt>saveDirectory</dt>
+<dd><p>Name of the folder where all the outputs will written to.</p></dd>
+
+
+<dt>sqliteLocation</dt>
+<dd><p>(optional) The location of the sqlite database with the results</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>A data frame with the following columns:</p><table class="table table"><tr><td><code>analysisId</code></td><td>The unique identifier
+for a set of analysis choices.</td></tr><tr><td><code>targetId</code></td><td>The ID of the target cohort populations.</td></tr><tr><td><code>outcomeId</code></td><td>The ID of the outcomeId.</td></tr><tr><td><code>dataLocation</code></td><td>The location where the plpData was saved</td></tr><tr><td><code>the settings ids</code></td><td>The ids for all other settings used for model development.</td></tr></table></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>This function will run all specified predictions as defined using .</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/runPlp.html b/docs/reference/runPlp.html
index d3a6a999e..2eb237596 100644
--- a/docs/reference/runPlp.html
+++ b/docs/reference/runPlp.html
@@ -1,72 +1,17 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>runPlp - Develop and internally evaluate a model using specified settings — runPlp • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="runPlp - Develop and internally evaluate a model using specified settings — runPlp" />
-<meta property="og:description" content="This provides a general framework for training patient level prediction models.  The user can select 
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>runPlp - Develop and internally evaluate a model using specified settings — runPlp • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="runPlp - Develop and internally evaluate a model using specified settings — runPlp"><meta property="og:description" content="This provides a general framework for training patient level prediction models.  The user can select 
 various default feature selection methods or incorporate their own,  The user can also select from
 a range of default classifiers or incorporate their own.  There are three types of evaluations for the model
 patient (randomly splits people into train/validation sets) or year (randomly splits data into train/validation sets
 based on index year - older in training, newer in validation) or both (same as year spliting but checks there are
-no overlaps in patients within training set and validaiton set - any overlaps are removed from validation set)" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+no overlaps in patients within training set and validaiton set - any overlaps are removed from validation set)"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -77,13 +22,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -99,13 +43,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -118,10 +61,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -130,13 +73,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -144,43 +90,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>runPlp - Develop and internally evaluate a model using specified settings</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/RunPlp.R'><code>R/RunPlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/RunPlp.R" class="external-link"><code>R/RunPlp.R</code></a></small>
     <div class="hidden name"><code>runPlp.Rd</code></div>
     </div>
 
@@ -193,145 +141,139 @@ <h1>runPlp - Develop and internally evaluate a model using specified settings</h
 no overlaps in patients within training set and validaiton set - any overlaps are removed from validation set)</p>
     </div>
 
-    <pre class="usage"><span class='fu'>runPlp</span><span class='op'>(</span>
-  <span class='va'>plpData</span>,
-  outcomeId <span class='op'>=</span> <span class='va'>plpData</span><span class='op'>$</span><span class='va'>metaData</span><span class='op'>$</span><span class='va'>call</span><span class='op'>$</span><span class='va'>outcomeIds</span><span class='op'>[</span><span class='fl'>1</span><span class='op'>]</span>,
-  analysisId <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/paste.html'>paste</a></span><span class='op'>(</span><span class='fu'><a href='https://rdrr.io/r/base/Sys.time.html'>Sys.Date</a></span><span class='op'>(</span><span class='op'>)</span>, <span class='va'>plpData</span><span class='op'>$</span><span class='va'>metaData</span><span class='op'>$</span><span class='va'>call</span><span class='op'>$</span><span class='va'>outcomeIds</span><span class='op'>[</span><span class='fl'>1</span><span class='op'>]</span>, sep <span class='op'>=</span> <span class='st'>"-"</span><span class='op'>)</span>,
-  analysisName <span class='op'>=</span> <span class='st'>"Study details"</span>,
-  populationSettings <span class='op'>=</span> <span class='fu'><a href='createStudyPopulationSettings.html'>createStudyPopulationSettings</a></span><span class='op'>(</span><span class='op'>)</span>,
-  splitSettings <span class='op'>=</span> <span class='fu'><a href='createDefaultSplitSetting.html'>createDefaultSplitSetting</a></span><span class='op'>(</span>type <span class='op'>=</span> <span class='st'>"stratified"</span>, testFraction <span class='op'>=</span> <span class='fl'>0.25</span>,
-    trainFraction <span class='op'>=</span> <span class='fl'>0.75</span>, splitSeed <span class='op'>=</span> <span class='fl'>123</span>, nfold <span class='op'>=</span> <span class='fl'>3</span><span class='op'>)</span>,
-  sampleSettings <span class='op'>=</span> <span class='fu'><a href='createSampleSettings.html'>createSampleSettings</a></span><span class='op'>(</span>type <span class='op'>=</span> <span class='st'>"none"</span><span class='op'>)</span>,
-  featureEngineeringSettings <span class='op'>=</span> <span class='fu'><a href='createFeatureEngineeringSettings.html'>createFeatureEngineeringSettings</a></span><span class='op'>(</span>type <span class='op'>=</span> <span class='st'>"none"</span><span class='op'>)</span>,
-  preprocessSettings <span class='op'>=</span> <span class='fu'><a href='createPreprocessSettings.html'>createPreprocessSettings</a></span><span class='op'>(</span>minFraction <span class='op'>=</span> <span class='fl'>0.001</span>, normalize <span class='op'>=</span> <span class='cn'>T</span><span class='op'>)</span>,
-  modelSettings <span class='op'>=</span> <span class='fu'><a href='setLassoLogisticRegression.html'>setLassoLogisticRegression</a></span><span class='op'>(</span><span class='op'>)</span>,
-  logSettings <span class='op'>=</span> <span class='fu'><a href='createLogSettings.html'>createLogSettings</a></span><span class='op'>(</span>verbosity <span class='op'>=</span> <span class='st'>"DEBUG"</span>, timeStamp <span class='op'>=</span> <span class='cn'>T</span>, logName <span class='op'>=</span>
-    <span class='st'>"runPlp Log"</span><span class='op'>)</span>,
-  executeSettings <span class='op'>=</span> <span class='fu'><a href='createDefaultExecuteSettings.html'>createDefaultExecuteSettings</a></span><span class='op'>(</span><span class='op'>)</span>,
-  saveDirectory <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/getwd.html'>getwd</a></span><span class='op'>(</span><span class='op'>)</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpData</th>
-      <td><p>An object of type <code>plpData</code> - the patient level prediction
-data extracted from the CDM.</p></td>
-    </tr>
-    <tr>
-      <th>outcomeId</th>
-      <td><p>(integer) The ID of the outcome.</p></td>
-    </tr>
-    <tr>
-      <th>analysisId</th>
-      <td><p>(integer) Identifier for the analysis. It is used to create, e.g., the result folder. Default is a timestamp.</p></td>
-    </tr>
-    <tr>
-      <th>analysisName</th>
-      <td><p>(character) Name for the analysis</p></td>
-    </tr>
-    <tr>
-      <th>populationSettings</th>
-      <td><p>An object of type <code>populationSettings</code> created using <code>createStudyPopulationSettings</code> that
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">runPlp</span><span class="op">(</span></span>
+<span>  <span class="va">plpData</span>,</span>
+<span>  outcomeId <span class="op">=</span> <span class="va">plpData</span><span class="op">$</span><span class="va">metaData</span><span class="op">$</span><span class="va">call</span><span class="op">$</span><span class="va">outcomeIds</span><span class="op">[</span><span class="fl">1</span><span class="op">]</span>,</span>
+<span>  analysisId <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html" class="external-link">paste</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/Sys.time.html" class="external-link">Sys.Date</a></span><span class="op">(</span><span class="op">)</span>, <span class="va">plpData</span><span class="op">$</span><span class="va">metaData</span><span class="op">$</span><span class="va">call</span><span class="op">$</span><span class="va">outcomeIds</span><span class="op">[</span><span class="fl">1</span><span class="op">]</span>, sep <span class="op">=</span> <span class="st">"-"</span><span class="op">)</span>,</span>
+<span>  analysisName <span class="op">=</span> <span class="st">"Study details"</span>,</span>
+<span>  populationSettings <span class="op">=</span> <span class="fu"><a href="createStudyPopulationSettings.html">createStudyPopulationSettings</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  splitSettings <span class="op">=</span> <span class="fu"><a href="createDefaultSplitSetting.html">createDefaultSplitSetting</a></span><span class="op">(</span>type <span class="op">=</span> <span class="st">"stratified"</span>, testFraction <span class="op">=</span> <span class="fl">0.25</span>,</span>
+<span>    trainFraction <span class="op">=</span> <span class="fl">0.75</span>, splitSeed <span class="op">=</span> <span class="fl">123</span>, nfold <span class="op">=</span> <span class="fl">3</span><span class="op">)</span>,</span>
+<span>  sampleSettings <span class="op">=</span> <span class="fu"><a href="createSampleSettings.html">createSampleSettings</a></span><span class="op">(</span>type <span class="op">=</span> <span class="st">"none"</span><span class="op">)</span>,</span>
+<span>  featureEngineeringSettings <span class="op">=</span> <span class="fu"><a href="createFeatureEngineeringSettings.html">createFeatureEngineeringSettings</a></span><span class="op">(</span>type <span class="op">=</span> <span class="st">"none"</span><span class="op">)</span>,</span>
+<span>  preprocessSettings <span class="op">=</span> <span class="fu"><a href="createPreprocessSettings.html">createPreprocessSettings</a></span><span class="op">(</span>minFraction <span class="op">=</span> <span class="fl">0.001</span>, normalize <span class="op">=</span> <span class="cn">T</span><span class="op">)</span>,</span>
+<span>  modelSettings <span class="op">=</span> <span class="fu"><a href="setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  logSettings <span class="op">=</span> <span class="fu"><a href="createLogSettings.html">createLogSettings</a></span><span class="op">(</span>verbosity <span class="op">=</span> <span class="st">"DEBUG"</span>, timeStamp <span class="op">=</span> <span class="cn">T</span>, logName <span class="op">=</span></span>
+<span>    <span class="st">"runPlp Log"</span><span class="op">)</span>,</span>
+<span>  executeSettings <span class="op">=</span> <span class="fu"><a href="createDefaultExecuteSettings.html">createDefaultExecuteSettings</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  saveDirectory <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpData</dt>
+<dd><p>An object of type <code>plpData</code> - the patient level prediction
+data extracted from the CDM.  Can also include an initial population as 
+plpData$popualtion.</p></dd>
+
+
+<dt>outcomeId</dt>
+<dd><p>(integer) The ID of the outcome.</p></dd>
+
+
+<dt>analysisId</dt>
+<dd><p>(integer) Identifier for the analysis. It is used to create, e.g., the result folder. Default is a timestamp.</p></dd>
+
+
+<dt>analysisName</dt>
+<dd><p>(character) Name for the analysis</p></dd>
+
+
+<dt>populationSettings</dt>
+<dd><p>An object of type <code>populationSettings</code> created using <code>createStudyPopulationSettings</code> that
 specifies how the data class labels are defined and addition any exclusions to apply to the 
-plpData cohort</p></td>
-    </tr>
-    <tr>
-      <th>splitSettings</th>
-      <td><p>An object of type <code>splitSettings</code> that specifies how to split the data into train/validation/test.
-The default settings can be created using <code>createDefaultSplitSetting</code>.</p></td>
-    </tr>
-    <tr>
-      <th>sampleSettings</th>
-      <td><p>An object of type <code>sampleSettings</code> that specifies any under/over sampling to be done.
-The default is none.</p></td>
-    </tr>
-    <tr>
-      <th>featureEngineeringSettings</th>
-      <td><p>An object of <code>featureEngineeringSettings</code> specifying any feature engineering to be learned (using the train data)</p></td>
-    </tr>
-    <tr>
-      <th>preprocessSettings</th>
-      <td><p>An object of <code>preprocessSettings</code>. This setting specifies the minimum fraction of 
+plpData cohort</p></dd>
+
+
+<dt>splitSettings</dt>
+<dd><p>An object of type <code>splitSettings</code> that specifies how to split the data into train/validation/test.
+The default settings can be created using <code>createDefaultSplitSetting</code>.</p></dd>
+
+
+<dt>sampleSettings</dt>
+<dd><p>An object of type <code>sampleSettings</code> that specifies any under/over sampling to be done.
+The default is none.</p></dd>
+
+
+<dt>featureEngineeringSettings</dt>
+<dd><p>An object of <code>featureEngineeringSettings</code> specifying any feature engineering to be learned (using the train data)</p></dd>
+
+
+<dt>preprocessSettings</dt>
+<dd><p>An object of <code>preprocessSettings</code>. This setting specifies the minimum fraction of 
 target population who must have a covariate for it to be included in the model training                            
-and whether to normalise the covariates before training</p></td>
-    </tr>
-    <tr>
-      <th>modelSettings</th>
-      <td><p>An object of class <code>modelSettings</code> created using one of the function:</p><ul>
-<li><p>setLassoLogisticRegression() A lasso logistic regression model</p></li>
+and whether to normalise the covariates before training</p></dd>
+
+
+<dt>modelSettings</dt>
+<dd><p>An object of class <code>modelSettings</code> created using one of the function:</p><ul><li><p>setLassoLogisticRegression() A lasso logistic regression model</p></li>
 <li><p>setGradientBoostingMachine() A gradient boosting machine</p></li>
 <li><p>setAdaBoost() An ada boost model</p></li>
 <li><p>setRandomForest() A random forest model</p></li>
 <li><p>setDecisionTree() A decision tree model</p></li>
-<li><p>setCovNN()) A convolutional neural network model</p></li>
-<li><p>setCIReNN() A recurrent neural network model</p></li>
-<li><p>setMLP() A neural network model</p></li>
-<li><p>setDeepNN() A deep neural network model</p></li>
 <li><p>setKNN() A KNN model</p></li>
-</ul></td>
-    </tr>
-    <tr>
-      <th>logSettings</th>
-      <td><p>An object of <code>logSettings</code> created using <code>createLogSettings</code> 
-specifying how the logging is done</p></td>
-    </tr>
-    <tr>
-      <th>executeSettings</th>
-      <td><p>An object of <code>executeSettings</code> specifying which parts of the analysis to run</p></td>
-    </tr>
-    <tr>
-      <th>saveDirectory</th>
-      <td><p>The path to the directory where the results will be saved (if NULL uses working directory)</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>An object containing the following:</p>
-<p></p><ul>
-<li><p>inputSettingsA list containing all the settings used to develop the model</p></li>
-<li><p>model The developed model of class <code>plpModel</code></p></li>
+</ul></dd>
+
+
+<dt>logSettings</dt>
+<dd><p>An object of <code>logSettings</code> created using <code>createLogSettings</code> 
+specifying how the logging is done</p></dd>
+
+
+<dt>executeSettings</dt>
+<dd><p>An object of <code>executeSettings</code> specifying which parts of the analysis to run</p></dd>
+
+
+<dt>saveDirectory</dt>
+<dd><p>The path to the directory where the results will be saved (if NULL uses working directory)</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>An object containing the following:</p>
+<p></p>
+<ul><li><p>model The developed model of class <code>plpModel</code></p></li>
 <li><p>executionSummary A list containing the hardward details, R package details and execution time</p></li>
 <li><p>performanceEvaluation Various internal performance metrics in sparse format</p></li>
 <li><p>prediction The plpData cohort table with the predicted risks added as a column (named value)</p></li>
-<li><p>covariateSummary) A characterization of the features for patients with and without the outcome during the time at risk</p></li>
+<li><p>covariateSummary A characterization of the features for patients with and without the outcome during the time at risk</p></li>
 <li><p>analysisRef A list with details about the analysis</p></li>
-</ul>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
+</ul></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>This function takes as input the plpData extracted from an OMOP CDM database and follows the specified settings to
 develop and internally validate a model for the specified outcomeId.</p>
+    </div>
 
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"></pre>
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/runPlpAnalyses.html b/docs/reference/runPlpAnalyses.html
deleted file mode 100644
index 4ab0d4ed9..000000000
--- a/docs/reference/runPlpAnalyses.html
+++ /dev/null
@@ -1,383 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Run a list of predictions — runPlpAnalyses • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Run a list of predictions — runPlpAnalyses" />
-<meta property="og:description" content="Run a list of predictions" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Run a list of predictions</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/RunMultiplePlp.R'><code>R/RunMultiplePlp.R</code></a></small>
-    <div class="hidden name"><code>runPlpAnalyses.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Run a list of predictions</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>runPlpAnalyses</span><span class='op'>(</span>
-  <span class='va'>connectionDetails</span>,
-  <span class='va'>cdmDatabaseSchema</span>,
-  <span class='va'>cdmDatabaseName</span>,
-  oracleTempSchema <span class='op'>=</span> <span class='va'>cdmDatabaseSchema</span>,
-  cohortDatabaseSchema <span class='op'>=</span> <span class='va'>cdmDatabaseSchema</span>,
-  cohortTable <span class='op'>=</span> <span class='st'>"cohort"</span>,
-  outcomeDatabaseSchema <span class='op'>=</span> <span class='va'>cdmDatabaseSchema</span>,
-  outcomeTable <span class='op'>=</span> <span class='st'>"cohort"</span>,
-  cdmVersion <span class='op'>=</span> <span class='fl'>5</span>,
-  onlyFetchData <span class='op'>=</span> <span class='cn'>FALSE</span>,
-  outputFolder <span class='op'>=</span> <span class='st'>"./PlpOutput"</span>,
-  <span class='va'>modelAnalysisList</span>,
-  <span class='va'>cohortIds</span>,
-  <span class='va'>cohortNames</span>,
-  <span class='va'>outcomeIds</span>,
-  <span class='va'>outcomeNames</span>,
-  washoutPeriod <span class='op'>=</span> <span class='fl'>0</span>,
-  maxSampleSize <span class='op'>=</span> <span class='cn'>NULL</span>,
-  minCovariateFraction <span class='op'>=</span> <span class='fl'>0</span>,
-  normalizeData <span class='op'>=</span> <span class='cn'>T</span>,
-  testSplit <span class='op'>=</span> <span class='st'>"person"</span>,
-  testFraction <span class='op'>=</span> <span class='fl'>0.25</span>,
-  splitSeed <span class='op'>=</span> <span class='cn'>NULL</span>,
-  nfold <span class='op'>=</span> <span class='fl'>3</span>,
-  verbosity <span class='op'>=</span> <span class='st'>"INFO"</span>,
-  settings <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>connectionDetails</th>
-      <td><p>An R object of type <code>connectionDetails</code> created using the
-function <code>createConnectionDetails</code> in the
-<code>DatabaseConnector</code> package.</p></td>
-    </tr>
-    <tr>
-      <th>cdmDatabaseSchema</th>
-      <td><p>The name of the database schema that contains the OMOP CDM
-instance. Requires read permissions to this database. On SQL
-Server, this should specifiy both the database and the schema,
-so for example 'cdm_instance.dbo'.</p></td>
-    </tr>
-    <tr>
-      <th>cdmDatabaseName</th>
-      <td><p>A string with a shareable name of the database (this will be shown to OHDSI researchers if the results get transported)</p></td>
-    </tr>
-    <tr>
-      <th>oracleTempSchema</th>
-      <td><p>For Oracle only: the name of the database schema where you
-want all temporary tables to be managed. Requires
-create/insert permissions to this database.</p></td>
-    </tr>
-    <tr>
-      <th>cohortDatabaseSchema</th>
-      <td><p>The name of the database schema that is the location where the
-target cohorts are available.  Requires read
-permissions to this database.</p></td>
-    </tr>
-    <tr>
-      <th>cohortTable</th>
-      <td><p>The tablename that contains the target cohorts.  Expectation is cohortTable
-has format of COHORT table: COHORT_DEFINITION_ID, SUBJECT_ID,
-COHORT_START_DATE, COHORT_END_DATE.</p></td>
-    </tr>
-    <tr>
-      <th>outcomeDatabaseSchema</th>
-      <td><p>The name of the database schema that is the location where the
-data used to define the outcome cohorts is available. Requires read permissions to
-this database.</p></td>
-    </tr>
-    <tr>
-      <th>outcomeTable</th>
-      <td><p>The tablename that contains the outcome cohorts.  Expectation is
-outcomeTable has format of COHORT table: COHORT_DEFINITION_ID,
-SUBJECT_ID, COHORT_START_DATE, COHORT_END_DATE.</p></td>
-    </tr>
-    <tr>
-      <th>cdmVersion</th>
-      <td><p>Define the OMOP CDM version used: currently support "4" and
-"5".</p></td>
-    </tr>
-    <tr>
-      <th>onlyFetchData</th>
-      <td><p>Only fetches and saves the data object to the output folder without running the analysis.</p></td>
-    </tr>
-    <tr>
-      <th>outputFolder</th>
-      <td><p>Name of the folder where all the outputs will written to.</p></td>
-    </tr>
-    <tr>
-      <th>modelAnalysisList</th>
-      <td><p>A list of objects of type <code>modelSettings</code> as created using
-the <code><a href='createPlpModelSettings.html'>createPlpModelSettings</a></code> function.</p></td>
-    </tr>
-    <tr>
-      <th>cohortIds</th>
-      <td><p>A vector of cohortIds that specify all the target cohorts</p></td>
-    </tr>
-    <tr>
-      <th>cohortNames</th>
-      <td><p>A vector of cohortNames corresponding to the cohortIds</p></td>
-    </tr>
-    <tr>
-      <th>outcomeIds</th>
-      <td><p>A vector of outcomeIds that specify all the outcome cohorts</p></td>
-    </tr>
-    <tr>
-      <th>outcomeNames</th>
-      <td><p>A vector of outcomeNames corresponding to the outcomeIds</p></td>
-    </tr>
-    <tr>
-      <th>washoutPeriod</th>
-      <td><p>Minimum number of prior observation days</p></td>
-    </tr>
-    <tr>
-      <th>maxSampleSize</th>
-      <td><p>Max number of target people to sample from to develop models</p></td>
-    </tr>
-    <tr>
-      <th>minCovariateFraction</th>
-      <td><p>Any covariate with an incidence less than this value if ignored</p></td>
-    </tr>
-    <tr>
-      <th>normalizeData</th>
-      <td><p>Whether to normalize the covariates</p></td>
-    </tr>
-    <tr>
-      <th>testSplit</th>
-      <td><p>How to split into test/train (time or person)</p></td>
-    </tr>
-    <tr>
-      <th>testFraction</th>
-      <td><p>Fraction of data to use as test set</p></td>
-    </tr>
-    <tr>
-      <th>splitSeed</th>
-      <td><p>The seed used for the randomization into test/train</p></td>
-    </tr>
-    <tr>
-      <th>nfold</th>
-      <td><p>Number of folds used to do cross validation</p></td>
-    </tr>
-    <tr>
-      <th>verbosity</th>
-      <td><p>The logging level</p></td>
-    </tr>
-    <tr>
-      <th>settings</th>
-      <td><p>Specify the T, O, population, covariate and model settings</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A data frame with the following columns:</p><table class='table'>
-<tr><td><code>analysisId</code></td><td>The unique identifier
-for a set of analysis choices.</td></tr>
-<tr><td><code>cohortId</code></td><td>The ID of the target cohort populations.</td></tr>
-<tr><td><code>outcomeId</code></td><td>The ID of the outcomeId.</td></tr>
-<tr><td><code>plpDataFolder</code></td><td>The location where the plpData was saved</td></tr>
-<tr><td><code>studyPopFile</code></td><td>The
-name of the file containing the study population</td></tr>
-<tr><td><code>evaluationFolder</code></td><td>The name of file containing the evaluation saved as a csv</td></tr>
-<tr><td><code>modelFolder</code></td><td>The name of the file containing the developed model.</td></tr>
-</table>
-
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Run a list of predictions for the target cohorts and outcomes of interest. This function will run all
-specified predictions, meaning that the total number of outcome
-models is `length(cohortIds) * length(outcomeIds) * length(modelAnalysisList)`.</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/saveEnsemblePlpModel.html b/docs/reference/saveEnsemblePlpModel.html
deleted file mode 100644
index 18b08bff5..000000000
--- a/docs/reference/saveEnsemblePlpModel.html
+++ /dev/null
@@ -1,226 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>saves the Ensmeble plp model — saveEnsemblePlpModel • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="saves the Ensmeble plp model — saveEnsemblePlpModel" />
-<meta property="og:description" content="saves the Ensmeble plp model" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>saves the Ensmeble plp model</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Ensemble.R'><code>R/Ensemble.R</code></a></small>
-    <div class="hidden name"><code>saveEnsemblePlpModel.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>saves the Ensmeble plp model</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>saveEnsemblePlpModel</span><span class='op'>(</span><span class='va'>ensembleModel</span>, <span class='va'>dirPath</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>ensembleModel</th>
-      <td><p>The ensemble model to save</p></td>
-    </tr>
-    <tr>
-      <th>dirPath</th>
-      <td><p>The location to save the model</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Saves a plp ensemble model</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/saveEnsemblePlpResult.html b/docs/reference/saveEnsemblePlpResult.html
deleted file mode 100644
index 323846315..000000000
--- a/docs/reference/saveEnsemblePlpResult.html
+++ /dev/null
@@ -1,226 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>saves the Ensemble plp results — saveEnsemblePlpResult • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="saves the Ensemble plp results — saveEnsemblePlpResult" />
-<meta property="og:description" content="saves the Ensemble plp results" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>saves the Ensemble plp results</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Ensemble.R'><code>R/Ensemble.R</code></a></small>
-    <div class="hidden name"><code>saveEnsemblePlpResult.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>saves the Ensemble plp results</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>saveEnsemblePlpResult</span><span class='op'>(</span><span class='va'>ensembleResult</span>, <span class='va'>dirPath</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>ensembleResult</th>
-      <td><p>The ensemble result</p></td>
-    </tr>
-    <tr>
-      <th>dirPath</th>
-      <td><p>The location to save the ensemble results</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Saves a plp ensemble results</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/savePlpAnalysesJson.html b/docs/reference/savePlpAnalysesJson.html
index d5d6cc2aa..675b05e57 100644
--- a/docs/reference/savePlpAnalysesJson.html
+++ b/docs/reference/savePlpAnalysesJson.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Save the modelDesignList to a json file — savePlpAnalysesJson • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Save the modelDesignList to a json file — savePlpAnalysesJson" />
-<meta property="og:description" content="Save the modelDesignList to a json file" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Save the modelDesignList to a json file — savePlpAnalysesJson • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Save the modelDesignList to a json file — savePlpAnalysesJson"><meta property="og:description" content="Save the modelDesignList to a json file"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Save the modelDesignList to a json file</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/RunMultiplePlp.R'><code>R/RunMultiplePlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/RunMultiplePlp.R" class="external-link"><code>R/RunMultiplePlp.R</code></a></small>
     <div class="hidden name"><code>savePlpAnalysesJson.Rd</code></div>
     </div>
 
@@ -183,67 +131,70 @@ <h1>Save the modelDesignList to a json file</h1>
     <p>Save the modelDesignList to a json file</p>
     </div>
 
-    <pre class="usage"><span class='fu'>savePlpAnalysesJson</span><span class='op'>(</span>
-  modelDesignList <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fu'><a href='createModelDesign.html'>createModelDesign</a></span><span class='op'>(</span>targetId <span class='op'>=</span> <span class='fl'>1</span>, outcomeId <span class='op'>=</span> <span class='fl'>2</span>, modelSettings <span class='op'>=</span>
-    <span class='fu'><a href='setLassoLogisticRegression.html'>setLassoLogisticRegression</a></span><span class='op'>(</span><span class='op'>)</span><span class='op'>)</span>, <span class='fu'><a href='createModelDesign.html'>createModelDesign</a></span><span class='op'>(</span>targetId <span class='op'>=</span> <span class='fl'>1</span>, outcomeId <span class='op'>=</span> <span class='fl'>3</span>,
-    modelSettings <span class='op'>=</span> <span class='fu'><a href='setLassoLogisticRegression.html'>setLassoLogisticRegression</a></span><span class='op'>(</span><span class='op'>)</span><span class='op'>)</span><span class='op'>)</span>,
-  saveDirectory <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>modelDesignList</th>
-      <td><p>A list of modelDesigns created using <code><a href='createModelDesign.html'>createModelDesign()</a></code></p></td>
-    </tr>
-    <tr>
-      <th>saveDirectory</th>
-      <td><p>The directory to save the modelDesignList settings</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">savePlpAnalysesJson</span><span class="op">(</span></span>
+<span>  modelDesignList <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fu"><a href="createModelDesign.html">createModelDesign</a></span><span class="op">(</span>targetId <span class="op">=</span> <span class="fl">1</span>, outcomeId <span class="op">=</span> <span class="fl">2</span>, modelSettings <span class="op">=</span></span>
+<span>    <span class="fu"><a href="setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span><span class="op">)</span>, <span class="fu"><a href="createModelDesign.html">createModelDesign</a></span><span class="op">(</span>targetId <span class="op">=</span> <span class="fl">1</span>, outcomeId <span class="op">=</span> <span class="fl">3</span>,</span>
+<span>    modelSettings <span class="op">=</span> <span class="fu"><a href="setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>,</span>
+<span>  cohortDefinitions <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  saveDirectory <span class="op">=</span> <span class="cn">NULL</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>modelDesignList</dt>
+<dd><p>A list of modelDesigns created using <code><a href="createModelDesign.html">createModelDesign()</a></code></p></dd>
+
+
+<dt>cohortDefinitions</dt>
+<dd><p>A list of the cohortDefinitions (generally extracted from ATLAS)</p></dd>
 
+
+<dt>saveDirectory</dt>
+<dd><p>The directory to save the modelDesignList settings</p></dd>
+
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>This function creates a json file with the modelDesignList saved</p>
+    </div>
 
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='fu'>savePlpAnalysesJson</span><span class='op'>(</span>
-modelDesignList <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span>
-<span class='fu'><a href='createModelDesign.html'>createModelDesign</a></span><span class='op'>(</span>targetId <span class='op'>=</span> <span class='fl'>1</span>, outcomeId <span class='op'>=</span> <span class='fl'>2</span>, modelSettings <span class='op'>=</span> <span class='fu'><a href='setLassoLogisticRegression.html'>setLassoLogisticRegression</a></span><span class='op'>(</span><span class='op'>)</span><span class='op'>)</span>, 
-<span class='fu'><a href='createModelDesign.html'>createModelDesign</a></span><span class='op'>(</span>targetId <span class='op'>=</span> <span class='fl'>1</span>, outcomeId <span class='op'>=</span> <span class='fl'>3</span>, modelSettings <span class='op'>=</span> <span class='fu'><a href='setLassoLogisticRegression.html'>setLassoLogisticRegression</a></span><span class='op'>(</span><span class='op'>)</span><span class='op'>)</span>
-<span class='op'>)</span>,
-saveDirectory <span class='op'>=</span> <span class='st'>'C:/bestModels'</span>
-<span class='op'>)</span>
-<span class='op'>}</span>
-
-</div></pre>
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="kw">if</span> <span class="op">(</span><span class="cn">FALSE</span><span class="op">)</span> <span class="op">{</span></span></span>
+<span class="r-in"><span><span class="fu">savePlpAnalysesJson</span><span class="op">(</span></span></span>
+<span class="r-in"><span>modelDesignList <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span></span></span>
+<span class="r-in"><span><span class="fu"><a href="createModelDesign.html">createModelDesign</a></span><span class="op">(</span>targetId <span class="op">=</span> <span class="fl">1</span>, outcomeId <span class="op">=</span> <span class="fl">2</span>, modelSettings <span class="op">=</span> <span class="fu"><a href="setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span><span class="op">)</span>, </span></span>
+<span class="r-in"><span><span class="fu"><a href="createModelDesign.html">createModelDesign</a></span><span class="op">(</span>targetId <span class="op">=</span> <span class="fl">1</span>, outcomeId <span class="op">=</span> <span class="fl">3</span>, modelSettings <span class="op">=</span> <span class="fu"><a href="setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span><span class="op">)</span></span></span>
+<span class="r-in"><span><span class="op">)</span>,</span></span>
+<span class="r-in"><span>saveDirectory <span class="op">=</span> <span class="st">'C:/bestModels'</span></span></span>
+<span class="r-in"><span><span class="op">)</span></span></span>
+<span class="r-in"><span><span class="op">}</span></span></span>
+<span class="r-in"><span></span></span>
+</code></pre></div>
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/savePlpData.html b/docs/reference/savePlpData.html
index e31608d82..54a2af0ae 100644
--- a/docs/reference/savePlpData.html
+++ b/docs/reference/savePlpData.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Save the cohort data to folder — savePlpData • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Save the cohort data to folder — savePlpData" />
-<meta property="og:description" content="savePlpData saves an object of type plpData to folder." />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Save the cohort data to folder — savePlpData • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Save the cohort data to folder — savePlpData"><meta property="og:description" content="savePlpData saves an object of type plpData to folder."><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Save the cohort data to folder</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SaveLoadPlp.R'><code>R/SaveLoadPlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SaveLoadPlp.R" class="external-link"><code>R/SaveLoadPlp.R</code></a></small>
     <div class="hidden name"><code>savePlpData.Rd</code></div>
     </div>
 
@@ -183,64 +131,63 @@ <h1>Save the cohort data to folder</h1>
     <p><code>savePlpData</code> saves an object of type plpData to folder.</p>
     </div>
 
-    <pre class="usage"><span class='fu'>savePlpData</span><span class='op'>(</span><span class='va'>plpData</span>, <span class='va'>file</span>, envir <span class='op'>=</span> <span class='cn'>NULL</span>, overwrite <span class='op'>=</span> <span class='cn'>F</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpData</th>
-      <td><p>An object of type <code>plpData</code> as generated using
-<code>getPlpData</code>.</p></td>
-    </tr>
-    <tr>
-      <th>file</th>
-      <td><p>The name of the folder where the data will be written. The folder should
-not yet exist.</p></td>
-    </tr>
-    <tr>
-      <th>envir</th>
-      <td><p>The environment for to evaluate variables when saving</p></td>
-    </tr>
-    <tr>
-      <th>overwrite</th>
-      <td><p>Whether to force overwrite an existing file</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">savePlpData</span><span class="op">(</span><span class="va">plpData</span>, <span class="va">file</span>, envir <span class="op">=</span> <span class="cn">NULL</span>, overwrite <span class="op">=</span> <span class="cn">F</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpData</dt>
+<dd><p>An object of type <code>plpData</code> as generated using
+<code>getPlpData</code>.</p></dd>
+
+
+<dt>file</dt>
+<dd><p>The name of the folder where the data will be written. The folder should
+not yet exist.</p></dd>
+
+
+<dt>envir</dt>
+<dd><p>The environment for to evaluate variables when saving</p></dd>
 
-    <p>The data will be written to a set of files in the folder specified by the user.</p>
 
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='co'># todo</span>
+<dt>overwrite</dt>
+<dd><p>Whether to force overwrite an existing file</p></dd>
 
-</div></pre>
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>The data will be written to a set of files in the folder specified by the user.</p>
+    </div>
+
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="co"># todo</span></span></span>
+<span class="r-in"><span></span></span>
+<span class="r-in"></span>
+</code></pre></div>
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/savePlpModel.html b/docs/reference/savePlpModel.html
index 16ae57e6c..786674942 100644
--- a/docs/reference/savePlpModel.html
+++ b/docs/reference/savePlpModel.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Saves the plp model — savePlpModel • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Saves the plp model — savePlpModel" />
-<meta property="og:description" content="Saves the plp model" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Saves the plp model — savePlpModel • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Saves the plp model — savePlpModel"><meta property="og:description" content="Saves the plp model"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Saves the plp model</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SaveLoadPlp.R'><code>R/SaveLoadPlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SaveLoadPlp.R" class="external-link"><code>R/SaveLoadPlp.R</code></a></small>
     <div class="hidden name"><code>savePlpModel.Rd</code></div>
     </div>
 
@@ -183,50 +131,46 @@ <h1>Saves the plp model</h1>
     <p>Saves the plp model</p>
     </div>
 
-    <pre class="usage"><span class='fu'>savePlpModel</span><span class='op'>(</span><span class='va'>plpModel</span>, <span class='va'>dirPath</span><span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">savePlpModel</span><span class="op">(</span><span class="va">plpModel</span>, <span class="va">dirPath</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpModel</dt>
+<dd><p>A trained classifier returned by running <code>runPlp()$model</code></p></dd>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpModel</th>
-      <td><p>A trained classifier returned by running <code>runPlp()$model</code></p></td>
-    </tr>
-    <tr>
-      <th>dirPath</th>
-      <td><p>A location to save the model to</p></td>
-    </tr>
-    </table>
 
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<dt>dirPath</dt>
+<dd><p>A location to save the model to</p></dd>
 
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Saves the plp model to a user specificed folder</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/savePlpResult.html b/docs/reference/savePlpResult.html
index 55990afa7..52064bba2 100644
--- a/docs/reference/savePlpResult.html
+++ b/docs/reference/savePlpResult.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Saves the result from runPlp into the location directory — savePlpResult • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Saves the result from runPlp into the location directory — savePlpResult" />
-<meta property="og:description" content="Saves the result from runPlp into the location directory" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Saves the result from runPlp into the location directory — savePlpResult • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Saves the result from runPlp into the location directory — savePlpResult"><meta property="og:description" content="Saves the result from runPlp into the location directory"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Saves the result from runPlp into the location directory</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SaveLoadPlp.R'><code>R/SaveLoadPlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SaveLoadPlp.R" class="external-link"><code>R/SaveLoadPlp.R</code></a></small>
     <div class="hidden name"><code>savePlpResult.Rd</code></div>
     </div>
 
@@ -183,50 +131,46 @@ <h1>Saves the result from runPlp into the location directory</h1>
     <p>Saves the result from runPlp into the location directory</p>
     </div>
 
-    <pre class="usage"><span class='fu'>savePlpResult</span><span class='op'>(</span><span class='va'>result</span>, <span class='va'>dirPath</span><span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">savePlpResult</span><span class="op">(</span><span class="va">result</span>, <span class="va">dirPath</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>result</dt>
+<dd><p>The result of running runPlp()</p></dd>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>result</th>
-      <td><p>The result of running runPlp()</p></td>
-    </tr>
-    <tr>
-      <th>dirPath</th>
-      <td><p>The directory to save the csv</p></td>
-    </tr>
-    </table>
 
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<dt>dirPath</dt>
+<dd><p>The directory to save the csv</p></dd>
 
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Saves the result from runPlp into the location directory</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/savePlpShareable.html b/docs/reference/savePlpShareable.html
index 90e5e99e2..828518125 100644
--- a/docs/reference/savePlpShareable.html
+++ b/docs/reference/savePlpShareable.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Save the plp result as json files and csv files for transparent sharing — savePlpShareable • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Save the plp result as json files and csv files for transparent sharing — savePlpShareable" />
-<meta property="og:description" content="Save the plp result as json files and csv files for transparent sharing" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Save the plp result as json files and csv files for transparent sharing — savePlpShareable • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Save the plp result as json files and csv files for transparent sharing — savePlpShareable"><meta property="og:description" content="Save the plp result as json files and csv files for transparent sharing"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Save the plp result as json files and csv files for transparent sharing</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SaveLoadPlp.R'><code>R/SaveLoadPlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SaveLoadPlp.R" class="external-link"><code>R/SaveLoadPlp.R</code></a></small>
     <div class="hidden name"><code>savePlpShareable.Rd</code></div>
     </div>
 
@@ -183,54 +131,50 @@ <h1>Save the plp result as json files and csv files for transparent sharing</h1>
     <p>Save the plp result as json files and csv files for transparent sharing</p>
     </div>
 
-    <pre class="usage"><span class='fu'>savePlpShareable</span><span class='op'>(</span><span class='va'>result</span>, <span class='va'>saveDirectory</span>, minCellCount <span class='op'>=</span> <span class='fl'>10</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>result</th>
-      <td><p>An object of class runPlp with development or validation results</p></td>
-    </tr>
-    <tr>
-      <th>saveDirectory</th>
-      <td><p>The directory the save the results as csv files</p></td>
-    </tr>
-    <tr>
-      <th>minCellCount</th>
-      <td><p>Minimum cell count for the covariateSummary and certain evaluation results</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">savePlpShareable</span><span class="op">(</span><span class="va">result</span>, <span class="va">saveDirectory</span>, minCellCount <span class="op">=</span> <span class="fl">10</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>result</dt>
+<dd><p>An object of class runPlp with development or validation results</p></dd>
+
+
+<dt>saveDirectory</dt>
+<dd><p>The directory the save the results as csv files</p></dd>
 
+
+<dt>minCellCount</dt>
+<dd><p>Minimum cell count for the covariateSummary and certain evaluation results</p></dd>
+
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Saves the main results json/csv files (these files can be read by the shiny app)</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/savePlpToCsv.html b/docs/reference/savePlpToCsv.html
deleted file mode 100644
index 367a94073..000000000
--- a/docs/reference/savePlpToCsv.html
+++ /dev/null
@@ -1,226 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Save parts of the plp result as a csv for transparent sharing — savePlpToCsv • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Save parts of the plp result as a csv for transparent sharing — savePlpToCsv" />
-<meta property="og:description" content="Save parts of the plp result as a csv for transparent sharing" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Save parts of the plp result as a csv for transparent sharing</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SaveLoadPlp.R'><code>R/SaveLoadPlp.R</code></a></small>
-    <div class="hidden name"><code>savePlpToCsv.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Save parts of the plp result as a csv for transparent sharing</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>savePlpToCsv</span><span class='op'>(</span><span class='va'>result</span>, <span class='va'>dirPath</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>result</th>
-      <td><p>An object of class runPlp with development or validation results</p></td>
-    </tr>
-    <tr>
-      <th>dirPath</th>
-      <td><p>The directory the save the results as csv files</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Saves the main results as a csv (these files can be read by the shiny app)</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/savePrediction.html b/docs/reference/savePrediction.html
index c6b104576..6ff5a6ff3 100644
--- a/docs/reference/savePrediction.html
+++ b/docs/reference/savePrediction.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Saves the prediction dataframe to RDS — savePrediction • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Saves the prediction dataframe to RDS — savePrediction" />
-<meta property="og:description" content="Saves the prediction dataframe to RDS" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Saves the prediction dataframe to RDS — savePrediction • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Saves the prediction dataframe to RDS — savePrediction"><meta property="og:description" content="Saves the prediction dataframe to RDS"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Saves the prediction dataframe to RDS</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SaveLoadPlp.R'><code>R/SaveLoadPlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SaveLoadPlp.R" class="external-link"><code>R/SaveLoadPlp.R</code></a></small>
     <div class="hidden name"><code>savePrediction.Rd</code></div>
     </div>
 
@@ -183,54 +131,50 @@ <h1>Saves the prediction dataframe to RDS</h1>
     <p>Saves the prediction dataframe to RDS</p>
     </div>
 
-    <pre class="usage"><span class='fu'>savePrediction</span><span class='op'>(</span><span class='va'>prediction</span>, <span class='va'>dirPath</span>, fileName <span class='op'>=</span> <span class='st'>"prediction.rds"</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>prediction</th>
-      <td><p>The prediciton data.frame</p></td>
-    </tr>
-    <tr>
-      <th>dirPath</th>
-      <td><p>The directory to save the prediction RDS</p></td>
-    </tr>
-    <tr>
-      <th>fileName</th>
-      <td><p>The name of the RDS file that will be saved in dirPath</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">savePrediction</span><span class="op">(</span><span class="va">prediction</span>, <span class="va">dirPath</span>, fileName <span class="op">=</span> <span class="st">"prediction.rds"</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>prediction</dt>
+<dd><p>The prediciton data.frame</p></dd>
+
+
+<dt>dirPath</dt>
+<dd><p>The directory to save the prediction RDS</p></dd>
 
+
+<dt>fileName</dt>
+<dd><p>The name of the RDS file that will be saved in dirPath</p></dd>
+
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Saves the prediction data frame returned by predict.R to an RDS file and returns the fileLocation where the prediction is saved</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/savePredictionAnalysisList.html b/docs/reference/savePredictionAnalysisList.html
deleted file mode 100644
index 05ec5e435..000000000
--- a/docs/reference/savePredictionAnalysisList.html
+++ /dev/null
@@ -1,295 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Saves a json prediction settings given R settings — savePredictionAnalysisList • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Saves a json prediction settings given R settings — savePredictionAnalysisList" />
-<meta property="og:description" content="Saves a json prediction settings given R settings" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Saves a json prediction settings given R settings</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/RunMultiplePlp.R'><code>R/RunMultiplePlp.R</code></a></small>
-    <div class="hidden name"><code>savePredictionAnalysisList.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Saves a json prediction settings given R settings</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>savePredictionAnalysisList</span><span class='op'>(</span>
-  workFolder <span class='op'>=</span> <span class='st'>"inst/settings"</span>,
-  <span class='va'>cohortIds</span>,
-  <span class='va'>outcomeIds</span>,
-  cohortSettingCsv <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/file.path.html'>file.path</a></span><span class='op'>(</span><span class='va'>workFolder</span>, <span class='st'>"CohortsToCreate.csv"</span><span class='op'>)</span>,
-  <span class='va'>covariateSettingList</span>,
-  <span class='va'>populationSettingList</span>,
-  <span class='va'>modelSettingList</span>,
-  maxSampleSize <span class='op'>=</span> <span class='cn'>NULL</span>,
-  washoutPeriod <span class='op'>=</span> <span class='fl'>0</span>,
-  minCovariateFraction <span class='op'>=</span> <span class='fl'>0</span>,
-  normalizeData <span class='op'>=</span> <span class='cn'>T</span>,
-  testSplit <span class='op'>=</span> <span class='st'>"person"</span>,
-  testFraction <span class='op'>=</span> <span class='fl'>0.25</span>,
-  splitSeed <span class='op'>=</span> <span class='fl'>1</span>,
-  nfold <span class='op'>=</span> <span class='fl'>3</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>workFolder</th>
-      <td><p>Location to save json specification</p></td>
-    </tr>
-    <tr>
-      <th>cohortIds</th>
-      <td><p>Vector of target population cohort ids</p></td>
-    </tr>
-    <tr>
-      <th>outcomeIds</th>
-      <td><p>Vector of outcome cohort ids</p></td>
-    </tr>
-    <tr>
-      <th>cohortSettingCsv</th>
-      <td><p>The location to the csv containing the cohort details</p></td>
-    </tr>
-    <tr>
-      <th>covariateSettingList</th>
-      <td><p>A list of covariate settings</p></td>
-    </tr>
-    <tr>
-      <th>populationSettingList</th>
-      <td><p>A list of population settings</p></td>
-    </tr>
-    <tr>
-      <th>modelSettingList</th>
-      <td><p>A list of model settings</p></td>
-    </tr>
-    <tr>
-      <th>maxSampleSize</th>
-      <td><p>If not NULL then max number of target population to sample for model training</p></td>
-    </tr>
-    <tr>
-      <th>washoutPeriod</th>
-      <td><p>Minimum prior observation for each person in target pop to be included</p></td>
-    </tr>
-    <tr>
-      <th>minCovariateFraction</th>
-      <td><p>Minimum covariate fraction to include</p></td>
-    </tr>
-    <tr>
-      <th>normalizeData</th>
-      <td><p>Whether to normalise data</p></td>
-    </tr>
-    <tr>
-      <th>testSplit</th>
-      <td><p>Split by person or time</p></td>
-    </tr>
-    <tr>
-      <th>testFraction</th>
-      <td><p>Fractiuon of data to use for test set</p></td>
-    </tr>
-    <tr>
-      <th>splitSeed</th>
-      <td><p>Seed used in test split</p></td>
-    </tr>
-    <tr>
-      <th>nfold</th>
-      <td><p>Number of folds used when training model</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>This function interprets a json with the multiple prediction settings and creates a list 
-that can be combined with connection settings to run a multiple prediction study</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/sensitivity.html b/docs/reference/sensitivity.html
index 8d96e3d1d..512fa41d5 100644
--- a/docs/reference/sensitivity.html
+++ b/docs/reference/sensitivity.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculate the sensitivity — sensitivity • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculate the sensitivity — sensitivity" />
-<meta property="og:description" content="Calculate the sensitivity" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculate the sensitivity — sensitivity • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculate the sensitivity — sensitivity"><meta property="og:description" content="Calculate the sensitivity"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculate the sensitivity</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ThresholdSummary.R'><code>R/ThresholdSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ThresholdSummary.R" class="external-link"><code>R/ThresholdSummary.R</code></a></small>
     <div class="hidden name"><code>sensitivity.Rd</code></div>
     </div>
 
@@ -183,61 +131,60 @@ <h1>Calculate the sensitivity</h1>
     <p>Calculate the sensitivity</p>
     </div>
 
-    <pre class="usage"><span class='fu'>sensitivity</span><span class='op'>(</span><span class='va'>TP</span>, <span class='va'>TN</span>, <span class='va'>FN</span>, <span class='va'>FP</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>TP</th>
-      <td><p>Number of true positives</p></td>
-    </tr>
-    <tr>
-      <th>TN</th>
-      <td><p>Number of true negatives</p></td>
-    </tr>
-    <tr>
-      <th>FN</th>
-      <td><p>Number of false negatives</p></td>
-    </tr>
-    <tr>
-      <th>FP</th>
-      <td><p>Number of false positives</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>sensitivity value</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">sensitivity</span><span class="op">(</span><span class="va">TP</span>, <span class="va">TN</span>, <span class="va">FN</span>, <span class="va">FP</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>TP</dt>
+<dd><p>Number of true positives</p></dd>
+
+
+<dt>TN</dt>
+<dd><p>Number of true negatives</p></dd>
+
+
+<dt>FN</dt>
+<dd><p>Number of false negatives</p></dd>
+
+
+<dt>FP</dt>
+<dd><p>Number of false positives</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>sensitivity value</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculate the sensitivity</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/setAdaBoost.html b/docs/reference/setAdaBoost.html
index ddc12d5d6..2ad4c4aea 100644
--- a/docs/reference/setAdaBoost.html
+++ b/docs/reference/setAdaBoost.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for AdaBoost with python DecisionTreeClassifier base estimator — setAdaBoost • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for AdaBoost with python DecisionTreeClassifier base estimator — setAdaBoost" />
-<meta property="og:description" content="Create setting for AdaBoost with python DecisionTreeClassifier base estimator" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create setting for AdaBoost with python DecisionTreeClassifier base estimator — setAdaBoost • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create setting for AdaBoost with python DecisionTreeClassifier base estimator — setAdaBoost"><meta property="og:description" content="Create setting for AdaBoost with python DecisionTreeClassifier base estimator"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create setting for AdaBoost with python DecisionTreeClassifier base estimator</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SklearnClassifierSettings.R'><code>R/SklearnClassifierSettings.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SklearnClassifierSettings.R" class="external-link"><code>R/SklearnClassifierSettings.R</code></a></small>
     <div class="hidden name"><code>setAdaBoost.Rd</code></div>
     </div>
 
@@ -183,68 +131,65 @@ <h1>Create setting for AdaBoost with python DecisionTreeClassifier base estimato
     <p>Create setting for AdaBoost with python DecisionTreeClassifier base estimator</p>
     </div>
 
-    <pre class="usage"><span class='fu'>setAdaBoost</span><span class='op'>(</span>
-  nEstimators <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>10</span>, <span class='fl'>50</span>, <span class='fl'>200</span><span class='op'>)</span>,
-  learningRate <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>1</span>, <span class='fl'>0.5</span>, <span class='fl'>0.1</span><span class='op'>)</span>,
-  algorithm <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='st'>"SAMME.R"</span><span class='op'>)</span>,
-  seed <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/sample.html'>sample</a></span><span class='op'>(</span><span class='fl'>1e+06</span>, <span class='fl'>1</span><span class='op'>)</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>nEstimators</th>
-      <td><p>(list) The maximum number of estimators at which boosting is terminated. In case of perfect fit, the learning procedure is stopped early.</p></td>
-    </tr>
-    <tr>
-      <th>learningRate</th>
-      <td><p>(list) Weight applied to each classifier at each boosting iteration. A higher learning rate increases the contribution of each classifier. There is a trade-off between the learningRate and nEstimators parameters
-There is a trade-off between learningRate and nEstimators.</p></td>
-    </tr>
-    <tr>
-      <th>algorithm</th>
-      <td><p>(list) If ‘SAMME.R’ then use the SAMME.R real boosting algorithm. base_estimator must support calculation of class probabilities. If ‘SAMME’ then use the SAMME discrete boosting algorithm. The SAMME.R algorithm typically converges faster than SAMME, achieving a lower test error with fewer boosting iterations.</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>A seed for the model</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>model.adaBoost</span> <span class='op'>&lt;-</span> <span class='fu'>setAdaBoost</span><span class='op'>(</span>nEstimators <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>10</span>,<span class='fl'>50</span>,<span class='fl'>200</span><span class='op'>)</span>, learningRate <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>1</span>, <span class='fl'>0.5</span>, <span class='fl'>0.1</span><span class='op'>)</span>, 
-                              algorithm <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='st'>'SAMME.R'</span><span class='op'>)</span>, seed <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/sample.html'>sample</a></span><span class='op'>(</span><span class='fl'>1000000</span>,<span class='fl'>1</span><span class='op'>)</span>
-                              <span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">setAdaBoost</span><span class="op">(</span></span>
+<span>  nEstimators <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">10</span>, <span class="fl">50</span>, <span class="fl">200</span><span class="op">)</span>,</span>
+<span>  learningRate <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">1</span>, <span class="fl">0.5</span>, <span class="fl">0.1</span><span class="op">)</span>,</span>
+<span>  algorithm <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="st">"SAMME.R"</span><span class="op">)</span>,</span>
+<span>  seed <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/sample.html" class="external-link">sample</a></span><span class="op">(</span><span class="fl">1e+06</span>, <span class="fl">1</span><span class="op">)</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>nEstimators</dt>
+<dd><p>(list) The maximum number of estimators at which boosting is terminated. In case of perfect fit, the learning procedure is stopped early.</p></dd>
+
+
+<dt>learningRate</dt>
+<dd><p>(list) Weight applied to each classifier at each boosting iteration. A higher learning rate increases the contribution of each classifier. There is a trade-off between the learningRate and nEstimators parameters
+There is a trade-off between learningRate and nEstimators.</p></dd>
+
+
+<dt>algorithm</dt>
+<dd><p>(list) If ‘SAMME.R’ then use the SAMME.R real boosting algorithm. base_estimator must support calculation of class probabilities. If ‘SAMME’ then use the SAMME discrete boosting algorithm. The SAMME.R algorithm typically converges faster than SAMME, achieving a lower test error with fewer boosting iterations.</p></dd>
+
+
+<dt>seed</dt>
+<dd><p>A seed for the model</p></dd>
+
+</dl></div>
+
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="kw">if</span> <span class="op">(</span><span class="cn">FALSE</span><span class="op">)</span> <span class="op">{</span></span></span>
+<span class="r-in"><span><span class="va">model.adaBoost</span> <span class="op">&lt;-</span> <span class="fu">setAdaBoost</span><span class="op">(</span>nEstimators <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">10</span>,<span class="fl">50</span>,<span class="fl">200</span><span class="op">)</span>, learningRate <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">1</span>, <span class="fl">0.5</span>, <span class="fl">0.1</span><span class="op">)</span>,</span></span>
+<span class="r-in"><span>                              algorithm <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="st">'SAMME.R'</span><span class="op">)</span>, seed <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/sample.html" class="external-link">sample</a></span><span class="op">(</span><span class="fl">1000000</span>,<span class="fl">1</span><span class="op">)</span></span></span>
+<span class="r-in"><span>                              <span class="op">)</span></span></span>
+<span class="r-in"><span><span class="op">}</span></span></span>
+</code></pre></div>
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/setCIReNN.html b/docs/reference/setCIReNN.html
deleted file mode 100644
index f62ecff7e..000000000
--- a/docs/reference/setCIReNN.html
+++ /dev/null
@@ -1,346 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for CIReNN model — setCIReNN • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for CIReNN model — setCIReNN" />
-<meta property="og:description" content="Create setting for CIReNN model" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.3.10</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Create setting for CIReNN model</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/CIReNN.R'><code>R/CIReNN.R</code></a></small>
-    <div class="hidden name"><code>setCIReNN.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Create setting for CIReNN model</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>setCIReNN</span><span class='op'>(</span>
-  numberOfRNNLayer <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>1</span><span class='op'>)</span>,
-  units <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>128</span>, <span class='fl'>64</span><span class='op'>)</span>,
-  recurrentDropout <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>0.2</span><span class='op'>)</span>,
-  layerDropout <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>0.2</span><span class='op'>)</span>,
-  lr <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>1e-04</span><span class='op'>)</span>,
-  decay <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>1e-05</span><span class='op'>)</span>,
-  outcomeWeight <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>0</span><span class='op'>)</span>,
-  batchSize <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>100</span><span class='op'>)</span>,
-  epochs <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>100</span><span class='op'>)</span>,
-  earlyStoppingMinDelta <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>1e-04</span><span class='op'>)</span>,
-  earlyStoppingPatience <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>10</span><span class='op'>)</span>,
-  bayes <span class='op'>=</span> <span class='cn'>T</span>,
-  useDeepEnsemble <span class='op'>=</span> <span class='cn'>F</span>,
-  numberOfEnsembleNetwork <span class='op'>=</span> <span class='fl'>5</span>,
-  useVae <span class='op'>=</span> <span class='cn'>T</span>,
-  vaeDataSamplingProportion <span class='op'>=</span> <span class='fl'>0.1</span>,
-  vaeValidationSplit <span class='op'>=</span> <span class='fl'>0.2</span>,
-  vaeBatchSize <span class='op'>=</span> <span class='fl'>100L</span>,
-  vaeLatentDim <span class='op'>=</span> <span class='fl'>10L</span>,
-  vaeIntermediateDim <span class='op'>=</span> <span class='fl'>256L</span>,
-  vaeEpoch <span class='op'>=</span> <span class='fl'>100L</span>,
-  vaeEpislonStd <span class='op'>=</span> <span class='fl'>1</span>,
-  useGPU <span class='op'>=</span> <span class='cn'>FALSE</span>,
-  maxGPUs <span class='op'>=</span> <span class='fl'>2</span>,
-  seed <span class='op'>=</span> <span class='fl'>1234</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>numberOfRNNLayer</th>
-      <td><p>The number of RNN layer, only 1, 2, or 3 layers available now. eg. 1, c(1,2), c(1,2,3)</p></td>
-    </tr>
-    <tr>
-      <th>units</th>
-      <td><p>The number of units of RNN layer - as a list of vectors</p></td>
-    </tr>
-    <tr>
-      <th>recurrentDropout</th>
-      <td><p>The reccurrent dropout rate (regularisation)</p></td>
-    </tr>
-    <tr>
-      <th>layerDropout</th>
-      <td><p>The layer dropout rate (regularisation)</p></td>
-    </tr>
-    <tr>
-      <th>lr</th>
-      <td><p>Learning rate</p></td>
-    </tr>
-    <tr>
-      <th>decay</th>
-      <td><p>Learning rate decay over each update.</p></td>
-    </tr>
-    <tr>
-      <th>outcomeWeight</th>
-      <td><p>The weight of the outcome class in the loss function. Default is 0, which will be replaced by balanced weight.</p></td>
-    </tr>
-    <tr>
-      <th>batchSize</th>
-      <td><p>The number of data points to use per training batch</p></td>
-    </tr>
-    <tr>
-      <th>epochs</th>
-      <td><p>Number of times to iterate over dataset</p></td>
-    </tr>
-    <tr>
-      <th>earlyStoppingMinDelta</th>
-      <td><p>minimum change in the monitored quantity to qualify as an improvement for early stopping, i.e. an absolute change of less than min_delta in loss of validation data, will count as no improvement.</p></td>
-    </tr>
-    <tr>
-      <th>earlyStoppingPatience</th>
-      <td><p>Number of epochs with no improvement after which training will be stopped.</p></td>
-    </tr>
-    <tr>
-      <th>bayes</th>
-      <td><p>logical (either TRUE or FALSE) value for using Bayesian Drop Out Layer to measure uncertainty. If it is TRUE, both Epistemic and Aleatoric uncertainty will be measured through Bayesian Drop Out layer</p></td>
-    </tr>
-    <tr>
-      <th>useDeepEnsemble</th>
-      <td><p>logical (either TRUE or FALSE) value for using Deep Ensemble (Lakshminarayanan et al., 2017) to measure uncertainty. It cannot be used together with Bayesian deep learing.</p></td>
-    </tr>
-    <tr>
-      <th>numberOfEnsembleNetwork</th>
-      <td><p>Integer. Number of network used for Deep Ensemble (Lakshminarayanan et al recommended 5).</p></td>
-    </tr>
-    <tr>
-      <th>useVae</th>
-      <td><p>logical (either TRUE or FALSE) value for using Variational AutoEncoder before RNN</p></td>
-    </tr>
-    <tr>
-      <th>vaeDataSamplingProportion</th>
-      <td><p>Data sampling proportion for VAE</p></td>
-    </tr>
-    <tr>
-      <th>vaeValidationSplit</th>
-      <td><p>Validation split proportion for VAE</p></td>
-    </tr>
-    <tr>
-      <th>vaeBatchSize</th>
-      <td><p>batch size for VAE</p></td>
-    </tr>
-    <tr>
-      <th>vaeLatentDim</th>
-      <td><p>Number of latent dimesion for VAE</p></td>
-    </tr>
-    <tr>
-      <th>vaeIntermediateDim</th>
-      <td><p>Number of intermediate dimesion for VAE</p></td>
-    </tr>
-    <tr>
-      <th>vaeEpoch</th>
-      <td><p>Number of times to interate over dataset for VAE</p></td>
-    </tr>
-    <tr>
-      <th>vaeEpislonStd</th>
-      <td><p>Epsilon</p></td>
-    </tr>
-    <tr>
-      <th>useGPU</th>
-      <td><p>logical (either TRUE or FALSE) value. If you have GPUs in your machine, and want to use multiple GPU for deep learning, set this value as TRUE</p></td>
-    </tr>
-    <tr>
-      <th>maxGPUs</th>
-      <td><p>Integer, If you will use GPU, how many GPUs will be used for deep learning in VAE? GPU parallelisation for deep learning will be activated only when parallel vae is true. Integer &gt;= 2 or list of integers, number of GPUs or list of GPU IDs on which to create model replicas.</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>Random seed used by deep learning model</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>model.CIReNN</span> <span class='op'>&lt;-</span> <span class='fu'>setCIReNN</span><span class='op'>(</span><span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/setCNNTorch.html b/docs/reference/setCNNTorch.html
deleted file mode 100644
index 798867100..000000000
--- a/docs/reference/setCNNTorch.html
+++ /dev/null
@@ -1,249 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for CNN model with python — setCNNTorch • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for CNN model with python — setCNNTorch" />
-<meta property="og:description" content="Create setting for CNN model with python" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.3.10</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Create setting for CNN model with python</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/CNNTorch.R'><code>R/CNNTorch.R</code></a></small>
-    <div class="hidden name"><code>setCNNTorch.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Create setting for CNN model with python</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>setCNNTorch</span><span class='op'>(</span>
-  nbfilters <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>16</span>, <span class='fl'>32</span><span class='op'>)</span>,
-  epochs <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>20</span>, <span class='fl'>50</span><span class='op'>)</span>,
-  seed <span class='op'>=</span> <span class='fl'>0</span>,
-  class_weight <span class='op'>=</span> <span class='fl'>0</span>,
-  type <span class='op'>=</span> <span class='st'>"CNN"</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>nbfilters</th>
-      <td><p>The number of filters</p></td>
-    </tr>
-    <tr>
-      <th>epochs</th>
-      <td><p>The number of epochs</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>A seed for the model</p></td>
-    </tr>
-    <tr>
-      <th>class_weight</th>
-      <td><p>The class weight used for imbalanced data: 
- 0: Inverse ratio between positives and negatives
--1: Focal loss</p></td>
-    </tr>
-    <tr>
-      <th>type</th>
-      <td><p>It can be normal 'CNN', 'CNN_LSTM', CNN_MLF' with multiple kernels with different kernel size, 
-'CNN_MIX', 'ResNet' and 'CNN_MULTI'</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>model.cnnTorch</span> <span class='op'>&lt;-</span> <span class='fu'>setCNNTorch</span><span class='op'>(</span><span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/setCompetingRiskModel.html b/docs/reference/setCompetingRiskModel.html
deleted file mode 100644
index 0c452ef69..000000000
--- a/docs/reference/setCompetingRiskModel.html
+++ /dev/null
@@ -1,222 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for competing risk model (uses Fine-Gray model in Cyclops) — setCompetingRiskModel • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for competing risk model (uses Fine-Gray model in Cyclops) — setCompetingRiskModel" />
-<meta property="og:description" content="Create setting for competing risk model (uses Fine-Gray model in Cyclops)" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Create setting for competing risk model (uses Fine-Gray model in Cyclops)</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/CompetingRiskModel.R'><code>R/CompetingRiskModel.R</code></a></small>
-    <div class="hidden name"><code>setCompetingRiskModel.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Create setting for competing risk model (uses Fine-Gray model in Cyclops)</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>setCompetingRiskModel</span><span class='op'>(</span>seed <span class='op'>=</span> <span class='cn'>NULL</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>seed</th>
-      <td><p>An option to add a seed when training the model</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='va'>model.lr</span> <span class='op'>&lt;-</span> <span class='fu'>setCompetingRiskModel</span><span class='op'>(</span><span class='op'>)</span>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/setCovNN.html b/docs/reference/setCovNN.html
deleted file mode 100644
index cc442a23a..000000000
--- a/docs/reference/setCovNN.html
+++ /dev/null
@@ -1,271 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for multi-resolution CovNN model (stucture based on https://arxiv.org/pdf/1608.00647.pdf CNN1) — setCovNN • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for multi-resolution CovNN model (stucture based on https://arxiv.org/pdf/1608.00647.pdf CNN1) — setCovNN" />
-<meta property="og:description" content="Create setting for multi-resolution CovNN model (stucture based on https://arxiv.org/pdf/1608.00647.pdf CNN1)" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.3.10</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Create setting for multi-resolution CovNN model (stucture based on https://arxiv.org/pdf/1608.00647.pdf CNN1)</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/CovNN.R'><code>R/CovNN.R</code></a></small>
-    <div class="hidden name"><code>setCovNN.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Create setting for multi-resolution CovNN model (stucture based on https://arxiv.org/pdf/1608.00647.pdf CNN1)</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>setCovNN</span><span class='op'>(</span>
-  batchSize <span class='op'>=</span> <span class='fl'>1000</span>,
-  outcomeWeight <span class='op'>=</span> <span class='fl'>1</span>,
-  lr <span class='op'>=</span> <span class='fl'>1e-05</span>,
-  decay <span class='op'>=</span> <span class='fl'>1e-06</span>,
-  dropout <span class='op'>=</span> <span class='fl'>0</span>,
-  epochs <span class='op'>=</span> <span class='fl'>10</span>,
-  filters <span class='op'>=</span> <span class='fl'>3</span>,
-  kernelSize <span class='op'>=</span> <span class='fl'>10</span>,
-  loss <span class='op'>=</span> <span class='st'>"binary_crossentropy"</span>,
-  seed <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>batchSize</th>
-      <td><p>The number of samples to used in each batch during model training</p></td>
-    </tr>
-    <tr>
-      <th>outcomeWeight</th>
-      <td><p>The weight assined to the outcome (make greater than 1 to reduce unballanced label issue)</p></td>
-    </tr>
-    <tr>
-      <th>lr</th>
-      <td><p>The learning rate</p></td>
-    </tr>
-    <tr>
-      <th>decay</th>
-      <td><p>The decay of the learning rate</p></td>
-    </tr>
-    <tr>
-      <th>dropout</th>
-      <td><p>[currently not used] the dropout rate for regularisation</p></td>
-    </tr>
-    <tr>
-      <th>epochs</th>
-      <td><p>The number of times data is used to train the model (e.g., epoches=1 means data only used once to train)</p></td>
-    </tr>
-    <tr>
-      <th>filters</th>
-      <td><p>The number of columns output by each convolution</p></td>
-    </tr>
-    <tr>
-      <th>kernelSize</th>
-      <td><p>The number of time dimensions used for each convolution</p></td>
-    </tr>
-    <tr>
-      <th>loss</th>
-      <td><p>The loss function implemented</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>The random seed</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>model.CovNN</span> <span class='op'>&lt;-</span> <span class='fu'>setCovNN</span><span class='op'>(</span><span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/setCovNN2.html b/docs/reference/setCovNN2.html
deleted file mode 100644
index bd8db7d28..000000000
--- a/docs/reference/setCovNN2.html
+++ /dev/null
@@ -1,271 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for CovNN2 model - convolution across input and time - https://arxiv.org/pdf/1608.00647.pdf — setCovNN2 • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for CovNN2 model - convolution across input and time - https://arxiv.org/pdf/1608.00647.pdf — setCovNN2" />
-<meta property="og:description" content="Create setting for CovNN2 model - convolution across input and time - https://arxiv.org/pdf/1608.00647.pdf" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.3.10</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Create setting for CovNN2 model - convolution across input and time - https://arxiv.org/pdf/1608.00647.pdf</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/CovNN2.R'><code>R/CovNN2.R</code></a></small>
-    <div class="hidden name"><code>setCovNN2.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Create setting for CovNN2 model - convolution across input and time - https://arxiv.org/pdf/1608.00647.pdf</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>setCovNN2</span><span class='op'>(</span>
-  batchSize <span class='op'>=</span> <span class='fl'>1000</span>,
-  outcomeWeight <span class='op'>=</span> <span class='fl'>1</span>,
-  lr <span class='op'>=</span> <span class='fl'>1e-05</span>,
-  decay <span class='op'>=</span> <span class='fl'>1e-06</span>,
-  dropout <span class='op'>=</span> <span class='fl'>0</span>,
-  epochs <span class='op'>=</span> <span class='fl'>10</span>,
-  filters <span class='op'>=</span> <span class='fl'>3</span>,
-  kernelSize <span class='op'>=</span> <span class='fl'>10</span>,
-  loss <span class='op'>=</span> <span class='st'>"binary_crossentropy"</span>,
-  seed <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>batchSize</th>
-      <td><p>The number of samples to used in each batch during model training</p></td>
-    </tr>
-    <tr>
-      <th>outcomeWeight</th>
-      <td><p>The weight assined to the outcome (make greater than 1 to reduce unballanced label issue)</p></td>
-    </tr>
-    <tr>
-      <th>lr</th>
-      <td><p>The learning rate</p></td>
-    </tr>
-    <tr>
-      <th>decay</th>
-      <td><p>The decay of the learning rate</p></td>
-    </tr>
-    <tr>
-      <th>dropout</th>
-      <td><p>[currently not used] the dropout rate for regularisation</p></td>
-    </tr>
-    <tr>
-      <th>epochs</th>
-      <td><p>The number of times data is used to train the model (e.g., epoches=1 means data only used once to train)</p></td>
-    </tr>
-    <tr>
-      <th>filters</th>
-      <td><p>The number of columns output by each convolution</p></td>
-    </tr>
-    <tr>
-      <th>kernelSize</th>
-      <td><p>The number of time dimensions used for each convolution</p></td>
-    </tr>
-    <tr>
-      <th>loss</th>
-      <td><p>The loss function implemented</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>The random seed</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>model.CovNN</span> <span class='op'>&lt;-</span> <span class='fu'><a href='setCovNN.html'>setCovNN</a></span><span class='op'>(</span><span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/setCoxModel.html b/docs/reference/setCoxModel.html
index 868631ba2..072cb424c 100644
--- a/docs/reference/setCoxModel.html
+++ b/docs/reference/setCoxModel.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for lasso Cox model — setCoxModel • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for lasso Cox model — setCoxModel" />
-<meta property="og:description" content="Create setting for lasso Cox model" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create setting for lasso Cox model — setCoxModel • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create setting for lasso Cox model — setCoxModel"><meta property="og:description" content="Create setting for lasso Cox model"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create setting for lasso Cox model</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/CyclopsSettings.R'><code>R/CyclopsSettings.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/CyclopsSettings.R" class="external-link"><code>R/CyclopsSettings.R</code></a></small>
     <div class="hidden name"><code>setCoxModel.Rd</code></div>
     </div>
 
@@ -183,88 +131,85 @@ <h1>Create setting for lasso Cox model</h1>
     <p>Create setting for lasso Cox model</p>
     </div>
 
-    <pre class="usage"><span class='fu'>setCoxModel</span><span class='op'>(</span>
-  variance <span class='op'>=</span> <span class='fl'>0.01</span>,
-  seed <span class='op'>=</span> <span class='cn'>NULL</span>,
-  includeCovariateIds <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='op'>)</span>,
-  noShrinkage <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='op'>)</span>,
-  threads <span class='op'>=</span> <span class='op'>-</span><span class='fl'>1</span>,
-  upperLimit <span class='op'>=</span> <span class='fl'>20</span>,
-  lowerLimit <span class='op'>=</span> <span class='fl'>0.01</span>,
-  tolerance <span class='op'>=</span> <span class='fl'>2e-07</span>,
-  maxIterations <span class='op'>=</span> <span class='fl'>3000</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>variance</th>
-      <td><p>Numeric: prior distribution starting variance</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>An option to add a seed when training the model</p></td>
-    </tr>
-    <tr>
-      <th>includeCovariateIds</th>
-      <td><p>a set of covariate IDS to limit the analysis to</p></td>
-    </tr>
-    <tr>
-      <th>noShrinkage</th>
-      <td><p>a set of covariates whcih are to be forced to be included in the final model. default is the intercept</p></td>
-    </tr>
-    <tr>
-      <th>threads</th>
-      <td><p>An option to set number of threads when training model</p></td>
-    </tr>
-    <tr>
-      <th>upperLimit</th>
-      <td><p>Numeric: Upper prior variance limit for grid-search</p></td>
-    </tr>
-    <tr>
-      <th>lowerLimit</th>
-      <td><p>Numeric: Lower prior variance limit for grid-search</p></td>
-    </tr>
-    <tr>
-      <th>tolerance</th>
-      <td><p>Numeric: maximum relative change in convergence criterion from successive iterations to achieve convergence</p></td>
-    </tr>
-    <tr>
-      <th>maxIterations</th>
-      <td><p>Integer: maximum iterations of Cyclops to attempt before returning a failed-to-converge error</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='va'>model.lr</span> <span class='op'>&lt;-</span> <span class='fu'>setCoxModel</span><span class='op'>(</span><span class='op'>)</span>
-</div></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">setCoxModel</span><span class="op">(</span></span>
+<span>  variance <span class="op">=</span> <span class="fl">0.01</span>,</span>
+<span>  seed <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  includeCovariateIds <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  noShrinkage <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  threads <span class="op">=</span> <span class="op">-</span><span class="fl">1</span>,</span>
+<span>  upperLimit <span class="op">=</span> <span class="fl">20</span>,</span>
+<span>  lowerLimit <span class="op">=</span> <span class="fl">0.01</span>,</span>
+<span>  tolerance <span class="op">=</span> <span class="fl">2e-07</span>,</span>
+<span>  maxIterations <span class="op">=</span> <span class="fl">3000</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>variance</dt>
+<dd><p>Numeric: prior distribution starting variance</p></dd>
+
+
+<dt>seed</dt>
+<dd><p>An option to add a seed when training the model</p></dd>
+
+
+<dt>includeCovariateIds</dt>
+<dd><p>a set of covariate IDS to limit the analysis to</p></dd>
+
+
+<dt>noShrinkage</dt>
+<dd><p>a set of covariates whcih are to be forced to be included in the final model. default is the intercept</p></dd>
+
+
+<dt>threads</dt>
+<dd><p>An option to set number of threads when training model</p></dd>
+
+
+<dt>upperLimit</dt>
+<dd><p>Numeric: Upper prior variance limit for grid-search</p></dd>
+
+
+<dt>lowerLimit</dt>
+<dd><p>Numeric: Lower prior variance limit for grid-search</p></dd>
+
+
+<dt>tolerance</dt>
+<dd><p>Numeric: maximum relative change in convergence criterion from successive iterations to achieve convergence</p></dd>
+
+
+<dt>maxIterations</dt>
+<dd><p>Integer: maximum iterations of Cyclops to attempt before returning a failed-to-converge error</p></dd>
+
+</dl></div>
+
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="va">model.lr</span> <span class="op">&lt;-</span> <span class="fu">setCoxModel</span><span class="op">(</span><span class="op">)</span></span></span>
+</code></pre></div>
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/setDecisionTree.html b/docs/reference/setDecisionTree.html
index fd8302977..1a6f0906f 100644
--- a/docs/reference/setDecisionTree.html
+++ b/docs/reference/setDecisionTree.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for the scikit-learn 1.0.1 DecisionTree with python — setDecisionTree • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for the scikit-learn 1.0.1 DecisionTree with python — setDecisionTree" />
-<meta property="og:description" content="Create setting for the scikit-learn 1.0.1 DecisionTree with python" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create setting for the scikit-learn 1.0.1 DecisionTree with python — setDecisionTree • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create setting for the scikit-learn 1.0.1 DecisionTree with python — setDecisionTree"><meta property="og:description" content="Create setting for the scikit-learn 1.0.1 DecisionTree with python"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create setting for the scikit-learn 1.0.1 DecisionTree with python</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SklearnClassifierSettings.R'><code>R/SklearnClassifierSettings.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SklearnClassifierSettings.R" class="external-link"><code>R/SklearnClassifierSettings.R</code></a></small>
     <div class="hidden name"><code>setDecisionTree.Rd</code></div>
     </div>
 
@@ -183,100 +131,97 @@ <h1>Create setting for the scikit-learn 1.0.1 DecisionTree with python</h1>
     <p>Create setting for the scikit-learn 1.0.1 DecisionTree with python</p>
     </div>
 
-    <pre class="usage"><span class='fu'>setDecisionTree</span><span class='op'>(</span>
-  criterion <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='st'>"gini"</span><span class='op'>)</span>,
-  splitter <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='st'>"best"</span><span class='op'>)</span>,
-  maxDepth <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fu'><a href='https://rdrr.io/r/base/integer.html'>as.integer</a></span><span class='op'>(</span><span class='fl'>4</span><span class='op'>)</span>, <span class='fu'><a href='https://rdrr.io/r/base/integer.html'>as.integer</a></span><span class='op'>(</span><span class='fl'>10</span><span class='op'>)</span>, <span class='cn'>NULL</span><span class='op'>)</span>,
-  minSamplesSplit <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>2</span>, <span class='fl'>10</span><span class='op'>)</span>,
-  minSamplesLeaf <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>10</span>, <span class='fl'>50</span><span class='op'>)</span>,
-  minWeightFractionLeaf <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>0</span><span class='op'>)</span>,
-  maxFeatures <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>100</span>, <span class='st'>"auto"</span>, <span class='cn'>NULL</span><span class='op'>)</span>,
-  maxLeafNodes <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='cn'>NULL</span><span class='op'>)</span>,
-  minImpurityDecrease <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>10</span><span class='op'>^</span><span class='op'>-</span><span class='fl'>7</span><span class='op'>)</span>,
-  classWeight <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='cn'>NULL</span>, <span class='st'>"balanced"</span><span class='op'>)</span>,
-  seed <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/sample.html'>sample</a></span><span class='op'>(</span><span class='fl'>1e+06</span>, <span class='fl'>1</span><span class='op'>)</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>criterion</th>
-      <td><p>The function to measure the quality of a split. Supported criteria are “gini” for the Gini impurity and “entropy” for the information gain.</p></td>
-    </tr>
-    <tr>
-      <th>splitter</th>
-      <td><p>The strategy used to choose the split at each node. Supported strategies are “best” to choose the best split and “random” to choose the best random split.</p></td>
-    </tr>
-    <tr>
-      <th>maxDepth</th>
-      <td><p>(list) The maximum depth of the tree. If NULL, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples.</p></td>
-    </tr>
-    <tr>
-      <th>minSamplesSplit</th>
-      <td><p>The minimum number of samples required to split an internal node</p></td>
-    </tr>
-    <tr>
-      <th>minSamplesLeaf</th>
-      <td><p>The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least minSamplesLeaf training samples in each of the left and right branches. This may have the effect of smoothing the model, especially in regression.</p></td>
-    </tr>
-    <tr>
-      <th>minWeightFractionLeaf</th>
-      <td><p>The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sampleWeight is not provided.</p></td>
-    </tr>
-    <tr>
-      <th>maxFeatures</th>
-      <td><p>(list) The number of features to consider when looking for the best split (int/'auto'/NULL)</p></td>
-    </tr>
-    <tr>
-      <th>maxLeafNodes</th>
-      <td><p>(list) Grow a tree with max_leaf_nodes in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes. (int/NULL)</p></td>
-    </tr>
-    <tr>
-      <th>minImpurityDecrease</th>
-      <td><p>Threshold for early stopping in tree growth. A node will split if its impurity is above the threshold, otherwise it is a leaf.</p></td>
-    </tr>
-    <tr>
-      <th>classWeight</th>
-      <td><p>(list) Weights associated with classes 'balance' or NULL</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>The random state seed</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>model.decisionTree</span> <span class='op'>&lt;-</span> <span class='fu'>setDecisionTree</span><span class='op'>(</span>maxDepth<span class='op'>=</span><span class='fl'>10</span>,minSamplesLeaf<span class='op'>=</span><span class='fl'>10</span>, seed<span class='op'>=</span><span class='cn'>NULL</span> <span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">setDecisionTree</span><span class="op">(</span></span>
+<span>  criterion <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="st">"gini"</span><span class="op">)</span>,</span>
+<span>  splitter <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="st">"best"</span><span class="op">)</span>,</span>
+<span>  maxDepth <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/integer.html" class="external-link">as.integer</a></span><span class="op">(</span><span class="fl">4</span><span class="op">)</span>, <span class="fu"><a href="https://rdrr.io/r/base/integer.html" class="external-link">as.integer</a></span><span class="op">(</span><span class="fl">10</span><span class="op">)</span>, <span class="cn">NULL</span><span class="op">)</span>,</span>
+<span>  minSamplesSplit <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">2</span>, <span class="fl">10</span><span class="op">)</span>,</span>
+<span>  minSamplesLeaf <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">10</span>, <span class="fl">50</span><span class="op">)</span>,</span>
+<span>  minWeightFractionLeaf <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">0</span><span class="op">)</span>,</span>
+<span>  maxFeatures <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">100</span>, <span class="st">"sqrt"</span>, <span class="cn">NULL</span><span class="op">)</span>,</span>
+<span>  maxLeafNodes <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="cn">NULL</span><span class="op">)</span>,</span>
+<span>  minImpurityDecrease <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">10</span><span class="op">^</span><span class="op">-</span><span class="fl">7</span><span class="op">)</span>,</span>
+<span>  classWeight <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="cn">NULL</span><span class="op">)</span>,</span>
+<span>  seed <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/sample.html" class="external-link">sample</a></span><span class="op">(</span><span class="fl">1e+06</span>, <span class="fl">1</span><span class="op">)</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>criterion</dt>
+<dd><p>The function to measure the quality of a split. Supported criteria are “gini” for the Gini impurity and “entropy” for the information gain.</p></dd>
+
+
+<dt>splitter</dt>
+<dd><p>The strategy used to choose the split at each node. Supported strategies are “best” to choose the best split and “random” to choose the best random split.</p></dd>
+
+
+<dt>maxDepth</dt>
+<dd><p>(list) The maximum depth of the tree. If NULL, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples.</p></dd>
+
+
+<dt>minSamplesSplit</dt>
+<dd><p>The minimum number of samples required to split an internal node</p></dd>
+
+
+<dt>minSamplesLeaf</dt>
+<dd><p>The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least minSamplesLeaf training samples in each of the left and right branches. This may have the effect of smoothing the model, especially in regression.</p></dd>
+
+
+<dt>minWeightFractionLeaf</dt>
+<dd><p>The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sampleWeight is not provided.</p></dd>
+
+
+<dt>maxFeatures</dt>
+<dd><p>(list) The number of features to consider when looking for the best split (int/'sqrt'/NULL)</p></dd>
+
+
+<dt>maxLeafNodes</dt>
+<dd><p>(list) Grow a tree with max_leaf_nodes in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes. (int/NULL)</p></dd>
+
+
+<dt>minImpurityDecrease</dt>
+<dd><p>Threshold for early stopping in tree growth. A node will split if its impurity is above the threshold, otherwise it is a leaf.</p></dd>
+
+
+<dt>classWeight</dt>
+<dd><p>(list) Weights associated with classes 'balance' or NULL</p></dd>
+
+
+<dt>seed</dt>
+<dd><p>The random state seed</p></dd>
+
+</dl></div>
+
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="kw">if</span> <span class="op">(</span><span class="cn">FALSE</span><span class="op">)</span> <span class="op">{</span></span></span>
+<span class="r-in"><span><span class="va">model.decisionTree</span> <span class="op">&lt;-</span> <span class="fu">setDecisionTree</span><span class="op">(</span>maxDepth<span class="op">=</span><span class="fl">10</span>,minSamplesLeaf<span class="op">=</span><span class="fl">10</span>, seed<span class="op">=</span><span class="cn">NULL</span> <span class="op">)</span></span></span>
+<span class="r-in"><span><span class="op">}</span></span></span>
+</code></pre></div>
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/setDeepNN.html b/docs/reference/setDeepNN.html
deleted file mode 100644
index d037ae6af..000000000
--- a/docs/reference/setDeepNN.html
+++ /dev/null
@@ -1,261 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for DeepNN model — setDeepNN • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for DeepNN model — setDeepNN" />
-<meta property="og:description" content="Create setting for DeepNN model" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.3.10</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Create setting for DeepNN model</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/DeepNN.R'><code>R/DeepNN.R</code></a></small>
-    <div class="hidden name"><code>setDeepNN.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Create setting for DeepNN model</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>setDeepNN</span><span class='op'>(</span>
-  units <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>128</span>, <span class='fl'>64</span><span class='op'>)</span>, <span class='fl'>128</span><span class='op'>)</span>,
-  layer_dropout <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>0.2</span><span class='op'>)</span>,
-  lr <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>1e-04</span><span class='op'>)</span>,
-  decay <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>1e-05</span><span class='op'>)</span>,
-  outcome_weight <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>1</span><span class='op'>)</span>,
-  batch_size <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>100</span><span class='op'>)</span>,
-  epochs <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>100</span><span class='op'>)</span>,
-  seed <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>units</th>
-      <td><p>The number of units of the deep network - as a list of vectors</p></td>
-    </tr>
-    <tr>
-      <th>layer_dropout</th>
-      <td><p>The layer dropout rate (regularisation)</p></td>
-    </tr>
-    <tr>
-      <th>lr</th>
-      <td><p>Learning rate</p></td>
-    </tr>
-    <tr>
-      <th>decay</th>
-      <td><p>Learning rate decay over each update.</p></td>
-    </tr>
-    <tr>
-      <th>outcome_weight</th>
-      <td><p>The weight of the outcome class in the loss function</p></td>
-    </tr>
-    <tr>
-      <th>batch_size</th>
-      <td><p>The number of data points to use per training batch</p></td>
-    </tr>
-    <tr>
-      <th>epochs</th>
-      <td><p>Number of times to iterate over dataset</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>Random seed used by deep learning model</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>model</span> <span class='op'>&lt;-</span> <span class='fu'>setDeepNN</span><span class='op'>(</span><span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/setGBMSurvival.html b/docs/reference/setGBMSurvival.html
deleted file mode 100644
index 037065ffb..000000000
--- a/docs/reference/setGBMSurvival.html
+++ /dev/null
@@ -1,320 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for GBM Survival with python
-#' @description
-This creates a setting for fitting  GBM surivial model.  You need sksurv python install.  To install this open your command line and type: conda install -c sebp scikit-survival — setGBMSurvival • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for GBM Survival with python
-#' @description
-This creates a setting for fitting  GBM surivial model.  You need sksurv python install.  To install this open your command line and type: conda install -c sebp scikit-survival — setGBMSurvival" />
-<meta property="og:description" content="Create setting for GBM Survival with python
-#' @description
-This creates a setting for fitting  GBM surivial model.  You need sksurv python install.  To install this open your command line and type: conda install -c sebp scikit-survival" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Create setting for GBM Survival with python
-#' @description
-This creates a setting for fitting  GBM surivial model.  You need sksurv python install.  To install this open your command line and type: conda install -c sebp scikit-survival</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/GBMSurvival.R'><code>R/GBMSurvival.R</code></a></small>
-    <div class="hidden name"><code>setGBMSurvival.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Create setting for GBM Survival with python
-#' @description
-This creates a setting for fitting  GBM surivial model.  You need sksurv python install.  To install this open your command line and type: conda install -c sebp scikit-survival</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>setGBMSurvival</span><span class='op'>(</span>
-  loss <span class='op'>=</span> <span class='st'>"coxph"</span>,
-  learningRate <span class='op'>=</span> <span class='fl'>0.1</span>,
-  nEstimators <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>100</span><span class='op'>)</span>,
-  criterion <span class='op'>=</span> <span class='st'>"friedman_mse"</span>,
-  minSamplesSplit <span class='op'>=</span> <span class='fl'>2</span>,
-  minSamplesLeaf <span class='op'>=</span> <span class='fl'>1</span>,
-  minWeightFractionLeaf <span class='op'>=</span> <span class='fl'>0</span>,
-  maxDepth <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>3</span>, <span class='fl'>10</span>, <span class='fl'>17</span><span class='op'>)</span>,
-  minImpuritySplit <span class='op'>=</span> <span class='cn'>NULL</span>,
-  minImpurityDecrease <span class='op'>=</span> <span class='fl'>0</span>,
-  maxFeatures <span class='op'>=</span> <span class='cn'>NULL</span>,
-  maxLeafNodes <span class='op'>=</span> <span class='cn'>NULL</span>,
-  presort <span class='op'>=</span> <span class='cn'>NULL</span>,
-  subsample <span class='op'>=</span> <span class='fl'>1</span>,
-  dropoutRate <span class='op'>=</span> <span class='fl'>0</span>,
-  seed <span class='op'>=</span> <span class='cn'>NULL</span>,
-  quiet <span class='op'>=</span> <span class='cn'>F</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>loss</th>
-      <td><p>A string specifying the loss function to minimise (default: 'coxph' )</p></td>
-    </tr>
-    <tr>
-      <th>learningRate</th>
-      <td><p>A double specifying the learning rate (controls convergence speed)</p></td>
-    </tr>
-    <tr>
-      <th>nEstimators</th>
-      <td><p>An integer specifying how many trees to build</p></td>
-    </tr>
-    <tr>
-      <th>criterion</th>
-      <td><p>Default: 'friedman_mse'</p></td>
-    </tr>
-    <tr>
-      <th>minSamplesSplit</th>
-      <td><p>An integer specifying min samples per tree split (complexity)</p></td>
-    </tr>
-    <tr>
-      <th>minSamplesLeaf</th>
-      <td><p>An integer specifying min samples per leaf (complexity)</p></td>
-    </tr>
-    <tr>
-      <th>minWeightFractionLeaf</th>
-      <td><p>Lookup</p></td>
-    </tr>
-    <tr>
-      <th>maxDepth</th>
-      <td><p>An integer specifying the max depth of trees (complexity)</p></td>
-    </tr>
-    <tr>
-      <th>minImpuritySplit</th>
-      <td><p>A double or NULL specifying the minimum impurity split</p></td>
-    </tr>
-    <tr>
-      <th>minImpurityDecrease</th>
-      <td><p>will add</p></td>
-    </tr>
-    <tr>
-      <th>maxFeatures</th>
-      <td><p>will add</p></td>
-    </tr>
-    <tr>
-      <th>maxLeafNodes</th>
-      <td><p>will add</p></td>
-    </tr>
-    <tr>
-      <th>presort</th>
-      <td><p>will add</p></td>
-    </tr>
-    <tr>
-      <th>subsample</th>
-      <td><p>will add</p></td>
-    </tr>
-    <tr>
-      <th>dropoutRate</th>
-      <td><p>will add</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>will add</p></td>
-    </tr>
-    <tr>
-      <th>quiet</th>
-      <td><p>will add</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Pick the hyper-parameters you want to do a grid search for</p>
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>gbmSurv</span> <span class='op'>&lt;-</span> <span class='fu'>setGBMSurvival</span><span class='op'>(</span>learningRate<span class='op'>=</span><span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>0.1</span>,<span class='fl'>0.01</span><span class='op'>)</span>, nEstimators <span class='op'>=</span><span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>10</span>,<span class='fl'>50</span>,<span class='fl'>100</span><span class='op'>)</span>,
- maxDepth<span class='op'>=</span><span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>4</span>,<span class='fl'>10</span>,<span class='fl'>17</span><span class='op'>)</span>, seed <span class='op'>=</span> <span class='fl'>2</span><span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/setGradientBoostingMachine.html b/docs/reference/setGradientBoostingMachine.html
index e37a071cd..125a8d43b 100644
--- a/docs/reference/setGradientBoostingMachine.html
+++ b/docs/reference/setGradientBoostingMachine.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for gradient boosting machine model using gbm_xgboost implementation — setGradientBoostingMachine • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for gradient boosting machine model using gbm_xgboost implementation — setGradientBoostingMachine" />
-<meta property="og:description" content="Create setting for gradient boosting machine model using gbm_xgboost implementation" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create setting for gradient boosting machine model using gbm_xgboost implementation — setGradientBoostingMachine • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create setting for gradient boosting machine model using gbm_xgboost implementation — setGradientBoostingMachine"><meta property="og:description" content="Create setting for gradient boosting machine model using gbm_xgboost implementation"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create setting for gradient boosting machine model using gbm_xgboost implementation</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/GradientBoostingMachine.R'><code>R/GradientBoostingMachine.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/GradientBoostingMachine.R" class="external-link"><code>R/GradientBoostingMachine.R</code></a></small>
     <div class="hidden name"><code>setGradientBoostingMachine.Rd</code></div>
     </div>
 
@@ -183,80 +131,92 @@ <h1>Create setting for gradient boosting machine model using gbm_xgboost impleme
     <p>Create setting for gradient boosting machine model using gbm_xgboost implementation</p>
     </div>
 
-    <pre class="usage"><span class='fu'>setGradientBoostingMachine</span><span class='op'>(</span>
-  ntrees <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>100</span>, <span class='fl'>1000</span><span class='op'>)</span>,
-  nthread <span class='op'>=</span> <span class='fl'>20</span>,
-  earlyStopRound <span class='op'>=</span> <span class='fl'>25</span>,
-  maxDepth <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>4</span>, <span class='fl'>6</span>, <span class='fl'>17</span><span class='op'>)</span>,
-  minRows <span class='op'>=</span> <span class='fl'>2</span>,
-  learnRate <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>0.005</span>, <span class='fl'>0.01</span>, <span class='fl'>0.1</span><span class='op'>)</span>,
-  seed <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/sample.html'>sample</a></span><span class='op'>(</span><span class='fl'>1e+07</span>, <span class='fl'>1</span><span class='op'>)</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>ntrees</th>
-      <td><p>The number of trees to build</p></td>
-    </tr>
-    <tr>
-      <th>nthread</th>
-      <td><p>The number of computer threads to (how many cores do you have?)</p></td>
-    </tr>
-    <tr>
-      <th>earlyStopRound</th>
-      <td><p>If the performance does not increase over earlyStopRound number of interactions then training stops (this prevents overfitting)</p></td>
-    </tr>
-    <tr>
-      <th>maxDepth</th>
-      <td><p>Maximum number of interactions - a large value will lead to slow model training</p></td>
-    </tr>
-    <tr>
-      <th>minRows</th>
-      <td><p>The minimum number of rows required at each end node of the tree</p></td>
-    </tr>
-    <tr>
-      <th>learnRate</th>
-      <td><p>The boosting learn rate</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>An option to add a seed when training the final model</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='va'>model.gbm</span> <span class='op'>&lt;-</span> <span class='fu'>setGradientBoostingMachine</span><span class='op'>(</span>ntrees<span class='op'>=</span><span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>10</span>,<span class='fl'>100</span><span class='op'>)</span>, nthread<span class='op'>=</span><span class='fl'>20</span>,
-                           maxDepth<span class='op'>=</span><span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>4</span>,<span class='fl'>6</span><span class='op'>)</span>, learnRate<span class='op'>=</span><span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>0.1</span>,<span class='fl'>0.3</span><span class='op'>)</span><span class='op'>)</span>
-
-</div></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">setGradientBoostingMachine</span><span class="op">(</span></span>
+<span>  ntrees <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">100</span>, <span class="fl">300</span><span class="op">)</span>,</span>
+<span>  nthread <span class="op">=</span> <span class="fl">20</span>,</span>
+<span>  earlyStopRound <span class="op">=</span> <span class="fl">25</span>,</span>
+<span>  maxDepth <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">4</span>, <span class="fl">6</span>, <span class="fl">8</span><span class="op">)</span>,</span>
+<span>  minChildWeight <span class="op">=</span> <span class="fl">1</span>,</span>
+<span>  learnRate <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">0.05</span>, <span class="fl">0.1</span>, <span class="fl">0.3</span><span class="op">)</span>,</span>
+<span>  scalePosWeight <span class="op">=</span> <span class="fl">1</span>,</span>
+<span>  lambda <span class="op">=</span> <span class="fl">1</span>,</span>
+<span>  alpha <span class="op">=</span> <span class="fl">0</span>,</span>
+<span>  seed <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/sample.html" class="external-link">sample</a></span><span class="op">(</span><span class="fl">1e+07</span>, <span class="fl">1</span><span class="op">)</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>ntrees</dt>
+<dd><p>The number of trees to build</p></dd>
+
+
+<dt>nthread</dt>
+<dd><p>The number of computer threads to use (how many cores do you have?)</p></dd>
+
+
+<dt>earlyStopRound</dt>
+<dd><p>If the performance does not increase over earlyStopRound number of trees then training stops (this prevents overfitting)</p></dd>
+
+
+<dt>maxDepth</dt>
+<dd><p>Maximum depth of each tree - a large value will lead to slow model training</p></dd>
+
+
+<dt>minChildWeight</dt>
+<dd><p>Minimum sum of of instance weight in a child node - larger values are more conservative</p></dd>
+
+
+<dt>learnRate</dt>
+<dd><p>The boosting learn rate</p></dd>
+
+
+<dt>scalePosWeight</dt>
+<dd><p>Controls weight of positive class in loss - useful for imbalanced classes</p></dd>
+
+
+<dt>lambda</dt>
+<dd><p>L2 regularization on weights - larger is more conservative</p></dd>
+
+
+<dt>alpha</dt>
+<dd><p>L1 regularization on weights - larger is more conservative</p></dd>
+
+
+<dt>seed</dt>
+<dd><p>An option to add a seed when training the final model</p></dd>
+
+</dl></div>
+
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="va">model.gbm</span> <span class="op">&lt;-</span> <span class="fu">setGradientBoostingMachine</span><span class="op">(</span>ntrees<span class="op">=</span><span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">10</span>,<span class="fl">100</span><span class="op">)</span>, nthread<span class="op">=</span><span class="fl">20</span>,</span></span>
+<span class="r-in"><span>                           maxDepth<span class="op">=</span><span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">4</span>,<span class="fl">6</span><span class="op">)</span>, learnRate<span class="op">=</span><span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">0.1</span>,<span class="fl">0.3</span><span class="op">)</span><span class="op">)</span></span></span>
+<span class="r-in"><span></span></span>
+</code></pre></div>
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/setIterativeHardThresholding.html b/docs/reference/setIterativeHardThresholding.html
index 8bd6b1afb..6a234802c 100644
--- a/docs/reference/setIterativeHardThresholding.html
+++ b/docs/reference/setIterativeHardThresholding.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for lasso logistic regression — setIterativeHardThresholding • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for lasso logistic regression — setIterativeHardThresholding" />
-<meta property="og:description" content="Create setting for lasso logistic regression" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create setting for lasso logistic regression — setIterativeHardThresholding • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create setting for lasso logistic regression — setIterativeHardThresholding"><meta property="og:description" content="Create setting for lasso logistic regression"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create setting for lasso logistic regression</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/CyclopsSettings.R'><code>R/CyclopsSettings.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/CyclopsSettings.R" class="external-link"><code>R/CyclopsSettings.R</code></a></small>
     <div class="hidden name"><code>setIterativeHardThresholding.Rd</code></div>
     </div>
 
@@ -183,98 +131,95 @@ <h1>Create setting for lasso logistic regression</h1>
     <p>Create setting for lasso logistic regression</p>
     </div>
 
-    <pre class="usage"><span class='fu'>setIterativeHardThresholding</span><span class='op'>(</span>
-  K <span class='op'>=</span> <span class='fl'>10</span>,
-  penalty <span class='op'>=</span> <span class='st'>"bic"</span>,
-  seed <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/sample.html'>sample</a></span><span class='op'>(</span><span class='fl'>1e+05</span>, <span class='fl'>1</span><span class='op'>)</span>,
-  exclude <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='op'>)</span>,
-  forceIntercept <span class='op'>=</span> <span class='cn'>F</span>,
-  fitBestSubset <span class='op'>=</span> <span class='cn'>FALSE</span>,
-  initialRidgeVariance <span class='op'>=</span> <span class='fl'>10000</span>,
-  tolerance <span class='op'>=</span> <span class='fl'>1e-08</span>,
-  maxIterations <span class='op'>=</span> <span class='fl'>10000</span>,
-  threshold <span class='op'>=</span> <span class='fl'>1e-06</span>,
-  delta <span class='op'>=</span> <span class='fl'>0</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>K</th>
-      <td><p>The maximum number of non-zero predictors</p></td>
-    </tr>
-    <tr>
-      <th>penalty</th>
-      <td><p>Specifies the IHT penalty; possible values are `BIC` or `AIC` or a numeric value</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>An option to add a seed when training the model</p></td>
-    </tr>
-    <tr>
-      <th>exclude</th>
-      <td><p>A vector of numbers or covariateId names to exclude from prior</p></td>
-    </tr>
-    <tr>
-      <th>forceIntercept</th>
-      <td><p>Logical: Force intercept coefficient into regularization</p></td>
-    </tr>
-    <tr>
-      <th>fitBestSubset</th>
-      <td><p>Logical: Fit final subset with no regularization</p></td>
-    </tr>
-    <tr>
-      <th>initialRidgeVariance</th>
-      <td><p>integer</p></td>
-    </tr>
-    <tr>
-      <th>tolerance</th>
-      <td><p>numeric</p></td>
-    </tr>
-    <tr>
-      <th>maxIterations</th>
-      <td><p>integer</p></td>
-    </tr>
-    <tr>
-      <th>threshold</th>
-      <td><p>numeric</p></td>
-    </tr>
-    <tr>
-      <th>delta</th>
-      <td><p>numeric</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='va'>model.lr</span> <span class='op'>&lt;-</span> <span class='fu'><a href='setLassoLogisticRegression.html'>setLassoLogisticRegression</a></span><span class='op'>(</span><span class='op'>)</span>
-</div></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">setIterativeHardThresholding</span><span class="op">(</span></span>
+<span>  K <span class="op">=</span> <span class="fl">10</span>,</span>
+<span>  penalty <span class="op">=</span> <span class="st">"bic"</span>,</span>
+<span>  seed <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/sample.html" class="external-link">sample</a></span><span class="op">(</span><span class="fl">1e+05</span>, <span class="fl">1</span><span class="op">)</span>,</span>
+<span>  exclude <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  forceIntercept <span class="op">=</span> <span class="cn">F</span>,</span>
+<span>  fitBestSubset <span class="op">=</span> <span class="cn">FALSE</span>,</span>
+<span>  initialRidgeVariance <span class="op">=</span> <span class="fl">10000</span>,</span>
+<span>  tolerance <span class="op">=</span> <span class="fl">1e-08</span>,</span>
+<span>  maxIterations <span class="op">=</span> <span class="fl">10000</span>,</span>
+<span>  threshold <span class="op">=</span> <span class="fl">1e-06</span>,</span>
+<span>  delta <span class="op">=</span> <span class="fl">0</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>K</dt>
+<dd><p>The maximum number of non-zero predictors</p></dd>
+
+
+<dt>penalty</dt>
+<dd><p>Specifies the IHT penalty; possible values are `BIC` or `AIC` or a numeric value</p></dd>
+
+
+<dt>seed</dt>
+<dd><p>An option to add a seed when training the model</p></dd>
+
+
+<dt>exclude</dt>
+<dd><p>A vector of numbers or covariateId names to exclude from prior</p></dd>
+
+
+<dt>forceIntercept</dt>
+<dd><p>Logical: Force intercept coefficient into regularization</p></dd>
+
+
+<dt>fitBestSubset</dt>
+<dd><p>Logical: Fit final subset with no regularization</p></dd>
+
+
+<dt>initialRidgeVariance</dt>
+<dd><p>integer</p></dd>
+
+
+<dt>tolerance</dt>
+<dd><p>numeric</p></dd>
+
+
+<dt>maxIterations</dt>
+<dd><p>integer</p></dd>
+
+
+<dt>threshold</dt>
+<dd><p>numeric</p></dd>
+
+
+<dt>delta</dt>
+<dd><p>numeric</p></dd>
+
+</dl></div>
+
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="va">model.lr</span> <span class="op">&lt;-</span> <span class="fu"><a href="setLassoLogisticRegression.html">setLassoLogisticRegression</a></span><span class="op">(</span><span class="op">)</span></span></span>
+</code></pre></div>
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/setKNN.html b/docs/reference/setKNN.html
index 8d8a8c2ca..e3025432b 100644
--- a/docs/reference/setKNN.html
+++ b/docs/reference/setKNN.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for knn model — setKNN • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for knn model — setKNN" />
-<meta property="og:description" content="Create setting for knn model" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create setting for knn model — setKNN • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create setting for knn model — setKNN"><meta property="og:description" content="Create setting for knn model"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create setting for knn model</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/KNN.R'><code>R/KNN.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/KNN.R" class="external-link"><code>R/KNN.R</code></a></small>
     <div class="hidden name"><code>setKNN.Rd</code></div>
     </div>
 
@@ -183,56 +131,53 @@ <h1>Create setting for knn model</h1>
     <p>Create setting for knn model</p>
     </div>
 
-    <pre class="usage"><span class='fu'>setKNN</span><span class='op'>(</span>k <span class='op'>=</span> <span class='fl'>1000</span>, indexFolder <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/file.path.html'>file.path</a></span><span class='op'>(</span><span class='fu'><a href='https://rdrr.io/r/base/getwd.html'>getwd</a></span><span class='op'>(</span><span class='op'>)</span>, <span class='st'>"knn"</span><span class='op'>)</span>, threads <span class='op'>=</span> <span class='fl'>1</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>k</th>
-      <td><p>The number of neighbors to consider</p></td>
-    </tr>
-    <tr>
-      <th>indexFolder</th>
-      <td><p>The directory where the results and intermediate steps are output</p></td>
-    </tr>
-    <tr>
-      <th>threads</th>
-      <td><p>The number of threads to use when applying big knn</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>model.knn</span> <span class='op'>&lt;-</span> <span class='fu'>setKNN</span><span class='op'>(</span>k<span class='op'>=</span><span class='fl'>10000</span><span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">setKNN</span><span class="op">(</span>k <span class="op">=</span> <span class="fl">1000</span>, indexFolder <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html" class="external-link">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/getwd.html" class="external-link">getwd</a></span><span class="op">(</span><span class="op">)</span>, <span class="st">"knn"</span><span class="op">)</span>, threads <span class="op">=</span> <span class="fl">1</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>k</dt>
+<dd><p>The number of neighbors to consider</p></dd>
+
+
+<dt>indexFolder</dt>
+<dd><p>The directory where the results and intermediate steps are output</p></dd>
+
+
+<dt>threads</dt>
+<dd><p>The number of threads to use when applying big knn</p></dd>
+
+</dl></div>
+
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="kw">if</span> <span class="op">(</span><span class="cn">FALSE</span><span class="op">)</span> <span class="op">{</span></span></span>
+<span class="r-in"><span><span class="va">model.knn</span> <span class="op">&lt;-</span> <span class="fu">setKNN</span><span class="op">(</span>k<span class="op">=</span><span class="fl">10000</span><span class="op">)</span></span></span>
+<span class="r-in"><span><span class="op">}</span></span></span>
+</code></pre></div>
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/setLRTorch.html b/docs/reference/setLRTorch.html
deleted file mode 100644
index 0ea3d5ac4..000000000
--- a/docs/reference/setLRTorch.html
+++ /dev/null
@@ -1,253 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for logistics regression model with python — setLRTorch • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for logistics regression model with python — setLRTorch" />
-<meta property="og:description" content="Create setting for logistics regression model with python" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.3.10</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Create setting for logistics regression model with python</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/LRTorch.R'><code>R/LRTorch.R</code></a></small>
-    <div class="hidden name"><code>setLRTorch.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Create setting for logistics regression model with python</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>setLRTorch</span><span class='op'>(</span>
-  w_decay <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>5e-04</span>, <span class='fl'>0.005</span><span class='op'>)</span>,
-  epochs <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>20</span>, <span class='fl'>50</span>, <span class='fl'>100</span><span class='op'>)</span>,
-  seed <span class='op'>=</span> <span class='cn'>NULL</span>,
-  class_weight <span class='op'>=</span> <span class='fl'>0</span>,
-  autoencoder <span class='op'>=</span> <span class='cn'>FALSE</span>,
-  vae <span class='op'>=</span> <span class='cn'>FALSE</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>w_decay</th>
-      <td><p>The l2 regularisation</p></td>
-    </tr>
-    <tr>
-      <th>epochs</th>
-      <td><p>The number of epochs</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>A seed for the model</p></td>
-    </tr>
-    <tr>
-      <th>class_weight</th>
-      <td><p>The class weight used for imbalanced data: 
- 0: Inverse ratio between positives and negatives
--1: Focal loss</p></td>
-    </tr>
-    <tr>
-      <th>autoencoder</th>
-      <td><p>First learn stakced autoencoder for input features, then train LR on the encoded features.</p></td>
-    </tr>
-    <tr>
-      <th>vae</th>
-      <td><p>First learn stakced varational autoencoder for input features, then train LR on the encoded features.</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>model.lrTorch</span> <span class='op'>&lt;-</span> <span class='fu'>setLRTorch</span><span class='op'>(</span><span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/setLassoLogisticRegression.html b/docs/reference/setLassoLogisticRegression.html
index aaccccd2e..05f3b0a0b 100644
--- a/docs/reference/setLassoLogisticRegression.html
+++ b/docs/reference/setLassoLogisticRegression.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for lasso logistic regression — setLassoLogisticRegression • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for lasso logistic regression — setLassoLogisticRegression" />
-<meta property="og:description" content="Create setting for lasso logistic regression" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create setting for lasso logistic regression — setLassoLogisticRegression • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create setting for lasso logistic regression — setLassoLogisticRegression"><meta property="og:description" content="Create setting for lasso logistic regression"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create setting for lasso logistic regression</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/CyclopsSettings.R'><code>R/CyclopsSettings.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/CyclopsSettings.R" class="external-link"><code>R/CyclopsSettings.R</code></a></small>
     <div class="hidden name"><code>setLassoLogisticRegression.Rd</code></div>
     </div>
 
@@ -183,93 +131,95 @@ <h1>Create setting for lasso logistic regression</h1>
     <p>Create setting for lasso logistic regression</p>
     </div>
 
-    <pre class="usage"><span class='fu'>setLassoLogisticRegression</span><span class='op'>(</span>
-  variance <span class='op'>=</span> <span class='fl'>0.01</span>,
-  seed <span class='op'>=</span> <span class='cn'>NULL</span>,
-  includeCovariateIds <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='op'>)</span>,
-  noShrinkage <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>0</span><span class='op'>)</span>,
-  threads <span class='op'>=</span> <span class='op'>-</span><span class='fl'>1</span>,
-  forceIntercept <span class='op'>=</span> <span class='cn'>F</span>,
-  upperLimit <span class='op'>=</span> <span class='fl'>20</span>,
-  lowerLimit <span class='op'>=</span> <span class='fl'>0.01</span>,
-  tolerance <span class='op'>=</span> <span class='fl'>2e-06</span>,
-  maxIterations <span class='op'>=</span> <span class='fl'>3000</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>variance</th>
-      <td><p>Numeric: prior distribution starting variance</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>An option to add a seed when training the model</p></td>
-    </tr>
-    <tr>
-      <th>includeCovariateIds</th>
-      <td><p>a set of covariate IDS to limit the analysis to</p></td>
-    </tr>
-    <tr>
-      <th>noShrinkage</th>
-      <td><p>a set of covariates whcih are to be forced to be included in the final model. default is the intercept</p></td>
-    </tr>
-    <tr>
-      <th>threads</th>
-      <td><p>An option to set number of threads when training model</p></td>
-    </tr>
-    <tr>
-      <th>forceIntercept</th>
-      <td><p>Logical: Force intercept coefficient into prior</p></td>
-    </tr>
-    <tr>
-      <th>upperLimit</th>
-      <td><p>Numeric: Upper prior variance limit for grid-search</p></td>
-    </tr>
-    <tr>
-      <th>lowerLimit</th>
-      <td><p>Numeric: Lower prior variance limit for grid-search</p></td>
-    </tr>
-    <tr>
-      <th>tolerance</th>
-      <td><p>Numeric: maximum relative change in convergence criterion from successive iterations to achieve convergence</p></td>
-    </tr>
-    <tr>
-      <th>maxIterations</th>
-      <td><p>Integer: maximum iterations of Cyclops to attempt before returning a failed-to-converge error</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='va'>model.lr</span> <span class='op'>&lt;-</span> <span class='fu'>setLassoLogisticRegression</span><span class='op'>(</span><span class='op'>)</span>
-</div></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">setLassoLogisticRegression</span><span class="op">(</span></span>
+<span>  variance <span class="op">=</span> <span class="fl">0.01</span>,</span>
+<span>  seed <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  includeCovariateIds <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  noShrinkage <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">0</span><span class="op">)</span>,</span>
+<span>  threads <span class="op">=</span> <span class="op">-</span><span class="fl">1</span>,</span>
+<span>  forceIntercept <span class="op">=</span> <span class="cn">F</span>,</span>
+<span>  upperLimit <span class="op">=</span> <span class="fl">20</span>,</span>
+<span>  lowerLimit <span class="op">=</span> <span class="fl">0.01</span>,</span>
+<span>  tolerance <span class="op">=</span> <span class="fl">2e-06</span>,</span>
+<span>  maxIterations <span class="op">=</span> <span class="fl">3000</span>,</span>
+<span>  priorCoefs <span class="op">=</span> <span class="cn">NULL</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>variance</dt>
+<dd><p>Numeric: prior distribution starting variance</p></dd>
+
+
+<dt>seed</dt>
+<dd><p>An option to add a seed when training the model</p></dd>
+
+
+<dt>includeCovariateIds</dt>
+<dd><p>a set of covariate IDS to limit the analysis to</p></dd>
+
+
+<dt>noShrinkage</dt>
+<dd><p>a set of covariates whcih are to be forced to be included in the final model. default is the intercept</p></dd>
+
+
+<dt>threads</dt>
+<dd><p>An option to set number of threads when training model</p></dd>
+
+
+<dt>forceIntercept</dt>
+<dd><p>Logical: Force intercept coefficient into prior</p></dd>
+
+
+<dt>upperLimit</dt>
+<dd><p>Numeric: Upper prior variance limit for grid-search</p></dd>
+
+
+<dt>lowerLimit</dt>
+<dd><p>Numeric: Lower prior variance limit for grid-search</p></dd>
+
+
+<dt>tolerance</dt>
+<dd><p>Numeric: maximum relative change in convergence criterion from successive iterations to achieve convergence</p></dd>
+
+
+<dt>maxIterations</dt>
+<dd><p>Integer: maximum iterations of Cyclops to attempt before returning a failed-to-converge error</p></dd>
+
+
+<dt>priorCoefs</dt>
+<dd><p>Use coefficients from a previous model as starting points for model fit (transfer learning)</p></dd>
+
+</dl></div>
+
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="va">model.lr</span> <span class="op">&lt;-</span> <span class="fu">setLassoLogisticRegression</span><span class="op">(</span><span class="op">)</span></span></span>
+</code></pre></div>
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/setLightGBM.html b/docs/reference/setLightGBM.html
new file mode 100644
index 000000000..d15720ee7
--- /dev/null
+++ b/docs/reference/setLightGBM.html
@@ -0,0 +1,234 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create setting for gradient boosting machine model using lightGBM (https://github.com/microsoft/LightGBM/tree/master/R-package). — setLightGBM • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create setting for gradient boosting machine model using lightGBM (https://github.com/microsoft/LightGBM/tree/master/R-package). — setLightGBM"><meta property="og:description" content="Create setting for gradient boosting machine model using lightGBM (https://github.com/microsoft/LightGBM/tree/master/R-package)."><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Create setting for gradient boosting machine model using lightGBM (https://github.com/microsoft/LightGBM/tree/master/R-package).</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/LightGBM.R" class="external-link"><code>R/LightGBM.R</code></a></small>
+    <div class="hidden name"><code>setLightGBM.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>Create setting for gradient boosting machine model using lightGBM (https://github.com/microsoft/LightGBM/tree/master/R-package).</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">setLightGBM</span><span class="op">(</span></span>
+<span>  nthread <span class="op">=</span> <span class="fl">20</span>,</span>
+<span>  earlyStopRound <span class="op">=</span> <span class="fl">25</span>,</span>
+<span>  numIterations <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">100</span><span class="op">)</span>,</span>
+<span>  numLeaves <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">31</span><span class="op">)</span>,</span>
+<span>  maxDepth <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">5</span>, <span class="fl">10</span><span class="op">)</span>,</span>
+<span>  minDataInLeaf <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">20</span><span class="op">)</span>,</span>
+<span>  learningRate <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">0.05</span>, <span class="fl">0.1</span>, <span class="fl">0.3</span><span class="op">)</span>,</span>
+<span>  lambdaL1 <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">0</span><span class="op">)</span>,</span>
+<span>  lambdaL2 <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">0</span><span class="op">)</span>,</span>
+<span>  scalePosWeight <span class="op">=</span> <span class="fl">1</span>,</span>
+<span>  isUnbalance <span class="op">=</span> <span class="cn">FALSE</span>,</span>
+<span>  seed <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/sample.html" class="external-link">sample</a></span><span class="op">(</span><span class="fl">1e+07</span>, <span class="fl">1</span><span class="op">)</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>nthread</dt>
+<dd><p>The number of computer threads to use (how many cores do you have?)</p></dd>
+
+
+<dt>earlyStopRound</dt>
+<dd><p>If the performance does not increase over earlyStopRound number of trees then training stops (this prevents overfitting)</p></dd>
+
+
+<dt>numIterations</dt>
+<dd><p>Number of boosting iterations.</p></dd>
+
+
+<dt>numLeaves</dt>
+<dd><p>This hyperparameter sets the maximum number of leaves. Increasing this parameter can lead to higher model complexity and potential overfitting.</p></dd>
+
+
+<dt>maxDepth</dt>
+<dd><p>This hyperparameter sets the maximum depth . Increasing this parameter can also lead to higher model complexity and potential overfitting.</p></dd>
+
+
+<dt>minDataInLeaf</dt>
+<dd><p>This hyperparameter sets the minimum number of data points that must be present in a leaf node. Increasing this parameter can help to reduce overfitting</p></dd>
+
+
+<dt>learningRate</dt>
+<dd><p>This hyperparameter controls the step size at each iteration of the gradient descent algorithm. Lower values can lead to slower convergence but may result in better performance.</p></dd>
+
+
+<dt>lambdaL1</dt>
+<dd><p>This hyperparameter controls L1 regularization, which can help to reduce overfitting by encouraging sparse models.</p></dd>
+
+
+<dt>lambdaL2</dt>
+<dd><p>This hyperparameter controls L2 regularization, which can also help to reduce overfitting by discouraging large weights in the model.</p></dd>
+
+
+<dt>scalePosWeight</dt>
+<dd><p>Controls weight of positive class in loss - useful for imbalanced classes</p></dd>
+
+
+<dt>isUnbalance</dt>
+<dd><p>This parameter cannot be used at the same time with scalePosWeight, choose only one of them. While enabling this should increase the overall performance metric of your model, it will also result in poor estimates of the individual class probabilities.</p></dd>
+
+
+<dt>seed</dt>
+<dd><p>An option to add a seed when training the final model</p></dd>
+
+</dl></div>
+
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="va">model.lightgbm</span> <span class="op">&lt;-</span> <span class="fu">setLightGBM</span><span class="op">(</span></span></span>
+<span class="r-in"><span>    numLeaves <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">20</span>, <span class="fl">31</span>, <span class="fl">50</span><span class="op">)</span>, maxDepth <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="op">-</span><span class="fl">1</span>, <span class="fl">5</span>, <span class="fl">10</span><span class="op">)</span>,</span></span>
+<span class="r-in"><span>    minDataInLeaf <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">10</span>, <span class="fl">20</span>, <span class="fl">30</span><span class="op">)</span>, learningRate <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">0.05</span>, <span class="fl">0.1</span>, <span class="fl">0.3</span><span class="op">)</span></span></span>
+<span class="r-in"><span><span class="op">)</span></span></span>
+<span class="r-in"><span></span></span>
+</code></pre></div>
+    </div>
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/setMLP.html b/docs/reference/setMLP.html
index 09cb9029f..f7f77fb07 100644
--- a/docs/reference/setMLP.html
+++ b/docs/reference/setMLP.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for neural network model with python — setMLP • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for neural network model with python — setMLP" />
-<meta property="og:description" content="Create setting for neural network model with python" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create setting for neural network model with python — setMLP • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create setting for neural network model with python — setMLP"><meta property="og:description" content="Create setting for neural network model with python"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create setting for neural network model with python</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SklearnClassifierSettings.R'><code>R/SklearnClassifierSettings.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SklearnClassifierSettings.R" class="external-link"><code>R/SklearnClassifierSettings.R</code></a></small>
     <div class="hidden name"><code>setMLP.Rd</code></div>
     </div>
 
@@ -183,155 +131,151 @@ <h1>Create setting for neural network model with python</h1>
     <p>Create setting for neural network model with python</p>
     </div>
 
-    <pre class="usage"><span class='fu'>setMLP</span><span class='op'>(</span>
-  hiddenLayerSizes <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>100</span><span class='op'>)</span>, <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>20</span>, <span class='fl'>4</span><span class='op'>)</span><span class='op'>)</span>,
-  activation <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='st'>"relu"</span><span class='op'>)</span>,
-  solver <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='st'>"adam"</span><span class='op'>)</span>,
-  alpha <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>0.3</span>, <span class='fl'>0.01</span>, <span class='fl'>1e-04</span>, <span class='fl'>1e-06</span><span class='op'>)</span>,
-  batchSize <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='st'>"auto"</span><span class='op'>)</span>,
-  learningRate <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='st'>"constant"</span><span class='op'>)</span>,
-  learningRateInit <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>0.001</span><span class='op'>)</span>,
-  powerT <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>0.5</span><span class='op'>)</span>,
-  maxIter <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>200</span>, <span class='fl'>100</span><span class='op'>)</span>,
-  shuffle <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='cn'>TRUE</span><span class='op'>)</span>,
-  tol <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>1e-04</span><span class='op'>)</span>,
-  warmStart <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='cn'>TRUE</span><span class='op'>)</span>,
-  momentum <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>0.9</span><span class='op'>)</span>,
-  nesterovsMomentum <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='cn'>TRUE</span><span class='op'>)</span>,
-  earlyStopping <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span>,
-  validationFraction <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>0.1</span><span class='op'>)</span>,
-  beta1 <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>0.9</span><span class='op'>)</span>,
-  beta2 <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>0.999</span><span class='op'>)</span>,
-  epsilon <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>1</span>, <span class='fl'>0.1</span>, <span class='fl'>1e-08</span><span class='op'>)</span>,
-  nIterNoChange <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>10</span><span class='op'>)</span>,
-  seed <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/sample.html'>sample</a></span><span class='op'>(</span><span class='fl'>1e+05</span>, <span class='fl'>1</span><span class='op'>)</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>hiddenLayerSizes</th>
-      <td><p>(list of vectors) The ith element represents the number of neurons in the ith hidden layer.</p></td>
-    </tr>
-    <tr>
-      <th>activation</th>
-      <td><p>(list) Activation function for the hidden layer.</p><ul>
-<li><p>"identity": no-op activation, useful to implement linear bottleneck, returns f(x) = x</p></li>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">setMLP</span><span class="op">(</span></span>
+<span>  hiddenLayerSizes <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">100</span><span class="op">)</span>, <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">20</span><span class="op">)</span><span class="op">)</span>,</span>
+<span>  activation <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="st">"relu"</span><span class="op">)</span>,</span>
+<span>  solver <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="st">"adam"</span><span class="op">)</span>,</span>
+<span>  alpha <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">0.3</span>, <span class="fl">0.01</span>, <span class="fl">1e-04</span>, <span class="fl">1e-06</span><span class="op">)</span>,</span>
+<span>  batchSize <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="st">"auto"</span><span class="op">)</span>,</span>
+<span>  learningRate <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="st">"constant"</span><span class="op">)</span>,</span>
+<span>  learningRateInit <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">0.001</span><span class="op">)</span>,</span>
+<span>  powerT <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">0.5</span><span class="op">)</span>,</span>
+<span>  maxIter <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">200</span>, <span class="fl">100</span><span class="op">)</span>,</span>
+<span>  shuffle <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="cn">TRUE</span><span class="op">)</span>,</span>
+<span>  tol <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">1e-04</span><span class="op">)</span>,</span>
+<span>  warmStart <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="cn">TRUE</span><span class="op">)</span>,</span>
+<span>  momentum <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">0.9</span><span class="op">)</span>,</span>
+<span>  nesterovsMomentum <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="cn">TRUE</span><span class="op">)</span>,</span>
+<span>  earlyStopping <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="cn">FALSE</span><span class="op">)</span>,</span>
+<span>  validationFraction <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">0.1</span><span class="op">)</span>,</span>
+<span>  beta1 <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">0.9</span><span class="op">)</span>,</span>
+<span>  beta2 <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">0.999</span><span class="op">)</span>,</span>
+<span>  epsilon <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">1e-08</span><span class="op">)</span>,</span>
+<span>  nIterNoChange <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">10</span><span class="op">)</span>,</span>
+<span>  seed <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/sample.html" class="external-link">sample</a></span><span class="op">(</span><span class="fl">1e+05</span>, <span class="fl">1</span><span class="op">)</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>hiddenLayerSizes</dt>
+<dd><p>(list of vectors) The ith element represents the number of neurons in the ith hidden layer.</p></dd>
+
+
+<dt>activation</dt>
+<dd><p>(list) Activation function for the hidden layer.</p><ul><li><p>"identity": no-op activation, useful to implement linear bottleneck, returns f(x) = x</p></li>
 <li><p>"logistic": the logistic sigmoid function, returns f(x) = 1 / (1 + exp(-x)).</p></li>
 <li><p>"tanh": the hyperbolic tan function, returns f(x) = tanh(x).</p></li>
 <li><p>"relu": the rectified linear unit function, returns f(x) = max(0, x)</p></li>
-</ul></td>
-    </tr>
-    <tr>
-      <th>solver</th>
-      <td><p>(list) The solver for weight optimization. (‘lbfgs’, ‘sgd’, ‘adam’)</p></td>
-    </tr>
-    <tr>
-      <th>alpha</th>
-      <td><p>(list) L2 penalty (regularization term) parameter.</p></td>
-    </tr>
-    <tr>
-      <th>batchSize</th>
-      <td><p>(list) Size of minibatches for stochastic optimizers. If the solver is ‘lbfgs’, the classifier will not use minibatch. When set to “auto”, batchSize=min(200, n_samples).</p></td>
-    </tr>
-    <tr>
-      <th>learningRate</th>
-      <td><p>(list) Only used when solver='sgd' Learning rate schedule for weight updates.‘constant’, ‘invscaling’, ‘adaptive’, default=’constant’</p></td>
-    </tr>
-    <tr>
-      <th>learningRateInit</th>
-      <td><p>(list) Only used when solver=’sgd’ or ‘adam’. The initial learning rate used. It controls the step-size in updating the weights.</p></td>
-    </tr>
-    <tr>
-      <th>powerT</th>
-      <td><p>(list) Only used when solver=’sgd’.  The exponent for inverse scaling learning rate. It is used in updating effective learning rate when the learning_rate is set to ‘invscaling’.</p></td>
-    </tr>
-    <tr>
-      <th>maxIter</th>
-      <td><p>(list)  Maximum number of iterations. The solver iterates until convergence (determined by ‘tol’) or this number of iterations. For stochastic solvers (‘sgd’, ‘adam’), note that this determines the number of epochs (how many times each data point will be used), not the number of gradient steps.</p></td>
-    </tr>
-    <tr>
-      <th>shuffle</th>
-      <td><p>(list) boolean: Whether to shuffle samples in each iteration. Only used when solver=’sgd’ or ‘adam’.</p></td>
-    </tr>
-    <tr>
-      <th>tol</th>
-      <td><p>(list) Tolerance for the optimization. When the loss or score is not improving by at least tol for nIterNoChange consecutive iterations, unless learning_rate is set to ‘adaptive’, convergence is considered to be reached and training stops.</p></td>
-    </tr>
-    <tr>
-      <th>warmStart</th>
-      <td><p>(list) When set to True, reuse the solution of the previous call to fit as initialization, otherwise, just erase the previous solution.</p></td>
-    </tr>
-    <tr>
-      <th>momentum</th>
-      <td><p>(list) Momentum for gradient descent update. Should be between 0 and 1. Only used when solver=’sgd’.</p></td>
-    </tr>
-    <tr>
-      <th>nesterovsMomentum</th>
-      <td><p>(list) Whether to use Nesterov’s momentum. Only used when solver=’sgd’ and momentum &gt; 0.</p></td>
-    </tr>
-    <tr>
-      <th>earlyStopping</th>
-      <td><p>(list) boolean Whether to use early stopping to terminate training when validation score is not improving. If set to true, it will automatically set aside 10 percent of training data as validation and terminate training when validation score is not improving by at least tol for n_iter_no_change consecutive epochs.</p></td>
-    </tr>
-    <tr>
-      <th>validationFraction</th>
-      <td><p>(list) The proportion of training data to set aside as validation set for early stopping. Must be between 0 and 1. Only used if earlyStopping is True.</p></td>
-    </tr>
-    <tr>
-      <th>beta1</th>
-      <td><p>(list) Exponential decay rate for estimates of first moment vector in adam, should be in 0 to 1.</p></td>
-    </tr>
-    <tr>
-      <th>beta2</th>
-      <td><p>(list) Exponential decay rate for estimates of second moment vector in adam, should be in 0 to 1.</p></td>
-    </tr>
-    <tr>
-      <th>epsilon</th>
-      <td><p>(list) Value for numerical stability in adam.</p></td>
-    </tr>
-    <tr>
-      <th>nIterNoChange</th>
-      <td><p>(list) Maximum number of epochs to not meet tol improvement. Only effective when solver=’sgd’ or ‘adam’.</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>A seed for the model</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>model.mlp</span> <span class='op'>&lt;-</span> <span class='fu'>setMLP</span><span class='op'>(</span><span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
+</ul></dd>
+
+
+<dt>solver</dt>
+<dd><p>(list) The solver for weight optimization. (‘lbfgs’, ‘sgd’, ‘adam’)</p></dd>
+
+
+<dt>alpha</dt>
+<dd><p>(list) L2 penalty (regularization term) parameter.</p></dd>
+
+
+<dt>batchSize</dt>
+<dd><p>(list) Size of minibatches for stochastic optimizers. If the solver is ‘lbfgs’, the classifier will not use minibatch. When set to “auto”, batchSize=min(200, n_samples).</p></dd>
+
+
+<dt>learningRate</dt>
+<dd><p>(list) Only used when solver='sgd' Learning rate schedule for weight updates. ‘constant’, ‘invscaling’, ‘adaptive’, default=’constant’</p></dd>
+
+
+<dt>learningRateInit</dt>
+<dd><p>(list) Only used when solver=’sgd’ or ‘adam’. The initial learning rate used. It controls the step-size in updating the weights.</p></dd>
+
+
+<dt>powerT</dt>
+<dd><p>(list) Only used when solver=’sgd’.  The exponent for inverse scaling learning rate. It is used in updating effective learning rate when the learning_rate is set to ‘invscaling’.</p></dd>
+
+
+<dt>maxIter</dt>
+<dd><p>(list)  Maximum number of iterations. The solver iterates until convergence (determined by ‘tol’) or this number of iterations. For stochastic solvers (‘sgd’, ‘adam’), note that this determines the number of epochs (how many times each data point will be used), not the number of gradient steps.</p></dd>
+
+
+<dt>shuffle</dt>
+<dd><p>(list) boolean: Whether to shuffle samples in each iteration. Only used when solver=’sgd’ or ‘adam’.</p></dd>
+
+
+<dt>tol</dt>
+<dd><p>(list) Tolerance for the optimization. When the loss or score is not improving by at least tol for nIterNoChange consecutive iterations, unless learning_rate is set to ‘adaptive’, convergence is considered to be reached and training stops.</p></dd>
+
+
+<dt>warmStart</dt>
+<dd><p>(list) When set to True, reuse the solution of the previous call to fit as initialization, otherwise, just erase the previous solution.</p></dd>
+
+
+<dt>momentum</dt>
+<dd><p>(list) Momentum for gradient descent update. Should be between 0 and 1. Only used when solver=’sgd’.</p></dd>
+
+
+<dt>nesterovsMomentum</dt>
+<dd><p>(list) Whether to use Nesterov’s momentum. Only used when solver=’sgd’ and momentum &gt; 0.</p></dd>
+
+
+<dt>earlyStopping</dt>
+<dd><p>(list) boolean Whether to use early stopping to terminate training when validation score is not improving. If set to true, it will automatically set aside 10 percent of training data as validation and terminate training when validation score is not improving by at least tol for n_iter_no_change consecutive epochs.</p></dd>
+
+
+<dt>validationFraction</dt>
+<dd><p>(list) The proportion of training data to set aside as validation set for early stopping. Must be between 0 and 1. Only used if earlyStopping is True.</p></dd>
+
+
+<dt>beta1</dt>
+<dd><p>(list) Exponential decay rate for estimates of first moment vector in adam, should be in 0 to 1.</p></dd>
+
+
+<dt>beta2</dt>
+<dd><p>(list) Exponential decay rate for estimates of second moment vector in adam, should be in 0 to 1.</p></dd>
+
+
+<dt>epsilon</dt>
+<dd><p>(list) Value for numerical stability in adam.</p></dd>
+
+
+<dt>nIterNoChange</dt>
+<dd><p>(list) Maximum number of epochs to not meet tol improvement. Only effective when solver=’sgd’ or ‘adam’.</p></dd>
+
+
+<dt>seed</dt>
+<dd><p>A seed for the model</p></dd>
+
+</dl></div>
+
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="kw">if</span> <span class="op">(</span><span class="cn">FALSE</span><span class="op">)</span> <span class="op">{</span></span></span>
+<span class="r-in"><span><span class="va">model.mlp</span> <span class="op">&lt;-</span> <span class="fu">setMLP</span><span class="op">(</span><span class="op">)</span></span></span>
+<span class="r-in"><span><span class="op">}</span></span></span>
+</code></pre></div>
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/setMLPTorch.html b/docs/reference/setMLPTorch.html
deleted file mode 100644
index e12be4bc0..000000000
--- a/docs/reference/setMLPTorch.html
+++ /dev/null
@@ -1,263 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for neural network model with python — setMLPTorch • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for neural network model with python — setMLPTorch" />
-<meta property="og:description" content="Create setting for neural network model with python" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.3.10</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Create setting for neural network model with python</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/MLPTorch.R'><code>R/MLPTorch.R</code></a></small>
-    <div class="hidden name"><code>setMLPTorch.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Create setting for neural network model with python</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>setMLPTorch</span><span class='op'>(</span>
-  size <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>500</span>, <span class='fl'>1000</span><span class='op'>)</span>,
-  w_decay <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>5e-04</span>, <span class='fl'>0.005</span><span class='op'>)</span>,
-  epochs <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>20</span>, <span class='fl'>50</span><span class='op'>)</span>,
-  seed <span class='op'>=</span> <span class='fl'>0</span>,
-  class_weight <span class='op'>=</span> <span class='fl'>0</span>,
-  mlp_type <span class='op'>=</span> <span class='st'>"MLP"</span>,
-  autoencoder <span class='op'>=</span> <span class='cn'>FALSE</span>,
-  vae <span class='op'>=</span> <span class='cn'>FALSE</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>size</th>
-      <td><p>The number of hidden nodes</p></td>
-    </tr>
-    <tr>
-      <th>w_decay</th>
-      <td><p>The l2 regularisation</p></td>
-    </tr>
-    <tr>
-      <th>epochs</th>
-      <td><p>The number of epochs</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>A seed for the model</p></td>
-    </tr>
-    <tr>
-      <th>class_weight</th>
-      <td><p>The class weight used for imbalanced data: 
- 0: Inverse ratio between positives and negatives
--1: Focal loss</p></td>
-    </tr>
-    <tr>
-      <th>mlp_type</th>
-      <td><p>The type of multiple layer network, inlcuding MLP and SNN (self-normalizing neural network)</p></td>
-    </tr>
-    <tr>
-      <th>autoencoder</th>
-      <td><p>First learn stakced autoencoder for input features, then train MLP on the encoded features.</p></td>
-    </tr>
-    <tr>
-      <th>vae</th>
-      <td><p>First learn stakced varational autoencoder for input features, then train MLP on the encoded features.</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>model.mlpTorch</span> <span class='op'>&lt;-</span> <span class='fu'>setMLPTorch</span><span class='op'>(</span><span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/setNaiveBayes.html b/docs/reference/setNaiveBayes.html
index 51cb57879..a0389c1d8 100644
--- a/docs/reference/setNaiveBayes.html
+++ b/docs/reference/setNaiveBayes.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for naive bayes model with python — setNaiveBayes • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for naive bayes model with python — setNaiveBayes" />
-<meta property="og:description" content="Create setting for naive bayes model with python" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create setting for naive bayes model with python — setNaiveBayes • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create setting for naive bayes model with python — setNaiveBayes"><meta property="og:description" content="Create setting for naive bayes model with python"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create setting for naive bayes model with python</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SklearnClassifierSettings.R'><code>R/SklearnClassifierSettings.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SklearnClassifierSettings.R" class="external-link"><code>R/SklearnClassifierSettings.R</code></a></small>
     <div class="hidden name"><code>setNaiveBayes.Rd</code></div>
     </div>
 
@@ -183,40 +131,39 @@ <h1>Create setting for naive bayes model with python</h1>
     <p>Create setting for naive bayes model with python</p>
     </div>
 
-    <pre class="usage"><span class='fu'>setNaiveBayes</span><span class='op'>(</span><span class='op'>)</span></pre>
-
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">setNaiveBayes</span><span class="op">(</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
 
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>model.nb</span> <span class='op'>&lt;-</span> <span class='fu'>setNaiveBayes</span><span class='op'>(</span><span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="kw">if</span> <span class="op">(</span><span class="cn">FALSE</span><span class="op">)</span> <span class="op">{</span></span></span>
+<span class="r-in"><span><span class="va">model.nb</span> <span class="op">&lt;-</span> <span class="fu">setNaiveBayes</span><span class="op">(</span><span class="op">)</span></span></span>
+<span class="r-in"><span><span class="op">}</span></span></span>
+</code></pre></div>
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/setPythonEnvironment.html b/docs/reference/setPythonEnvironment.html
index 3d8362dca..ce517c7b7 100644
--- a/docs/reference/setPythonEnvironment.html
+++ b/docs/reference/setPythonEnvironment.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Use the virtual environment created using configurePython() — setPythonEnvironment • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Use the virtual environment created using configurePython() — setPythonEnvironment" />
-<meta property="og:description" content="Use the virtual environment created using configurePython()" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Use the virtual environment created using configurePython() — setPythonEnvironment • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Use the virtual environment created using configurePython() — setPythonEnvironment"><meta property="og:description" content="Use the virtual environment created using configurePython()"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Use the virtual environment created using configurePython()</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/HelperFunctions.R'><code>R/HelperFunctions.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/HelperFunctions.R" class="external-link"><code>R/HelperFunctions.R</code></a></small>
     <div class="hidden name"><code>setPythonEnvironment.Rd</code></div>
     </div>
 
@@ -183,50 +131,46 @@ <h1>Use the virtual environment created using configurePython()</h1>
     <p>Use the virtual environment created using configurePython()</p>
     </div>
 
-    <pre class="usage"><span class='fu'>setPythonEnvironment</span><span class='op'>(</span>envname <span class='op'>=</span> <span class='st'>"PLP"</span>, envtype <span class='op'>=</span> <span class='cn'>NULL</span><span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">setPythonEnvironment</span><span class="op">(</span>envname <span class="op">=</span> <span class="st">"PLP"</span>, envtype <span class="op">=</span> <span class="cn">NULL</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>envname</dt>
+<dd><p>A string for the name of the virtual environment (default is 'PLP')</p></dd>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>envname</th>
-      <td><p>A string for the name of the virtual environment (default is 'PLP')</p></td>
-    </tr>
-    <tr>
-      <th>envtype</th>
-      <td><p>An option for specifying the environment as'conda' or 'python'.  If NULL then the default is 'conda' for windows users and 'python' for non-windows users</p></td>
-    </tr>
-    </table>
 
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<dt>envtype</dt>
+<dd><p>An option for specifying the environment as'conda' or 'python'.  If NULL then the default is 'conda' for windows users and 'python' for non-windows users</p></dd>
 
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>This function sets PatientLevelPrediction to use a virtual environment</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/setRNNTorch.html b/docs/reference/setRNNTorch.html
deleted file mode 100644
index 96663666f..000000000
--- a/docs/reference/setRNNTorch.html
+++ /dev/null
@@ -1,248 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for RNN model with python — setRNNTorch • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for RNN model with python — setRNNTorch" />
-<meta property="og:description" content="Create setting for RNN model with python" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.3.10</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Create setting for RNN model with python</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/RNNTorch.R'><code>R/RNNTorch.R</code></a></small>
-    <div class="hidden name"><code>setRNNTorch.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Create setting for RNN model with python</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>setRNNTorch</span><span class='op'>(</span>
-  hidden_size <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>50</span>, <span class='fl'>100</span><span class='op'>)</span>,
-  epochs <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>20</span>, <span class='fl'>50</span><span class='op'>)</span>,
-  seed <span class='op'>=</span> <span class='fl'>0</span>,
-  class_weight <span class='op'>=</span> <span class='fl'>0</span>,
-  type <span class='op'>=</span> <span class='st'>"RNN"</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>hidden_size</th>
-      <td><p>The hidden size</p></td>
-    </tr>
-    <tr>
-      <th>epochs</th>
-      <td><p>The number of epochs</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>A seed for the model</p></td>
-    </tr>
-    <tr>
-      <th>class_weight</th>
-      <td><p>The class weight used for imbalanced data: 
- 0: Inverse ratio between positives and negatives
--1: Focal loss</p></td>
-    </tr>
-    <tr>
-      <th>type</th>
-      <td><p>It can be normal 'RNN', 'BiRNN' (bidirectional RNN) and 'GRU'</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>model.rnnTorch</span> <span class='op'>&lt;-</span> <span class='fu'>setRNNTorch</span><span class='op'>(</span><span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/setRNNTorch2.html b/docs/reference/setRNNTorch2.html
deleted file mode 100644
index 8193cd04e..000000000
--- a/docs/reference/setRNNTorch2.html
+++ /dev/null
@@ -1,226 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for RNN model with python — setRNNTorch2 • PatientLevelPrediction</title>
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script>
-
-<!-- sticky kit -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-<meta property="og:title" content="Create setting for RNN model with python — setRNNTorch2" />
-
-<meta property="og:description" content="Create setting for RNN model with python" />
-<meta name="twitter:card" content="summary" />
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-  </head>
-
-  <body>
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.5</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Create setting for RNN model with python</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/RNNTorch2.R'><code>R/RNNTorch2.R</code></a></small>
-    <div class="hidden name"><code>setRNNTorch2.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    
-    <p>Create setting for RNN model with python</p>
-    
-    </div>
-
-    <pre class="usage"><span class='fu'>setRNNTorch2</span>(<span class='kw'>hidden_size</span> <span class='kw'>=</span> <span class='fu'><a href='https://www.rdocumentation.org/packages/base/topics/c'>c</a></span>(<span class='fl'>50</span>, <span class='fl'>100</span>), <span class='kw'>epochs</span> <span class='kw'>=</span> <span class='fu'><a href='https://www.rdocumentation.org/packages/base/topics/c'>c</a></span>(<span class='fl'>20</span>, <span class='fl'>50</span>), <span class='kw'>seed</span> <span class='kw'>=</span> <span class='fl'>0</span>,
-  <span class='kw'>class_weight</span> <span class='kw'>=</span> <span class='fl'>0</span>, <span class='kw'>type</span> <span class='kw'>=</span> <span class='st'>"RNN"</span>)</pre>
-    
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>hidden_size</th>
-      <td><p>The hidden size</p></td>
-    </tr>
-    <tr>
-      <th>epochs</th>
-      <td><p>The number of epochs</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>A seed for the model</p></td>
-    </tr>
-    <tr>
-      <th>class_weight</th>
-      <td><p>The class weight used for imbalanced data: 
- 0: Inverse ratio between positives and negatives
--1: Focal loss</p></td>
-    </tr>
-    <tr>
-      <th>type</th>
-      <td><p>It can be normal 'RNN', 'BiRNN' (bidirectional RNN) and 'GRU'</p></td>
-    </tr>
-    </table>
-    
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><span class='co'># NOT RUN {</span>
-<span class='no'>model.rnnTorch</span> <span class='kw'>&lt;-</span> <span class='fu'><a href='setRNNTorch.html'>setRNNTorch</a></span>()
-<span class='co'># }</span></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-    <h2>Contents</h2>
-    <ul class="nav nav-pills nav-stacked">
-      <li><a href="#arguments">Arguments</a></li>
-            
-      <li><a href="#examples">Examples</a></li>
-    </ul>
-
-  </div>
-</div>
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-   </div>
-
-  
-
-  </body>
-</html>
-
diff --git a/docs/reference/setRandomForest.html b/docs/reference/setRandomForest.html
index d45605d22..1b44fefc4 100644
--- a/docs/reference/setRandomForest.html
+++ b/docs/reference/setRandomForest.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for random forest model with python (very fast) — setRandomForest • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for random forest model with python (very fast) — setRandomForest" />
-<meta property="og:description" content="Create setting for random forest model with python (very fast)" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create setting for random forest model with python (very fast) — setRandomForest • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create setting for random forest model with python (very fast) — setRandomForest"><meta property="og:description" content="Create setting for random forest model with python (very fast)"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create setting for random forest model with python (very fast)</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SklearnClassifierSettings.R'><code>R/SklearnClassifierSettings.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SklearnClassifierSettings.R" class="external-link"><code>R/SklearnClassifierSettings.R</code></a></small>
     <div class="hidden name"><code>setRandomForest.Rd</code></div>
     </div>
 
@@ -183,128 +131,123 @@ <h1>Create setting for random forest model with python (very fast)</h1>
     <p>Create setting for random forest model with python (very fast)</p>
     </div>
 
-    <pre class="usage"><span class='fu'>setRandomForest</span><span class='op'>(</span>
-  ntrees <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>100</span>, <span class='fl'>500</span><span class='op'>)</span>,
-  criterion <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='st'>"gini"</span><span class='op'>)</span>,
-  maxDepth <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>4</span>, <span class='fl'>10</span>, <span class='fl'>17</span><span class='op'>)</span>,
-  minSamplesSplit <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>2</span>, <span class='fl'>5</span><span class='op'>)</span>,
-  minSamplesLeaf <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>1</span>, <span class='fl'>10</span><span class='op'>)</span>,
-  minWeightFractionLeaf <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>0</span><span class='op'>)</span>,
-  mtries <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='st'>"auto"</span>, <span class='st'>"log2"</span><span class='op'>)</span>,
-  maxLeafNodes <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='cn'>NULL</span><span class='op'>)</span>,
-  minImpurityDecrease <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>0</span><span class='op'>)</span>,
-  bootstrap <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='cn'>TRUE</span><span class='op'>)</span>,
-  maxSamples <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='cn'>NULL</span>, <span class='fl'>0.9</span><span class='op'>)</span>,
-  oobScore <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span>,
-  nJobs <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='cn'>NULL</span><span class='op'>)</span>,
-  classWeight <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='st'>"balanced_subsample"</span>, <span class='cn'>NULL</span><span class='op'>)</span>,
-  seed <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/sample.html'>sample</a></span><span class='op'>(</span><span class='fl'>1e+05</span>, <span class='fl'>1</span><span class='op'>)</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>ntrees</th>
-      <td><p>(list) The number of trees to build</p></td>
-    </tr>
-    <tr>
-      <th>criterion</th>
-      <td><p>(list) The function to measure the quality of a split. Supported criteria are “gini” for the Gini impurity and “entropy” for the information gain. Note: this parameter is tree-specific.</p></td>
-    </tr>
-    <tr>
-      <th>maxDepth</th>
-      <td><p>(list) The maximum depth of the tree. If NULL, then nodes are expanded until all leaves are pure or until all leaves contain less than minSamplesSplit samples.</p></td>
-    </tr>
-    <tr>
-      <th>minSamplesSplit</th>
-      <td><p>(list) The minimum number of samples required to split an internal node</p></td>
-    </tr>
-    <tr>
-      <th>minSamplesLeaf</th>
-      <td><p>(list) The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least minSamplesLeaf training samples in each of the left and right branches. This may have the effect of smoothing the model, especially in regression.</p></td>
-    </tr>
-    <tr>
-      <th>minWeightFractionLeaf</th>
-      <td><p>(list) The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sampleWeight is not provided.</p></td>
-    </tr>
-    <tr>
-      <th>mtries</th>
-      <td><p>(list) The number of features to consider when looking for the best split:</p><ul>
-<li><p>intthen consider max_features features at each split.</p></li>
-<li><p>floatthen max_features is a fraction and round(max_features * n_features) features are considered at each split</p></li>
-<li><p>'auto'then max_features=sqrt(n_features)</p></li>
-<li><p>'sqrt'then max_features=sqrt(n_features) (same as “auto”)</p></li>
-<li><p>'log2'then max_features=log2(n_features).</p></li>
-<li><p>NULLthen max_features=n_features</p></li>
-</ul></td>
-    </tr>
-    <tr>
-      <th>maxLeafNodes</th>
-      <td><p>(list) Grow trees with max_leaf_nodes in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes.</p></td>
-    </tr>
-    <tr>
-      <th>minImpurityDecrease</th>
-      <td><p>(list) A node will be split if this split induces a decrease of the impurity greater than or equal to this value.</p></td>
-    </tr>
-    <tr>
-      <th>bootstrap</th>
-      <td><p>(list) Whether bootstrap samples are used when building trees. If False, the whole dataset is used to build each tree.</p></td>
-    </tr>
-    <tr>
-      <th>maxSamples</th>
-      <td><p>(list) If bootstrap is True, the number of samples to draw from X to train each base estimator.</p></td>
-    </tr>
-    <tr>
-      <th>oobScore</th>
-      <td><p>(list) Whether to use out-of-bag samples to estimate the generalization score. Only available if bootstrap=True.</p></td>
-    </tr>
-    <tr>
-      <th>nJobs</th>
-      <td><p>The number of jobs to run in parallel.</p></td>
-    </tr>
-    <tr>
-      <th>classWeight</th>
-      <td><p>(list) Weights associated with classes. If not given, all classes are supposed to have weight one. NULL, “balanced”, “balanced_subsample”</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>A seed when training the final model</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>model.rf</span> <span class='op'>&lt;-</span> <span class='fu'>setRandomForest</span><span class='op'>(</span>mtries<span class='op'>=</span><span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='st'>'auto'</span>,<span class='fl'>5</span>,<span class='fl'>20</span><span class='op'>)</span>,  ntrees<span class='op'>=</span><span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>10</span>,<span class='fl'>100</span><span class='op'>)</span>, 
-                           maxDepth<span class='op'>=</span><span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>5</span>,<span class='fl'>20</span><span class='op'>)</span><span class='op'>)</span>
-<span class='op'>}</span>                           
-</div></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">setRandomForest</span><span class="op">(</span></span>
+<span>  ntrees <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">100</span>, <span class="fl">500</span><span class="op">)</span>,</span>
+<span>  criterion <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="st">"gini"</span><span class="op">)</span>,</span>
+<span>  maxDepth <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">4</span>, <span class="fl">10</span>, <span class="fl">17</span><span class="op">)</span>,</span>
+<span>  minSamplesSplit <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">2</span>, <span class="fl">5</span><span class="op">)</span>,</span>
+<span>  minSamplesLeaf <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">1</span>, <span class="fl">10</span><span class="op">)</span>,</span>
+<span>  minWeightFractionLeaf <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">0</span><span class="op">)</span>,</span>
+<span>  mtries <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="st">"sqrt"</span>, <span class="st">"log2"</span><span class="op">)</span>,</span>
+<span>  maxLeafNodes <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="cn">NULL</span><span class="op">)</span>,</span>
+<span>  minImpurityDecrease <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">0</span><span class="op">)</span>,</span>
+<span>  bootstrap <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="cn">TRUE</span><span class="op">)</span>,</span>
+<span>  maxSamples <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="cn">NULL</span>, <span class="fl">0.9</span><span class="op">)</span>,</span>
+<span>  oobScore <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="cn">FALSE</span><span class="op">)</span>,</span>
+<span>  nJobs <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="cn">NULL</span><span class="op">)</span>,</span>
+<span>  classWeight <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="cn">NULL</span><span class="op">)</span>,</span>
+<span>  seed <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/sample.html" class="external-link">sample</a></span><span class="op">(</span><span class="fl">1e+05</span>, <span class="fl">1</span><span class="op">)</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>ntrees</dt>
+<dd><p>(list) The number of trees to build</p></dd>
+
+
+<dt>criterion</dt>
+<dd><p>(list) The function to measure the quality of a split. Supported criteria are “gini” for the Gini impurity and “entropy” for the information gain. Note: this parameter is tree-specific.</p></dd>
+
+
+<dt>maxDepth</dt>
+<dd><p>(list) The maximum depth of the tree. If NULL, then nodes are expanded until all leaves are pure or until all leaves contain less than minSamplesSplit samples.</p></dd>
+
+
+<dt>minSamplesSplit</dt>
+<dd><p>(list) The minimum number of samples required to split an internal node</p></dd>
+
+
+<dt>minSamplesLeaf</dt>
+<dd><p>(list) The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least minSamplesLeaf training samples in each of the left and right branches. This may have the effect of smoothing the model, especially in regression.</p></dd>
+
+
+<dt>minWeightFractionLeaf</dt>
+<dd><p>(list) The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sampleWeight is not provided.</p></dd>
+
+
+<dt>mtries</dt>
+<dd><p>(list) The number of features to consider when looking for the best split:</p><ul><li><p>int then consider max_features features at each split.</p></li>
+<li><p>float then max_features is a fraction and round(max_features * n_features) features are considered at each split</p></li>
+<li><p>'sqrt' then max_features=sqrt(n_features)</p></li>
+<li><p>'log2' then max_features=log2(n_features)</p></li>
+<li><p>NULL then max_features=n_features</p></li>
+</ul></dd>
+
+
+<dt>maxLeafNodes</dt>
+<dd><p>(list) Grow trees with max_leaf_nodes in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes.</p></dd>
+
+
+<dt>minImpurityDecrease</dt>
+<dd><p>(list) A node will be split if this split induces a decrease of the impurity greater than or equal to this value.</p></dd>
+
+
+<dt>bootstrap</dt>
+<dd><p>(list) Whether bootstrap samples are used when building trees. If False, the whole dataset is used to build each tree.</p></dd>
+
+
+<dt>maxSamples</dt>
+<dd><p>(list) If bootstrap is True, the number of samples to draw from X to train each base estimator.</p></dd>
+
+
+<dt>oobScore</dt>
+<dd><p>(list) Whether to use out-of-bag samples to estimate the generalization score. Only available if bootstrap=True.</p></dd>
+
+
+<dt>nJobs</dt>
+<dd><p>The number of jobs to run in parallel.</p></dd>
+
+
+<dt>classWeight</dt>
+<dd><p>(list) Weights associated with classes. If not given, all classes are supposed to have weight one. NULL, “balanced”, “balanced_subsample”</p></dd>
+
+
+<dt>seed</dt>
+<dd><p>A seed when training the final model</p></dd>
+
+</dl></div>
+
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="kw">if</span> <span class="op">(</span><span class="cn">FALSE</span><span class="op">)</span> <span class="op">{</span></span></span>
+<span class="r-in"><span><span class="va">model.rf</span> <span class="op">&lt;-</span> <span class="fu">setRandomForest</span><span class="op">(</span>mtries<span class="op">=</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="st">'auto'</span>,<span class="fl">5</span>,<span class="fl">20</span><span class="op">)</span>,  ntrees<span class="op">=</span><span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">10</span>,<span class="fl">100</span><span class="op">)</span>,</span></span>
+<span class="r-in"><span>                           maxDepth<span class="op">=</span><span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">5</span>,<span class="fl">20</span><span class="op">)</span><span class="op">)</span></span></span>
+<span class="r-in"><span><span class="op">}</span></span></span>
+</code></pre></div>
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/setRandomForestQuantileRegressor.html b/docs/reference/setRandomForestQuantileRegressor.html
deleted file mode 100644
index c01acfc7c..000000000
--- a/docs/reference/setRandomForestQuantileRegressor.html
+++ /dev/null
@@ -1,300 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for RandomForestQuantileRegressor with python scikit-garden (skgarden.quantile.RandomForestQuantileRegressor)
-#' @description
-This creates a setting for fitting a RandomForestQuantileRegressor model.  You need skgarden python install.  To install this open your command line and type: conda install -c conda-forge scikit-garden — setRandomForestQuantileRegressor • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for RandomForestQuantileRegressor with python scikit-garden (skgarden.quantile.RandomForestQuantileRegressor)
-#' @description
-This creates a setting for fitting a RandomForestQuantileRegressor model.  You need skgarden python install.  To install this open your command line and type: conda install -c conda-forge scikit-garden — setRandomForestQuantileRegressor" />
-<meta property="og:description" content="Create setting for RandomForestQuantileRegressor with python scikit-garden (skgarden.quantile.RandomForestQuantileRegressor)
-#' @description
-This creates a setting for fitting a RandomForestQuantileRegressor model.  You need skgarden python install.  To install this open your command line and type: conda install -c conda-forge scikit-garden" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.3.10</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Create setting for RandomForestQuantileRegressor with python scikit-garden (skgarden.quantile.RandomForestQuantileRegressor)
-#' @description
-This creates a setting for fitting a RandomForestQuantileRegressor model.  You need skgarden python install.  To install this open your command line and type: conda install -c conda-forge scikit-garden</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/RandomForestQuantileRegressor.R'><code>R/RandomForestQuantileRegressor.R</code></a></small>
-    <div class="hidden name"><code>setRandomForestQuantileRegressor.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Create setting for RandomForestQuantileRegressor with python scikit-garden (skgarden.quantile.RandomForestQuantileRegressor)
-#' @description
-This creates a setting for fitting a RandomForestQuantileRegressor model.  You need skgarden python install.  To install this open your command line and type: conda install -c conda-forge scikit-garden</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>setRandomForestQuantileRegressor</span><span class='op'>(</span>
-  nEstimators <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>100</span><span class='op'>)</span>,
-  criterion <span class='op'>=</span> <span class='st'>"mse"</span>,
-  maxFeatures <span class='op'>=</span> <span class='op'>-</span><span class='fl'>1</span>,
-  maxDepth <span class='op'>=</span> <span class='fl'>4</span>,
-  minSamplesSplit <span class='op'>=</span> <span class='fl'>2</span>,
-  minSamplesLeaf <span class='op'>=</span> <span class='fl'>1</span>,
-  minWeightFractionLeaf <span class='op'>=</span> <span class='fl'>0</span>,
-  maxLeafNodes <span class='op'>=</span> <span class='cn'>NULL</span>,
-  bootstrap <span class='op'>=</span> <span class='cn'>TRUE</span>,
-  oobScore <span class='op'>=</span> <span class='cn'>FALSE</span>,
-  warmStart <span class='op'>=</span> <span class='cn'>FALSE</span>,
-  seed <span class='op'>=</span> <span class='cn'>NULL</span>,
-  quiet <span class='op'>=</span> <span class='cn'>F</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>nEstimators</th>
-      <td><p>(int default:100) The number of trees in the forest.</p></td>
-    </tr>
-    <tr>
-      <th>criterion</th>
-      <td><p>(string default="mse")) The function to measure the quality of a split. Supported criteria are "mse" for the mean squared error, which is equal to variance reduction as feature selection criterion, and "mae" for the mean absolute error.</p></td>
-    </tr>
-    <tr>
-      <th>maxFeatures</th>
-      <td><p>(int default: -1) The number of features to consider when looking for the best split.  If -1 then use sqrt of total number of features.</p></td>
-    </tr>
-    <tr>
-      <th>maxDepth</th>
-      <td><p>(int default:4) The maximum depth of the tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than minSamplesSplit samples.</p></td>
-    </tr>
-    <tr>
-      <th>minSamplesSplit</th>
-      <td><p>An integer specifying min samples per tree split (complexity)</p></td>
-    </tr>
-    <tr>
-      <th>minSamplesLeaf</th>
-      <td><p>An integer specifying min samples per leaf (complexity)</p></td>
-    </tr>
-    <tr>
-      <th>minWeightFractionLeaf</th>
-      <td><p>Lookup</p></td>
-    </tr>
-    <tr>
-      <th>maxLeafNodes</th>
-      <td><p>(int) Grow trees with maxLeafNodes in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes.</p></td>
-    </tr>
-    <tr>
-      <th>bootstrap</th>
-      <td><p>(boolean default:TRUE) Whether bootstrap samples are used when building trees.</p></td>
-    </tr>
-    <tr>
-      <th>oobScore</th>
-      <td><p>(boolean default:FALSE) Whether to use out-of-bag samples to estimate the R^2 on unseen data.</p></td>
-    </tr>
-    <tr>
-      <th>warmStart</th>
-      <td><p>(boolean default:FALSE) When set to True, reuse the solution of the previous call to fit and add more estimators to the ensemble, otherwise, just fit a whole new forest.</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>will add</p></td>
-    </tr>
-    <tr>
-      <th>quiet</th>
-      <td><p>will add</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Pick the hyper-parameters you want to do a grid search for</p>
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>rfQR</span> <span class='op'>&lt;-</span> <span class='fu'>setRandomForestQuantileRegressor</span><span class='op'>(</span>nEstimators <span class='op'>=</span><span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>10</span>,<span class='fl'>50</span>,<span class='fl'>100</span><span class='op'>)</span>,
- maxDepth<span class='op'>=</span><span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>4</span>,<span class='fl'>10</span>,<span class='fl'>17</span><span class='op'>)</span>, seed <span class='op'>=</span> <span class='fl'>2</span><span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/setSVM.html b/docs/reference/setSVM.html
index 7c2c3a575..c8ca9a9e7 100644
--- a/docs/reference/setSVM.html
+++ b/docs/reference/setSVM.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for the python sklearn SVM (SVC function) — setSVM • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for the python sklearn SVM (SVC function) — setSVM" />
-<meta property="og:description" content="Create setting for the python sklearn SVM (SVC function)" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Create setting for the python sklearn SVM (SVC function) — setSVM • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Create setting for the python sklearn SVM (SVC function) — setSVM"><meta property="og:description" content="Create setting for the python sklearn SVM (SVC function)"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Create setting for the python sklearn SVM (SVC function)</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SklearnClassifierSettings.R'><code>R/SklearnClassifierSettings.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SklearnClassifierSettings.R" class="external-link"><code>R/SklearnClassifierSettings.R</code></a></small>
     <div class="hidden name"><code>setSVM.Rd</code></div>
     </div>
 
@@ -183,95 +131,92 @@ <h1>Create setting for the python sklearn SVM (SVC function)</h1>
     <p>Create setting for the python sklearn SVM (SVC function)</p>
     </div>
 
-    <pre class="usage"><span class='fu'>setSVM</span><span class='op'>(</span>
-  C <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>1</span>, <span class='fl'>0.9</span>, <span class='fl'>2</span>, <span class='fl'>0.1</span><span class='op'>)</span>,
-  kernel <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='st'>"rbf"</span><span class='op'>)</span>,
-  degree <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>1</span>, <span class='fl'>3</span>, <span class='fl'>5</span><span class='op'>)</span>,
-  gamma <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='st'>"scale"</span>, <span class='fl'>1e-04</span>, <span class='fl'>3e-05</span>, <span class='fl'>0.001</span>, <span class='fl'>0.01</span>, <span class='fl'>0.25</span><span class='op'>)</span>,
-  coef0 <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>0</span><span class='op'>)</span>,
-  shrinking <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='cn'>TRUE</span><span class='op'>)</span>,
-  tol <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='fl'>0.001</span><span class='op'>)</span>,
-  classWeight <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/list.html'>list</a></span><span class='op'>(</span><span class='st'>"balanced"</span>, <span class='cn'>NULL</span><span class='op'>)</span>,
-  cacheSize <span class='op'>=</span> <span class='fl'>500</span>,
-  seed <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/sample.html'>sample</a></span><span class='op'>(</span><span class='fl'>1e+05</span>, <span class='fl'>1</span><span class='op'>)</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>C</th>
-      <td><p>(list) Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive. The penalty is a squared l2 penalty.</p></td>
-    </tr>
-    <tr>
-      <th>kernel</th>
-      <td><p>(list) Specifies the kernel type to be used in the algorithm. one of ‘linear’, ‘poly’, ‘rbf’, ‘sigmoid’, ‘precomputed’. If none is given ‘rbf’ will be used.</p></td>
-    </tr>
-    <tr>
-      <th>degree</th>
-      <td><p>(list) degree of kernel function is significant only in poly, rbf, sigmoid</p></td>
-    </tr>
-    <tr>
-      <th>gamma</th>
-      <td><p>(list) kernel coefficient for rbf and poly, by default 1/n_features will be taken. ‘scale’, ‘auto’ or float, default=’scale’</p></td>
-    </tr>
-    <tr>
-      <th>coef0</th>
-      <td><p>(list) independent term in kernel function. It is only significant in poly/sigmoid.</p></td>
-    </tr>
-    <tr>
-      <th>shrinking</th>
-      <td><p>(list) whether to use the shrinking heuristic.</p></td>
-    </tr>
-    <tr>
-      <th>tol</th>
-      <td><p>(list) Tolerance for stopping criterion.</p></td>
-    </tr>
-    <tr>
-      <th>classWeight</th>
-      <td><p>(list) Class weight based on imbalance either 'balanced' or NULL</p></td>
-    </tr>
-    <tr>
-      <th>cacheSize</th>
-      <td><p>Specify the size of the kernel cache (in MB).</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>A seed for the model</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>model.svm</span> <span class='op'>&lt;-</span> <span class='fu'>setSVM</span><span class='op'>(</span>kernel<span class='op'>=</span><span class='st'>'rbf'</span>, seed <span class='op'>=</span> <span class='cn'>NULL</span><span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">setSVM</span><span class="op">(</span></span>
+<span>  C <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">1</span>, <span class="fl">0.9</span>, <span class="fl">2</span>, <span class="fl">0.1</span><span class="op">)</span>,</span>
+<span>  kernel <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="st">"rbf"</span><span class="op">)</span>,</span>
+<span>  degree <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">1</span>, <span class="fl">3</span>, <span class="fl">5</span><span class="op">)</span>,</span>
+<span>  gamma <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="st">"scale"</span>, <span class="fl">1e-04</span>, <span class="fl">3e-05</span>, <span class="fl">0.001</span>, <span class="fl">0.01</span>, <span class="fl">0.25</span><span class="op">)</span>,</span>
+<span>  coef0 <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">0</span><span class="op">)</span>,</span>
+<span>  shrinking <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="cn">TRUE</span><span class="op">)</span>,</span>
+<span>  tol <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">0.001</span><span class="op">)</span>,</span>
+<span>  classWeight <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="cn">NULL</span><span class="op">)</span>,</span>
+<span>  cacheSize <span class="op">=</span> <span class="fl">500</span>,</span>
+<span>  seed <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/sample.html" class="external-link">sample</a></span><span class="op">(</span><span class="fl">1e+05</span>, <span class="fl">1</span><span class="op">)</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>C</dt>
+<dd><p>(list) Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive. The penalty is a squared l2 penalty.</p></dd>
+
+
+<dt>kernel</dt>
+<dd><p>(list) Specifies the kernel type to be used in the algorithm. one of ‘linear’, ‘poly’, ‘rbf’, ‘sigmoid’, ‘precomputed’. If none is given ‘rbf’ will be used.</p></dd>
+
+
+<dt>degree</dt>
+<dd><p>(list) degree of kernel function is significant only in poly, rbf, sigmoid</p></dd>
+
+
+<dt>gamma</dt>
+<dd><p>(list) kernel coefficient for rbf and poly, by default 1/n_features will be taken. ‘scale’, ‘auto’ or float, default=’scale’</p></dd>
+
+
+<dt>coef0</dt>
+<dd><p>(list) independent term in kernel function. It is only significant in poly/sigmoid.</p></dd>
+
+
+<dt>shrinking</dt>
+<dd><p>(list) whether to use the shrinking heuristic.</p></dd>
+
+
+<dt>tol</dt>
+<dd><p>(list) Tolerance for stopping criterion.</p></dd>
+
+
+<dt>classWeight</dt>
+<dd><p>(list) Class weight based on imbalance either 'balanced' or NULL</p></dd>
+
+
+<dt>cacheSize</dt>
+<dd><p>Specify the size of the kernel cache (in MB).</p></dd>
+
+
+<dt>seed</dt>
+<dd><p>A seed for the model</p></dd>
+
+</dl></div>
+
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="kw">if</span> <span class="op">(</span><span class="cn">FALSE</span><span class="op">)</span> <span class="op">{</span></span></span>
+<span class="r-in"><span><span class="va">model.svm</span> <span class="op">&lt;-</span> <span class="fu">setSVM</span><span class="op">(</span>kernel<span class="op">=</span><span class="st">'rbf'</span>, seed <span class="op">=</span> <span class="cn">NULL</span><span class="op">)</span></span></span>
+<span class="r-in"><span><span class="op">}</span></span></span>
+</code></pre></div>
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/setSagemakerBinary.html b/docs/reference/setSagemakerBinary.html
deleted file mode 100644
index a0d5969de..000000000
--- a/docs/reference/setSagemakerBinary.html
+++ /dev/null
@@ -1,251 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Create setting for sagemaker model — setSagemakerBinary • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Create setting for sagemaker model — setSagemakerBinary" />
-<meta property="og:description" content="Create setting for sagemaker model" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.3.10</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Create setting for sagemaker model</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/SageMakerBinary.R'><code>R/SageMakerBinary.R</code></a></small>
-    <div class="hidden name"><code>setSagemakerBinary.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Create setting for sagemaker model</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>setSagemakerBinary</span><span class='op'>(</span>
-  classifier <span class='op'>=</span> <span class='st'>"xgboost"</span>,
-  <span class='va'>bucket</span>,
-  prefix <span class='op'>=</span> <span class='st'>"data"</span>,
-  <span class='va'>roleArn</span>,
-  otherparams <span class='op'>=</span> <span class='cn'>NULL</span>,
-  seed <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>classifier</th>
-      <td><p>The name of the sagemaker binary classifier to use (pick from: knn, xgboost or linear-learner)</p></td>
-    </tr>
-    <tr>
-      <th>bucket</th>
-      <td><p>The s3 bucker string to save data for model training</p></td>
-    </tr>
-    <tr>
-      <th>prefix</th>
-      <td><p>The s3 subdirectory for the data</p></td>
-    </tr>
-    <tr>
-      <th>roleArn</th>
-      <td><p>The amazon roleArn</p></td>
-    </tr>
-    <tr>
-      <th>otherparams</th>
-      <td><p>Other parameters for training (currently not working)</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>The seed for the training</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>model.sm</span> <span class='op'>&lt;-</span> <span class='fu'>setSagemakerBinary</span><span class='op'>(</span>classifier<span class='op'>=</span><span class='st'>'gxboost'</span>,  bucket<span class='op'>=</span><span class='st'>'ohdsi3'</span><span class='op'>)</span>
-<span class='op'>}</span>                           
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/similarPlpData.html b/docs/reference/similarPlpData.html
deleted file mode 100644
index c7640b066..000000000
--- a/docs/reference/similarPlpData.html
+++ /dev/null
@@ -1,309 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Extract new plpData using plpModel settings
-use metadata in plpModel to extract similar data and population for new databases: — similarPlpData • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Extract new plpData using plpModel settings
-use metadata in plpModel to extract similar data and population for new databases: — similarPlpData" />
-<meta property="og:description" content="Extract new plpData using plpModel settings
-use metadata in plpModel to extract similar data and population for new databases:" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Extract new plpData using plpModel settings
-use metadata in plpModel to extract similar data and population for new databases:</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ApplyPlp.R'><code>R/ApplyPlp.R</code></a></small>
-    <div class="hidden name"><code>similarPlpData.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Extract new plpData using plpModel settings
-use metadata in plpModel to extract similar data and population for new databases:</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>similarPlpData</span><span class='op'>(</span>
-  plpModel <span class='op'>=</span> <span class='cn'>NULL</span>,
-  <span class='va'>newConnectionDetails</span>,
-  newCdmDatabaseSchema <span class='op'>=</span> <span class='cn'>NULL</span>,
-  newCohortDatabaseSchema <span class='op'>=</span> <span class='cn'>NULL</span>,
-  newCohortTable <span class='op'>=</span> <span class='cn'>NULL</span>,
-  newCohortId <span class='op'>=</span> <span class='cn'>NULL</span>,
-  newOutcomeDatabaseSchema <span class='op'>=</span> <span class='cn'>NULL</span>,
-  newOutcomeTable <span class='op'>=</span> <span class='cn'>NULL</span>,
-  newOutcomeId <span class='op'>=</span> <span class='cn'>NULL</span>,
-  newOracleTempSchema <span class='op'>=</span> <span class='va'>newCdmDatabaseSchema</span>,
-  sample <span class='op'>=</span> <span class='cn'>NULL</span>,
-  createPopulation <span class='op'>=</span> <span class='cn'>T</span>,
-  createCohorts <span class='op'>=</span> <span class='cn'>T</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpModel</th>
-      <td><p>The trained PatientLevelPrediction model or object returned by runPlp()</p></td>
-    </tr>
-    <tr>
-      <th>newConnectionDetails</th>
-      <td><p>The connectionDetails for the new database</p></td>
-    </tr>
-    <tr>
-      <th>newCdmDatabaseSchema</th>
-      <td><p>The database schema for the new CDM database</p></td>
-    </tr>
-    <tr>
-      <th>newCohortDatabaseSchema</th>
-      <td><p>The database schema where the cohort table is stored</p></td>
-    </tr>
-    <tr>
-      <th>newCohortTable</th>
-      <td><p>The table name of the cohort table</p></td>
-    </tr>
-    <tr>
-      <th>newCohortId</th>
-      <td><p>The cohort_definition_id for the cohort of at risk people</p></td>
-    </tr>
-    <tr>
-      <th>newOutcomeDatabaseSchema</th>
-      <td><p>The database schema where the outcome table is stored</p></td>
-    </tr>
-    <tr>
-      <th>newOutcomeTable</th>
-      <td><p>The table name of the outcome table</p></td>
-    </tr>
-    <tr>
-      <th>newOutcomeId</th>
-      <td><p>The cohort_definition_id for the outcome</p></td>
-    </tr>
-    <tr>
-      <th>newOracleTempSchema</th>
-      <td><p>The temp coracle schema</p></td>
-    </tr>
-    <tr>
-      <th>sample</th>
-      <td><p>The number of people to sample (default is NULL meaning use all data)</p></td>
-    </tr>
-    <tr>
-      <th>createPopulation</th>
-      <td><p>Whether to create the study population as well</p></td>
-    </tr>
-    <tr>
-      <th>createCohorts</th>
-      <td><p>No longer used</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='co'># set the connection</span>
-<span class='va'>connectionDetails</span> <span class='op'>&lt;-</span> <span class='fu'>DatabaseConnector</span><span class='fu'>::</span><span class='fu'><a href='https://ohdsi.github.io/DatabaseConnector/reference/createConnectionDetails.html'>createConnectionDetails</a></span><span class='op'>(</span><span class='op'>)</span>
-   
-<span class='co'># load the model and data</span>
-<span class='va'>plpModel</span> <span class='op'>&lt;-</span> <span class='fu'><a href='loadPlpModel.html'>loadPlpModel</a></span><span class='op'>(</span><span class='st'>"C:/plpmodel"</span><span class='op'>)</span>
-
-<span class='co'># extract the new data in the 'newData.dbo' schema using the model settings </span>
-<span class='va'>newDataList</span> <span class='op'>&lt;-</span> <span class='fu'>similarPlpData</span><span class='op'>(</span>plpModel<span class='op'>=</span><span class='va'>plpModel</span>, 
-                              newConnectionDetails <span class='op'>=</span> <span class='va'>connectionDetails</span>,
-                              newCdmDatabaseSchema <span class='op'>=</span> <span class='st'>'newData.dbo'</span>,
-                              newCohortDatabaseSchema <span class='op'>=</span> <span class='st'>'newData.dbo'</span>,   
-                              newCohortTable <span class='op'>=</span> <span class='st'>'cohort'</span>, 
-                              newCohortId <span class='op'>=</span> <span class='fl'>1</span>, 
-                              newOutcomeDatabaseSchema <span class='op'>=</span> <span class='st'>'newData.dbo'</span>, 
-                              newOutcomeTable <span class='op'>=</span> <span class='st'>'outcome'</span>,     
-                              newOutcomeId <span class='op'>=</span> <span class='fl'>2</span><span class='op'>)</span>    
-               
-<span class='co'># get the prediction:</span>
-<span class='va'>prediction</span> <span class='op'>&lt;-</span> <span class='fu'><a href='applyModel.html'>applyModel</a></span><span class='op'>(</span><span class='va'>newDataList</span><span class='op'>$</span><span class='va'>population</span>, <span class='va'>newDataList</span><span class='op'>$</span><span class='va'>plpData</span>, <span class='va'>plpModel</span><span class='op'>)</span><span class='op'>$</span><span class='va'>prediction</span>
-<span class='op'>}</span>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/simulatePlpData.html b/docs/reference/simulatePlpData.html
index 505986878..28d042f5a 100644
--- a/docs/reference/simulatePlpData.html
+++ b/docs/reference/simulatePlpData.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Generate simulated data — simulatePlpData • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Generate simulated data — simulatePlpData" />
-<meta property="og:description" content="simulateplpData creates a plpData object with simulated data." />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Generate simulated data — simulatePlpData • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Generate simulated data — simulatePlpData"><meta property="og:description" content="simulateplpData creates a plpData object with simulated data."><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Generate simulated data</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Simulation.R'><code>R/Simulation.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Simulation.R" class="external-link"><code>R/Simulation.R</code></a></small>
     <div class="hidden name"><code>simulatePlpData.Rd</code></div>
     </div>
 
@@ -183,56 +131,55 @@ <h1>Generate simulated data</h1>
     <p><code>simulateplpData</code> creates a plpData object with simulated data.</p>
     </div>
 
-    <pre class="usage"><span class='fu'>simulatePlpData</span><span class='op'>(</span><span class='va'>plpDataSimulationProfile</span>, n <span class='op'>=</span> <span class='fl'>10000</span><span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">simulatePlpData</span><span class="op">(</span><span class="va">plpDataSimulationProfile</span>, n <span class="op">=</span> <span class="fl">10000</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpDataSimulationProfile</th>
-      <td><p>An object of type <code>plpDataSimulationProfile</code> as generated
-using the <br /><code>createplpDataSimulationProfile</code> function.</p></td>
-    </tr>
-    <tr>
-      <th>n</th>
-      <td><p>The size of the population to be generated.</p></td>
-    </tr>
-    </table>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpDataSimulationProfile</dt>
+<dd><p>An object of type <code>plpDataSimulationProfile</code> as generated
+using the <br><code>createplpDataSimulationProfile</code> function.</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
 
-    <p>An object of type <code>plpData</code>.</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<dt>n</dt>
+<dd><p>The size of the population to be generated.</p></dd>
 
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>An object of type <code>plpData</code>.</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>This function generates simulated data that is in many ways similar to the original data on which
 the simulation profile is based. The contains same outcome, comparator, and outcome concept IDs,
 and the covariates and their 1st order statistics should be comparable.</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/sklearnFromJson.html b/docs/reference/sklearnFromJson.html
new file mode 100644
index 000000000..207925bfd
--- /dev/null
+++ b/docs/reference/sklearnFromJson.html
@@ -0,0 +1,168 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Loads sklearn python model from json — sklearnFromJson • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Loads sklearn python model from json — sklearnFromJson"><meta property="og:description" content="Loads sklearn python model from json"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Loads sklearn python model from json</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SklearnToJson.R" class="external-link"><code>R/SklearnToJson.R</code></a></small>
+    <div class="hidden name"><code>sklearnFromJson.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>Loads sklearn python model from json</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">sklearnFromJson</span><span class="op">(</span><span class="va">path</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>path</dt>
+<dd><p>path to the model json file</p></dd>
+
+</dl></div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/sklearnToJson.html b/docs/reference/sklearnToJson.html
new file mode 100644
index 000000000..4f18ac540
--- /dev/null
+++ b/docs/reference/sklearnToJson.html
@@ -0,0 +1,172 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Saves sklearn python model object to json in path — sklearnToJson • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Saves sklearn python model object to json in path — sklearnToJson"><meta property="og:description" content="Saves sklearn python model object to json in path"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>Saves sklearn python model object to json in path</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/SklearnToJson.R" class="external-link"><code>R/SklearnToJson.R</code></a></small>
+    <div class="hidden name"><code>sklearnToJson.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>Saves sklearn python model object to json in path</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">sklearnToJson</span><span class="op">(</span><span class="va">model</span>, <span class="va">path</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>model</dt>
+<dd><p>a fitted sklearn python model object</p></dd>
+
+
+<dt>path</dt>
+<dd><p>path to the saved model file</p></dd>
+
+</dl></div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/specificity.html b/docs/reference/specificity.html
index 6f582daef..e16bf46a6 100644
--- a/docs/reference/specificity.html
+++ b/docs/reference/specificity.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Calculate the specificity — specificity • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Calculate the specificity — specificity" />
-<meta property="og:description" content="Calculate the specificity" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Calculate the specificity — specificity • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Calculate the specificity — specificity"><meta property="og:description" content="Calculate the specificity"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Calculate the specificity</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ThresholdSummary.R'><code>R/ThresholdSummary.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ThresholdSummary.R" class="external-link"><code>R/ThresholdSummary.R</code></a></small>
     <div class="hidden name"><code>specificity.Rd</code></div>
     </div>
 
@@ -183,61 +131,60 @@ <h1>Calculate the specificity</h1>
     <p>Calculate the specificity</p>
     </div>
 
-    <pre class="usage"><span class='fu'>specificity</span><span class='op'>(</span><span class='va'>TP</span>, <span class='va'>TN</span>, <span class='va'>FN</span>, <span class='va'>FP</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>TP</th>
-      <td><p>Number of true positives</p></td>
-    </tr>
-    <tr>
-      <th>TN</th>
-      <td><p>Number of true negatives</p></td>
-    </tr>
-    <tr>
-      <th>FN</th>
-      <td><p>Number of false negatives</p></td>
-    </tr>
-    <tr>
-      <th>FP</th>
-      <td><p>Number of false positives</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>specificity value</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">specificity</span><span class="op">(</span><span class="va">TP</span>, <span class="va">TN</span>, <span class="va">FN</span>, <span class="va">FP</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>TP</dt>
+<dd><p>Number of true positives</p></dd>
+
+
+<dt>TN</dt>
+<dd><p>Number of true negatives</p></dd>
+
+
+<dt>FN</dt>
+<dd><p>Number of false negatives</p></dd>
+
+
+<dt>FP</dt>
+<dd><p>Number of false positives</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>specificity value</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Calculate the specificity</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/splitData.html b/docs/reference/splitData.html
index f4a91d37d..1b6baefdf 100644
--- a/docs/reference/splitData.html
+++ b/docs/reference/splitData.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Split the plpData into test/train sets using a splitting settings of class splitSettings — splitData • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Split the plpData into test/train sets using a splitting settings of class splitSettings — splitData" />
-<meta property="og:description" content="Split the plpData into test/train sets using a splitting settings of class splitSettings" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Split the plpData into test/train sets using a splitting settings of class splitSettings — splitData • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Split the plpData into test/train sets using a splitting settings of class splitSettings — splitData"><meta property="og:description" content="Split the plpData into test/train sets using a splitting settings of class splitSettings"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Split the plpData into test/train sets using a splitting settings of class <code>splitSettings</code></h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/DataSplitting.R'><code>R/DataSplitting.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/DataSplitting.R" class="external-link"><code>R/DataSplitting.R</code></a></small>
     <div class="hidden name"><code>splitData.Rd</code></div>
     </div>
 
@@ -183,70 +131,67 @@ <h1>Split the plpData into test/train sets using a splitting settings of class <
     <p>Split the plpData into test/train sets using a splitting settings of class <code>splitSettings</code></p>
     </div>
 
-    <pre class="usage"><span class='fu'>splitData</span><span class='op'>(</span>
-  plpData <span class='op'>=</span> <span class='va'>plpData</span>,
-  population <span class='op'>=</span> <span class='va'>population</span>,
-  splitSettings <span class='op'>=</span> <span class='va'>splitSettings</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpData</th>
-      <td><p>An object of type <code>plpData</code> - the patient level prediction
-data extracted from the CDM.</p></td>
-    </tr>
-    <tr>
-      <th>population</th>
-      <td><p>The population created using <code>createStudyPopulation</code> that define who will be used to develop the model</p></td>
-    </tr>
-    <tr>
-      <th>splitSettings</th>
-      <td><p>An object of type <code>splitSettings</code> specifying the split - the default can be created using <code>createDefaultSplitSetting</code></p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>An object of class <code>splitSettings</code></p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Returns a list containing the training data (Train) and optionally the test data (Test).  Train is an Andromeda object containing</p><ul>
-<li><p>covariateRef a table with the covariate information</p></li>
-<li><p>labels) a table (rowId, outcomeCount, ...) for each data point in the train data (outcomeCount is the class label)</p></li>
-<li><p>folds a table (rowId, index) specifying which training fold each data point is in.</p></li>
-</ul><p>Test is an Andromeda object containing</p><ul>
-<li><p>covariateRef a table with the covariate information</p></li>
-<li><p>labels) a table (rowId, outcomeCount, ...) for each data point in the test data (outcomeCount is the class label)</p></li>
-</ul>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">splitData</span><span class="op">(</span></span>
+<span>  plpData <span class="op">=</span> <span class="va">plpData</span>,</span>
+<span>  population <span class="op">=</span> <span class="va">population</span>,</span>
+<span>  splitSettings <span class="op">=</span> <span class="va">splitSettings</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpData</dt>
+<dd><p>An object of type <code>plpData</code> - the patient level prediction
+data extracted from the CDM.</p></dd>
+
+
+<dt>population</dt>
+<dd><p>The population created using <code>createStudyPopulation</code> that define who will be used to develop the model</p></dd>
+
+
+<dt>splitSettings</dt>
+<dd><p>An object of type <code>splitSettings</code> specifying the split - the default can be created using <code>createDefaultSplitSetting</code></p></dd>
 
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>An object of class <code>splitSettings</code></p>
+
+
+    </div>
+    <div id="details">
+    <h2>Details</h2>
+    <p>Returns a list containing the training data (Train) and optionally the test data (Test).  Train is an Andromeda object containing</p><ul><li><p>covariateRef: a table with the covariate information</p></li>
+<li><p>labels: a table (rowId, outcomeCount, ...) for each data point in the train data (outcomeCount is the class label)</p></li>
+<li><p>folds: a table (rowId, index) specifying which training fold each data point is in.</p></li>
+</ul><p>Test is an Andromeda object containing</p><ul><li><p>covariateRef: a table with the covariate information</p></li>
+<li><p>labels: a table (rowId, outcomeCount, ...) for each data point in the test data (outcomeCount is the class label)</p></li>
+</ul></div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/subjectSplitter.html b/docs/reference/subjectSplitter.html
deleted file mode 100644
index 94a1942d5..000000000
--- a/docs/reference/subjectSplitter.html
+++ /dev/null
@@ -1,249 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Split data when patients are in the data multiple times such that the same patient is always either in the 
-train set or the test set (the same patient cannot be in both the test and train set at different times) — subjectSplitter • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Split data when patients are in the data multiple times such that the same patient is always either in the 
-train set or the test set (the same patient cannot be in both the test and train set at different times) — subjectSplitter" />
-<meta property="og:description" content="Split data when patients are in the data multiple times such that the same patient is always either in the 
-train set or the test set (the same patient cannot be in both the test and train set at different times)" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Split data when patients are in the data multiple times such that the same patient is always either in the 
-train set or the test set (the same patient cannot be in both the test and train set at different times)</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/DataSplitting.R'><code>R/DataSplitting.R</code></a></small>
-    <div class="hidden name"><code>subjectSplitter.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Split data when patients are in the data multiple times such that the same patient is always either in the 
-train set or the test set (the same patient cannot be in both the test and train set at different times)</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>subjectSplitter</span><span class='op'>(</span><span class='va'>population</span>, test <span class='op'>=</span> <span class='fl'>0.3</span>, train <span class='op'>=</span> <span class='cn'>NULL</span>, nfold <span class='op'>=</span> <span class='fl'>3</span>, seed <span class='op'>=</span> <span class='cn'>NULL</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>population</th>
-      <td><p>An object created using createStudyPopulation().</p></td>
-    </tr>
-    <tr>
-      <th>test</th>
-      <td><p>A real number between 0 and 1 indicating the test set fraction of the data</p></td>
-    </tr>
-    <tr>
-      <th>train</th>
-      <td><p>A real number between 0 and 1 indicating the train set fraction of the data.
-If not set train is equal to 1 - test</p></td>
-    </tr>
-    <tr>
-      <th>nfold</th>
-      <td><p>An integer &gt;= 1 specifying the number of folds used in cross validation</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>If set a fixed seed is used, otherwise a random split is performed</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A dataframe containing the columns: rowId and index</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Returns a dataframe of rowIds and indexes with a -1 index indicating the rowId belongs to the test
-set and a positive integer index value indicating the rowId's cross valiation fold within the train
-set.</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/summaryPlpAnalyses.html b/docs/reference/summaryPlpAnalyses.html
deleted file mode 100644
index 3f9fd8e36..000000000
--- a/docs/reference/summaryPlpAnalyses.html
+++ /dev/null
@@ -1,207 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>summarises the multiple PLP results into a dataframe — summaryPlpAnalyses • PatientLevelPrediction</title>
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script>
-
-<!-- sticky kit -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-<meta property="og:title" content="summarises the multiple PLP results into a dataframe — summaryPlpAnalyses" />
-
-<meta property="og:description" content="summarises the multiple PLP results into a dataframe" />
-<meta name="twitter:card" content="summary" />
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-  </head>
-
-  <body>
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.4</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>summarises the multiple PLP results into a dataframe</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ViewMultiplePlp.R'><code>R/ViewMultiplePlp.R</code></a></small>
-    <div class="hidden name"><code>summaryPlpAnalyses.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    
-    <p>summarises the multiple PLP results into a dataframe</p>
-    
-    </div>
-
-    <pre class="usage"><span class='fu'>summaryPlpAnalyses</span>(<span class='no'>analysesLocation</span>)</pre>
-    
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>analysesLocation</th>
-      <td><p>The directory containing the results (with the analysis_x folders)</p></td>
-    </tr>
-    </table>
-    
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Loads all the study results contained in the analysesLocation and aggregates a summary of the results</p>
-    
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-    <h2>Contents</h2>
-    <ul class="nav nav-pills nav-stacked">
-      <li><a href="#arguments">Arguments</a></li>
-      
-      <li><a href="#details">Details</a></li>
-          </ul>
-
-  </div>
-</div>
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-   </div>
-
-  
-
-  </body>
-</html>
-
diff --git a/docs/reference/timeSplitter.html b/docs/reference/timeSplitter.html
deleted file mode 100644
index 216968c66..000000000
--- a/docs/reference/timeSplitter.html
+++ /dev/null
@@ -1,248 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Split test/train data by time and then partitions training set into random folds stratified by
-class — timeSplitter • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Split test/train data by time and then partitions training set into random folds stratified by
-class — timeSplitter" />
-<meta property="og:description" content="Split test/train data by time and then partitions training set into random folds stratified by
-class" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Split test/train data by time and then partitions training set into random folds stratified by
-class</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/DataSplitting.R'><code>R/DataSplitting.R</code></a></small>
-    <div class="hidden name"><code>timeSplitter.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Split test/train data by time and then partitions training set into random folds stratified by
-class</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>timeSplitter</span><span class='op'>(</span><span class='va'>population</span>, test <span class='op'>=</span> <span class='fl'>0.3</span>, train <span class='op'>=</span> <span class='cn'>NULL</span>, nfold <span class='op'>=</span> <span class='fl'>3</span>, seed <span class='op'>=</span> <span class='cn'>NULL</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>population</th>
-      <td><p>An object created using createStudyPopulation().</p></td>
-    </tr>
-    <tr>
-      <th>test</th>
-      <td><p>A real number between 0 and 1 indicating the test set fraction of the data</p></td>
-    </tr>
-    <tr>
-      <th>train</th>
-      <td><p>A real number between 0 and 1 indicating the training set fraction of the data</p></td>
-    </tr>
-    <tr>
-      <th>nfold</th>
-      <td><p>An integer &gt;= 1 specifying the number of folds used in cross validation</p></td>
-    </tr>
-    <tr>
-      <th>seed</th>
-      <td><p>If set a fixed seed is used, otherwise a random split is performed</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>A dataframe containing the columns: rowId and index</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>Returns a dataframe of rowIds and indexes with a -1 index indicating the rowId belongs to the test
-set and a positive integer index value indicating the rowId's cross valiation fold within the train
-set.</p>
-
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/toPlpData.html b/docs/reference/toPlpData.html
deleted file mode 100644
index 6d54bce3a..000000000
--- a/docs/reference/toPlpData.html
+++ /dev/null
@@ -1,240 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Convert matrix into plpData — toPlpData • PatientLevelPrediction</title>
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script>
-
-<!-- sticky kit -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-<meta property="og:title" content="Convert matrix into plpData — toPlpData" />
-
-<meta property="og:description" content="Converts a matrix (rows = people, columns=variables) into the standard plpData" />
-<meta name="twitter:card" content="summary" />
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-  </head>
-
-  <body>
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.5</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Convert matrix into plpData</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ImportExport.R'><code>R/ImportExport.R</code></a></small>
-    <div class="hidden name"><code>toPlpData.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    
-    <p>Converts a matrix (rows = people, columns=variables) into the standard plpData</p>
-    
-    </div>
-
-    <pre class="usage"><span class='fu'>toPlpData</span>(<span class='no'>data</span>, <span class='no'>columnInfo</span>, <span class='no'>outcomeId</span>, <span class='kw'>outcomeThreshold</span> <span class='kw'>=</span> <span class='fl'>0.5</span>,
-  <span class='kw'>indexTime</span> <span class='kw'>=</span> <span class='fl'>0</span>, <span class='kw'>includeIndexDay</span> <span class='kw'>=</span> <span class='no'>T</span>)</pre>
-    
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>data</th>
-      <td><p>An data.frame or matrix.</p></td>
-    </tr>
-    <tr>
-      <th>columnInfo</th>
-      <td><p>A dataframe with three columns, column 1 contains columnId, column 2 contains columnName for each column id and column 3 contains the columnTime - the time prior to index the variable was recorded</p></td>
-    </tr>
-    <tr>
-      <th>outcomeId</th>
-      <td><p>The column id containing the outcome</p></td>
-    </tr>
-    <tr>
-      <th>outcomeThreshold</th>
-      <td><p>The outcome value must be higher or equal to this for the person to have the outcome</p></td>
-    </tr>
-    <tr>
-      <th>indexTime</th>
-      <td><p>The time defining the index date</p></td>
-    </tr>
-    <tr>
-      <th>includeIndexDay</th>
-      <td><p>Boolean - whether to include variables recorded on index date</p></td>
-    </tr>
-    </table>
-    
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>Returns an object of class plpData</p>
-    
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>This function converts matrix into plpData</p>
-    
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'>#TODO
-
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-    <h2>Contents</h2>
-    <ul class="nav nav-pills nav-stacked">
-      <li><a href="#arguments">Arguments</a></li>
-      
-      <li><a href="#value">Value</a></li>
-
-      <li><a href="#details">Details</a></li>
-      
-      <li><a href="#examples">Examples</a></li>
-    </ul>
-
-  </div>
-</div>
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-   </div>
-
-  
-
-  </body>
-</html>
-
diff --git a/docs/reference/toSparseM.html b/docs/reference/toSparseM.html
index aa2b22c83..7c240da30 100644
--- a/docs/reference/toSparseM.html
+++ b/docs/reference/toSparseM.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Convert the plpData in COO format into a sparse R matrix — toSparseM • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Convert the plpData in COO format into a sparse R matrix — toSparseM" />
-<meta property="og:description" content="Converts the standard plpData to a sparse matrix" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Convert the plpData in COO format into a sparse R matrix — toSparseM • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="Convert the plpData in COO format into a sparse R matrix — toSparseM"><meta property="og:description" content="Converts the standard plpData to a sparse matrix"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Convert the plpData in COO format into a sparse R matrix</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Formatting.R'><code>R/Formatting.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/Formatting.R" class="external-link"><code>R/Formatting.R</code></a></small>
     <div class="hidden name"><code>toSparseM.Rd</code></div>
     </div>
 
@@ -183,71 +131,76 @@ <h1>Convert the plpData in COO format into a sparse R matrix</h1>
     <p>Converts the standard plpData to a sparse matrix</p>
     </div>
 
-    <pre class="usage"><span class='fu'>toSparseM</span><span class='op'>(</span><span class='va'>plpData</span>, cohort <span class='op'>=</span> <span class='cn'>NULL</span>, map <span class='op'>=</span> <span class='cn'>NULL</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpData</th>
-      <td><p>An object of type <code>plpData</code> with covariate in coo format - the patient level prediction
-data extracted from the CDM.</p></td>
-    </tr>
-    <tr>
-      <th>cohort</th>
-      <td><p>If specified the plpData is restricted to the rowIds in the cohort (otherwise plpData$labels is used)</p></td>
-    </tr>
-    <tr>
-      <th>map</th>
-      <td><p>A covariate map (telling us the column number for covariates)</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>Returns a list, containing the data as a sparse matrix, the plpData covariateRef
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">toSparseM</span><span class="op">(</span><span class="va">plpData</span>, cohort <span class="op">=</span> <span class="cn">NULL</span>, map <span class="op">=</span> <span class="cn">NULL</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>plpData</dt>
+<dd><p>An object of type <code>plpData</code> with covariate in coo format - the patient level prediction
+data extracted from the CDM.</p></dd>
+
+
+<dt>cohort</dt>
+<dd><p>If specified the plpData is restricted to the rowIds in the cohort (otherwise plpData$labels is used)</p></dd>
+
+
+<dt>map</dt>
+<dd><p>A covariate map (telling us the column number for covariates)</p></dd>
+
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>Returns a list, containing the data as a sparse matrix, the plpData covariateRef
 and a data.frame named map that tells us what covariate corresponds to each column
-This object is a list with the following components:</p><dl>
-<dt>data</dt><dd><p>A sparse matrix with the rows corresponding to each person in the plpData and the columns corresponding to the covariates.</p></dd>
-<dt>covariateRef</dt><dd><p>The plpData covariateRef.</p></dd>
-<dt>map</dt><dd><p>A data.frame containing the data column ids and the corresponding covariateId from covariateRef.</p></dd>
+This object is a list with the following components:</p><dl><dt>data</dt>
+<dd><p>A sparse matrix with the rows corresponding to each person in the plpData and the columns corresponding to the covariates.</p></dd>
 
-</dl>
+<dt>covariateRef</dt>
+<dd><p>The plpData covariateRef.</p></dd>
 
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<dt>map</dt>
+<dd><p>A data.frame containing the data column ids and the corresponding covariateId from covariateRef.</p></dd>
 
+
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>This function converts the covariate file from ffdf in COO format into a sparse matrix from
 the package Matrix</p>
+    </div>
 
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='co'>#TODO</span>
-
-</div></pre>
+    <div id="ref-examples">
+    <h2>Examples</h2>
+    <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="co">#TODO</span></span></span>
+<span class="r-in"><span></span></span>
+<span class="r-in"></span>
+</code></pre></div>
+    </div>
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/toSparsePython.html b/docs/reference/toSparsePython.html
deleted file mode 100644
index 638d441e8..000000000
--- a/docs/reference/toSparsePython.html
+++ /dev/null
@@ -1,245 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Convert the plpData in COO format into a sparse python matrix — toSparsePython • PatientLevelPrediction</title>
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script>
-
-<!-- sticky kit -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-<meta property="og:title" content="Convert the plpData in COO format into a sparse python matrix — toSparsePython" />
-
-<meta property="og:description" content="Converts the standard plpData to a sparse matrix firectly into python" />
-<meta name="twitter:card" content="summary" />
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-  </head>
-
-  <body>
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.5</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Convert the plpData in COO format into a sparse python matrix</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Formatting.R'><code>R/Formatting.R</code></a></small>
-    <div class="hidden name"><code>toSparsePython.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    
-    <p>Converts the standard plpData to a sparse matrix firectly into python</p>
-    
-    </div>
-
-    <pre class="usage"><span class='fu'>toSparsePython</span>(<span class='no'>plpData</span>, <span class='no'>population</span>, <span class='kw'>map</span> <span class='kw'>=</span> <span class='kw'>NULL</span>, <span class='kw'>temporal</span> <span class='kw'>=</span> <span class='no'>F</span>,
-  <span class='kw'>pythonExePath</span> <span class='kw'>=</span> <span class='kw'>NULL</span>)</pre>
-    
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpData</th>
-      <td><p>An object of type <code>plpData</code> with covariate in coo format - the patient level prediction
-data extracted from the CDM.</p></td>
-    </tr>
-    <tr>
-      <th>population</th>
-      <td><p>The population to include in the matrix</p></td>
-    </tr>
-    <tr>
-      <th>map</th>
-      <td><p>A covariate map (telling us the column number for covariates)</p></td>
-    </tr>
-    <tr>
-      <th>temporal</th>
-      <td><p>Whether to include timeId into tensor</p></td>
-    </tr>
-    <tr>
-      <th>pythonExePath</th>
-      <td><p>Location of python exe you want to use</p></td>
-    </tr>
-    </table>
-    
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>Returns a list, containing the python object name of the sparse matrix, the plpData covariateRef
-and a data.frame named map that tells us what covariate corresponds to each column
-This object is a list with the following components:</p><dl class='dl-horizontal'>
-<dt>data</dt><dd><p>The python object name containing a sparse matrix with the rows corresponding to each person in the plpData and the columns corresponding to the covariates.</p></dd>
-<dt>covariateRef</dt><dd><p>The plpData covariateRef.</p></dd>
-<dt>map</dt><dd><p>A data.frame containing the data column ids and the corresponding covariateId from covariateRef.</p></dd>
-</dl>
-
-    
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>This function converts the covariate file from ffdf in COO format into a sparse matrix from
-the package Matrix</p>
-    
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'>#TODO
-
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-    <h2>Contents</h2>
-    <ul class="nav nav-pills nav-stacked">
-      <li><a href="#arguments">Arguments</a></li>
-      
-      <li><a href="#value">Value</a></li>
-
-      <li><a href="#details">Details</a></li>
-      
-      <li><a href="#examples">Examples</a></li>
-    </ul>
-
-  </div>
-</div>
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-   </div>
-
-  
-
-  </body>
-</html>
-
diff --git a/docs/reference/toSparseTorchPython.html b/docs/reference/toSparseTorchPython.html
deleted file mode 100644
index 64c4fb439..000000000
--- a/docs/reference/toSparseTorchPython.html
+++ /dev/null
@@ -1,261 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Convert the plpData in COO format into a sparse python matrix using torch.sparse — toSparseTorchPython • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Convert the plpData in COO format into a sparse python matrix using torch.sparse — toSparseTorchPython" />
-<meta property="og:description" content="Converts the standard plpData to a sparse matrix firectly into python" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.3.10</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Convert the plpData in COO format into a sparse python matrix using torch.sparse</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Formatting.R'><code>R/Formatting.R</code></a></small>
-    <div class="hidden name"><code>toSparseTorchPython.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Converts the standard plpData to a sparse matrix firectly into python</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>toSparseTorchPython</span><span class='op'>(</span>
-  <span class='va'>plpData</span>,
-  <span class='va'>population</span>,
-  map <span class='op'>=</span> <span class='cn'>NULL</span>,
-  temporal <span class='op'>=</span> <span class='cn'>F</span>,
-  pythonExePath <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpData</th>
-      <td><p>An object of type <code>plpData</code> with covariate in coo format - the patient level prediction
-data extracted from the CDM.</p></td>
-    </tr>
-    <tr>
-      <th>population</th>
-      <td><p>The population to include in the matrix</p></td>
-    </tr>
-    <tr>
-      <th>map</th>
-      <td><p>A covariate map (telling us the column number for covariates)</p></td>
-    </tr>
-    <tr>
-      <th>temporal</th>
-      <td><p>Whether to include timeId into tensor</p></td>
-    </tr>
-    <tr>
-      <th>pythonExePath</th>
-      <td><p>Location of python exe you want to use</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>Returns a list, containing the python object name of the sparse matrix, the plpData covariateRef
-and a data.frame named map that tells us what covariate corresponds to each column
-This object is a list with the following components:</p><dl>
-<dt>data</dt><dd><p>The python object name containing a sparse matrix with the rows corresponding to each person in the plpData and the columns corresponding to the covariates.</p></dd>
-<dt>covariateRef</dt><dd><p>The plpData covariateRef.</p></dd>
-<dt>map</dt><dd><p>A data.frame containing the data column ids and the corresponding covariateId from covariateRef.</p></dd>
-
-</dl>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>This function converts the covariate file from ffdf in COO format into a sparse matrix from
-the package Matrix</p>
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='co'>#TODO</span>
-
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/toSparseTorchPython2.html b/docs/reference/toSparseTorchPython2.html
deleted file mode 100644
index 79b057c34..000000000
--- a/docs/reference/toSparseTorchPython2.html
+++ /dev/null
@@ -1,245 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Convert the plpData in COO format into a sparse python matrix using torch.sparse — toSparseTorchPython2 • PatientLevelPrediction</title>
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.3.7/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js" integrity="sha256-FiZwavyI2V6+EXO1U+xzLG3IKldpiTFf3153ea9zikQ=" crossorigin="anonymous"></script>
-
-<!-- sticky kit -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/sticky-kit/1.1.3/sticky-kit.min.js" integrity="sha256-c4Rlo1ZozqTPE2RLuvbusY3+SU1pQaJC0TjuhygMipw=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-<meta property="og:title" content="Convert the plpData in COO format into a sparse python matrix using torch.sparse — toSparseTorchPython2" />
-
-<meta property="og:description" content="Converts the standard plpData to a sparse matrix firectly into python" />
-<meta name="twitter:card" content="summary" />
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-  </head>
-
-  <body>
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">3.0.5</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fa fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/GeneratingLearningCurves.html">Generating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/ImplementingExistingModels.html">Implementing Existing Prediction Models using the OHDSI PatientLevelPrediction Framework</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fa fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Convert the plpData in COO format into a sparse python matrix using torch.sparse</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/Formatting.R'><code>R/Formatting.R</code></a></small>
-    <div class="hidden name"><code>toSparseTorchPython2.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    
-    <p>Converts the standard plpData to a sparse matrix firectly into python</p>
-    
-    </div>
-
-    <pre class="usage"><span class='fu'>toSparseTorchPython2</span>(<span class='no'>plpData</span>, <span class='no'>population</span>, <span class='kw'>map</span> <span class='kw'>=</span> <span class='kw'>NULL</span>, <span class='kw'>temporal</span> <span class='kw'>=</span> <span class='no'>F</span>,
-  <span class='kw'>pythonExePath</span> <span class='kw'>=</span> <span class='kw'>NULL</span>)</pre>
-    
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpData</th>
-      <td><p>An object of type <code>plpData</code> with covariate in coo format - the patient level prediction
-data extracted from the CDM.</p></td>
-    </tr>
-    <tr>
-      <th>population</th>
-      <td><p>The population to include in the matrix</p></td>
-    </tr>
-    <tr>
-      <th>map</th>
-      <td><p>A covariate map (telling us the column number for covariates)</p></td>
-    </tr>
-    <tr>
-      <th>temporal</th>
-      <td><p>Whether to include timeId into tensor</p></td>
-    </tr>
-    <tr>
-      <th>pythonExePath</th>
-      <td><p>Location of python exe you want to use</p></td>
-    </tr>
-    </table>
-    
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
-
-    <p>Returns a list, containing the python object name of the sparse matrix, the plpData covariateRef
-and a data.frame named map that tells us what covariate corresponds to each column
-This object is a list with the following components:</p><dl class='dl-horizontal'>
-<dt>data</dt><dd><p>The python object name containing a sparse matrix with the rows corresponding to each person in the plpData and the columns corresponding to the covariates.</p></dd>
-<dt>covariateRef</dt><dd><p>The plpData covariateRef.</p></dd>
-<dt>map</dt><dd><p>A data.frame containing the data column ids and the corresponding covariateId from covariateRef.</p></dd>
-</dl>
-
-    
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>This function converts the covariate file from ffdf in COO format into a sparse matrix from
-the package Matrix</p>
-    
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'>#TODO
-
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="sidebar">
-    <h2>Contents</h2>
-    <ul class="nav nav-pills nav-stacked">
-      <li><a href="#arguments">Arguments</a></li>
-      
-      <li><a href="#value">Value</a></li>
-
-      <li><a href="#details">Details</a></li>
-      
-      <li><a href="#examples">Examples</a></li>
-    </ul>
-
-  </div>
-</div>
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.3.0.</p>
-</div>
-      </footer>
-   </div>
-
-  
-
-  </body>
-</html>
-
diff --git a/docs/reference/transferLearning.html b/docs/reference/transferLearning.html
deleted file mode 100644
index 1cf29eb56..000000000
--- a/docs/reference/transferLearning.html
+++ /dev/null
@@ -1,279 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>[Under development] Transfer learning — transferLearning • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="[Under development] Transfer learning — transferLearning" />
-<meta property="og:description" content="[Under development] Transfer learning" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.3.10</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>[Under development] Transfer learning</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/DeepNN.R'><code>R/DeepNN.R</code></a></small>
-    <div class="hidden name"><code>transferLearning.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>[Under development] Transfer learning</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>transferLearning</span><span class='op'>(</span>
-  <span class='va'>plpResult</span>,
-  <span class='va'>plpData</span>,
-  <span class='va'>population</span>,
-  fixLayers <span class='op'>=</span> <span class='cn'>T</span>,
-  includeTop <span class='op'>=</span> <span class='cn'>F</span>,
-  addLayers <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='fl'>100</span>, <span class='fl'>10</span><span class='op'>)</span>,
-  layerDropout <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='cn'>T</span>, <span class='cn'>T</span><span class='op'>)</span>,
-  layerActivation <span class='op'>=</span> <span class='fu'><a href='https://rdrr.io/r/base/c.html'>c</a></span><span class='op'>(</span><span class='st'>"relu"</span>, <span class='st'>"softmax"</span><span class='op'>)</span>,
-  outcomeWeight <span class='op'>=</span> <span class='fl'>1</span>,
-  batchSize <span class='op'>=</span> <span class='fl'>10000</span>,
-  epochs <span class='op'>=</span> <span class='fl'>20</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpResult</th>
-      <td><p>The plp result when training a kersa deep learning model on big data</p></td>
-    </tr>
-    <tr>
-      <th>plpData</th>
-      <td><p>The new data to fine tune the model on</p></td>
-    </tr>
-    <tr>
-      <th>population</th>
-      <td><p>The population for the new data</p></td>
-    </tr>
-    <tr>
-      <th>fixLayers</th>
-      <td><p>boolean specificying whether to fix weights in model being transferred</p></td>
-    </tr>
-    <tr>
-      <th>includeTop</th>
-      <td><p>If TRUE the final layer of the model being transferred is removed</p></td>
-    </tr>
-    <tr>
-      <th>addLayers</th>
-      <td><p>vector specifying nodes in each layer to add e.g. c(100,10) will add another layer with 100 nodels and then a final layer with 10</p></td>
-    </tr>
-    <tr>
-      <th>layerDropout</th>
-      <td><p>Add dropout to each new layer (binary vector length of addLayers)</p></td>
-    </tr>
-    <tr>
-      <th>layerActivation</th>
-      <td><p>Activation function for each new layer (string vector length of addLayers)</p></td>
-    </tr>
-    <tr>
-      <th>outcomeWeight</th>
-      <td><p>The weight to assign the class 1 when training the model</p></td>
-    </tr>
-    <tr>
-      <th>batchSize</th>
-      <td><p>Size of each batch for updating layers</p></td>
-    </tr>
-    <tr>
-      <th>epochs</th>
-      <td><p>Number of epoches to run</p></td>
-    </tr>
-    </table>
-
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='va'>modelSet</span> <span class='op'>&lt;-</span> <span class='fu'><a href='setDeepNN.html'>setDeepNN</a></span><span class='op'>(</span><span class='op'>)</span>
-<span class='va'>plpResult</span> <span class='op'>&lt;-</span> <span class='fu'><a href='runPlp.html'>runPlp</a></span><span class='op'>(</span><span class='va'>plpData</span>, <span class='va'>population</span>, modelSettings <span class='op'>=</span> <span class='va'>modelSet</span>, <span class='va'>...</span><span class='op'>)</span>
-
-<span class='fu'>transferLearning</span><span class='op'>(</span><span class='va'>...</span><span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/transportModel.html b/docs/reference/transportModel.html
deleted file mode 100644
index 166870fe2..000000000
--- a/docs/reference/transportModel.html
+++ /dev/null
@@ -1,232 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Transports a plpModel to a new location and removes sensitive data — transportModel • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Transports a plpModel to a new location and removes sensitive data — transportModel" />
-<meta property="og:description" content="Transports a plpModel to a new location and removes sensitive data" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Transports a plpModel to a new location and removes sensitive data</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ImportExport.R'><code>R/ImportExport.R</code></a></small>
-    <div class="hidden name"><code>transportModel.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Transports a plpModel to a new location and removes sensitive data</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>transportModel</span><span class='op'>(</span><span class='va'>plpModel</span>, <span class='va'>outputFolder</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpModel</th>
-      <td><p>A trianed model.</p></td>
-    </tr>
-    <tr>
-      <th>outputFolder</th>
-      <td><p>The folder on the file system where the CSV files will be created. If the
-folder does not yet exist it will be created.</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>This function is used to</p>
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='fu'>transportModel</span><span class='op'>(</span><span class='va'>plpModel</span>, <span class='st'>"s:/temp/exportTest"</span><span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/transportPlp.html b/docs/reference/transportPlp.html
deleted file mode 100644
index 57f63a709..000000000
--- a/docs/reference/transportPlp.html
+++ /dev/null
@@ -1,290 +0,0 @@
-<!-- Generated by pkgdown: do not edit by hand -->
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>Transports a plpResult to a new location and removed sensitive data — transportPlp • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="Transports a plpResult to a new location and removed sensitive data — transportPlp" />
-<meta property="og:description" content="Transports a plpResult to a new location and removed sensitive data" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
-<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
-    <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
-  <div class="container">
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
-        <span class="sr-only">Toggle navigation</span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </button>
-      <span class="navbar-brand">
-        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.4.0</span>
-      </span>
-    </div>
-
-    <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
-  <a href="../index.html">
-    <span class="fas fa-home fa-lg"></span>
-     
-  </a>
-</li>
-<li>
-  <a href="../articles/PatientLevelPrediction.html">Get started</a>
-</li>
-<li>
-  <a href="../articles/Videos.html">Videos</a>
-</li>
-<li>
-  <a href="../reference/index.html">Reference</a>
-</li>
-<li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-    Articles
-     
-    <span class="caret"></span>
-  </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
-      <a href="../articles/AddingCustomAlgorithms.html">Adding Custom Patient-Level Prediction Algorithms</a>
-    </li>
-    <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingDeepLearningModels.html">Building Deep Learning Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
-    </li>
-    <li>
-      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
-    </li>
-    <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
-    </li>
-    <li>
-      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
-    </li>
-    <li>
-      <a href="../articles/Videos.html">Demo Videos</a>
-    </li>
-  </ul>
-</li>
-<li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
-</li>
-<li>
-  <a href="../articles/BestPractices.html">Best Practices</a>
-</li>
-<li>
-  <a href="../news/index.html">Changelog</a>
-</li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
-</li>
-<li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
-    <span class="fas fa-github fa-lg"></span>
-     
-  </a>
-</li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
-  </div><!--/.container -->
-</div><!--/.navbar -->
-
-      
-
-      </header>
-
-<div class="row">
-  <div class="col-md-9 contents">
-    <div class="page-header">
-    <h1>Transports a plpResult to a new location and removed sensitive data</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ImportExport.R'><code>R/ImportExport.R</code></a></small>
-    <div class="hidden name"><code>transportPlp.Rd</code></div>
-    </div>
-
-    <div class="ref-description">
-    <p>Transports a plpResult to a new location and removed sensitive data</p>
-    </div>
-
-    <pre class="usage"><span class='fu'>transportPlp</span><span class='op'>(</span>
-  <span class='va'>plpResult</span>,
-  modelName <span class='op'>=</span> <span class='cn'>NULL</span>,
-  dataName <span class='op'>=</span> <span class='cn'>NULL</span>,
-  <span class='va'>outputFolder</span>,
-  n <span class='op'>=</span> <span class='cn'>NULL</span>,
-  includeEvaluationStatistics <span class='op'>=</span> <span class='cn'>T</span>,
-  includeThresholdSummary <span class='op'>=</span> <span class='cn'>T</span>,
-  includeDemographicSummary <span class='op'>=</span> <span class='cn'>T</span>,
-  includeCalibrationSummary <span class='op'>=</span> <span class='cn'>T</span>,
-  includePredictionDistribution <span class='op'>=</span> <span class='cn'>T</span>,
-  includeCovariateSummary <span class='op'>=</span> <span class='cn'>T</span>,
-  save <span class='op'>=</span> <span class='cn'>T</span>,
-  reduceSize <span class='op'>=</span> <span class='cn'>F</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>plpResult</th>
-      <td><p>An object returned by running <code>runPlp</code>.</p></td>
-    </tr>
-    <tr>
-      <th>modelName</th>
-      <td><p>A string of the name of the model</p></td>
-    </tr>
-    <tr>
-      <th>dataName</th>
-      <td><p>A string of the name of the data</p></td>
-    </tr>
-    <tr>
-      <th>outputFolder</th>
-      <td><p>The folder on the file system where the CSV files will be created. If the
-folder does not yet exist it will be created.</p></td>
-    </tr>
-    <tr>
-      <th>n</th>
-      <td><p>The minimum number of people required for each result summary to be included</p></td>
-    </tr>
-    <tr>
-      <th>includeEvaluationStatistics</th>
-      <td><p>Whether to include the evaluationStatistics</p></td>
-    </tr>
-    <tr>
-      <th>includeThresholdSummary</th>
-      <td><p>Whether to include the thresholdSummary</p></td>
-    </tr>
-    <tr>
-      <th>includeDemographicSummary</th>
-      <td><p>Whether to include the demographicSummary</p></td>
-    </tr>
-    <tr>
-      <th>includeCalibrationSummary</th>
-      <td><p>Whether to include the calibrationSummary</p></td>
-    </tr>
-    <tr>
-      <th>includePredictionDistribution</th>
-      <td><p>Whether to include the predictionDistribution</p></td>
-    </tr>
-    <tr>
-      <th>includeCovariateSummary</th>
-      <td><p>Whether to include the covariateSummary</p></td>
-    </tr>
-    <tr>
-      <th>save</th>
-      <td><p>Whether to save the result or just return the transportable object</p></td>
-    </tr>
-    <tr>
-      <th>reduceSize</th>
-      <td><p>Remove parts of runPlp object that are not needed but take up space</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
-
-    <p>This function is used to</p>
-
-    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
-    <pre class="examples"><div class='input'><span class='kw'>if</span> <span class='op'>(</span><span class='cn'>FALSE</span><span class='op'>)</span> <span class='op'>{</span>
-<span class='fu'>transportPlp</span><span class='op'>(</span><span class='va'>plpResult</span>, <span class='st'>"s:/temp/exportTest"</span>, n<span class='op'>=</span><span class='fl'>10</span><span class='op'>)</span>
-<span class='op'>}</span>
-</div></pre>
-  </div>
-  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
-</div>
-
-
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
-</div>
-
-<div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
-</div>
-
-      </footer>
-   </div>
-
-  
-
-
-  </body>
-</html>
-
-
diff --git a/docs/reference/validateExternal.html b/docs/reference/validateExternal.html
new file mode 100644
index 000000000..ce79f9c15
--- /dev/null
+++ b/docs/reference/validateExternal.html
@@ -0,0 +1,188 @@
+<!DOCTYPE html>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>externalValidatePlp - Validate model performance on new data — validateExternal • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="externalValidatePlp - Validate model performance on new data — validateExternal"><meta property="og:description" content="externalValidatePlp - Validate model performance on new data"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
+<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
+
+    <div class="container template-reference-topic">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
+  <div class="container">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <span class="navbar-brand">
+        <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
+      </span>
+    </div>
+
+    <div id="navbar" class="navbar-collapse collapse">
+      <ul class="nav navbar-nav"><li>
+  <a href="../index.html">
+    <span class="fa fa-home fa-lg"></span>
+     
+  </a>
+</li>
+<li>
+  <a href="../articles/InstallationGuide.html">Get started</a>
+</li>
+<li>
+  <a href="../articles/Videos.html">Videos</a>
+</li>
+<li>
+  <a href="../reference/index.html">Reference</a>
+</li>
+<li class="dropdown">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+    Articles
+     
+    <span class="caret"></span>
+  </a>
+  <ul class="dropdown-menu" role="menu"><li>
+      <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomModels.html">Adding Custom Patient-Level Prediction Algorithms</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSamples.html">Adding Custom Sampling Functions</a>
+    </li>
+    <li>
+      <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
+    </li>
+    <li>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
+    </li>
+    <li>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
+    </li>
+    <li>
+      <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
+    </li>
+    <li>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
+    </li>
+    <li>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+    </li>
+    <li>
+      <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
+    </li>
+    <li>
+      <a href="../articles/Videos.html">Demo Videos</a>
+    </li>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
+</li>
+<li>
+  <a href="../articles/BestPractices.html">Best Practices</a>
+</li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
+<li>
+  <a href="../news/index.html">Changelog</a>
+</li>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+</li>
+<li>
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
+    <span class="fa fa-github fa-lg"></span>
+     
+  </a>
+</li>
+      </ul></div><!--/.nav-collapse -->
+  </div><!--/.container -->
+</div><!--/.navbar -->
+
+      
+
+      </header><div class="row">
+  <div class="col-md-9 contents">
+    <div class="page-header">
+    <h1>externalValidatePlp - Validate model performance on new data</h1>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ExternalValidatePlp.R" class="external-link"><code>R/ExternalValidatePlp.R</code></a></small>
+    <div class="hidden name"><code>validateExternal.Rd</code></div>
+    </div>
+
+    <div class="ref-description">
+    <p>externalValidatePlp - Validate model performance on new data</p>
+    </div>
+
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">validateExternal</span><span class="op">(</span></span>
+<span>  <span class="va">validationDesignList</span>,</span>
+<span>  <span class="va">databaseDetails</span>,</span>
+<span>  <span class="va">logSettings</span>,</span>
+<span>  <span class="va">outputFolder</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>validationDesignList</dt>
+<dd><p>A list of objects created with <code>createValidationDesign</code></p></dd>
+
+
+<dt>databaseDetails</dt>
+<dd><p>A list of objects of class
+<code>databaseDetails</code> created using <code>createDatabaseDetails</code></p></dd>
+
+
+<dt>logSettings</dt>
+<dd><p>An object of <code>logSettings</code> created
+using <code>createLogSettings</code></p></dd>
+
+
+<dt>outputFolder</dt>
+<dd><p>The directory to save the validation results to
+(subfolders are created per database in validationDatabaseDetails)</p></dd>
+
+</dl></div>
+
+  </div>
+  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
+</div>
+
+
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
+</div>
+
+<div class="pkgdown">
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
+</div>
+
+      </footer></div>
+
+  
+
+
+  
+
+  </body></html>
+
diff --git a/docs/reference/validateMultiplePlp.html b/docs/reference/validateMultiplePlp.html
index 06aac497f..c75da7891 100644
--- a/docs/reference/validateMultiplePlp.html
+++ b/docs/reference/validateMultiplePlp.html
@@ -1,68 +1,13 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>externally validate the multiple plp models across new datasets — validateMultiplePlp • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="externally validate the multiple plp models across new datasets — validateMultiplePlp" />
-<meta property="og:description" content="This function loads all the models in a multiple plp analysis folder and
-validates the models on new data" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>externally validate the multiple plp models across new datasets — validateMultiplePlp • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="externally validate the multiple plp models across new datasets — validateMultiplePlp"><meta property="og:description" content="This function loads all the models in a multiple plp analysis folder and
+validates the models on new data"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -73,13 +18,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -95,13 +39,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -114,10 +57,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -126,13 +69,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -140,43 +86,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>externally validate the multiple plp models across new datasets</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/RunMultiplePlp.R'><code>R/RunMultiplePlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/RunMultiplePlp.R" class="external-link"><code>R/RunMultiplePlp.R</code></a></small>
     <div class="hidden name"><code>validateMultiplePlp.Rd</code></div>
     </div>
 
@@ -185,69 +133,70 @@ <h1>externally validate the multiple plp models across new datasets</h1>
 validates the models on new data</p>
     </div>
 
-    <pre class="usage"><span class='fu'>validateMultiplePlp</span><span class='op'>(</span>
-  <span class='va'>analysesLocation</span>,
-  <span class='va'>validationDatabaseDetails</span>,
-  validationRestrictPlpDataSettings <span class='op'>=</span> <span class='fu'><a href='createRestrictPlpDataSettings.html'>createRestrictPlpDataSettings</a></span><span class='op'>(</span><span class='op'>)</span>,
-  recalibrate <span class='op'>=</span> <span class='cn'>NULL</span>,
-  saveDirectory <span class='op'>=</span> <span class='cn'>NULL</span>
-<span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>analysesLocation</th>
-      <td><p>The location where the multiple plp analyses are</p></td>
-    </tr>
-    <tr>
-      <th>validationDatabaseDetails</th>
-      <td><p>The validation database settings created using <code><a href='createDatabaseDetails.html'>createDatabaseDetails()</a></code></p></td>
-    </tr>
-    <tr>
-      <th>validationRestrictPlpDataSettings</th>
-      <td><p>The settings specifying the extra restriction settings when extracting the data created using <code><a href='createRestrictPlpDataSettings.html'>createRestrictPlpDataSettings()</a></code>.</p></td>
-    </tr>
-    <tr>
-      <th>recalibrate</th>
-      <td><p>A vector of recalibration methods (currently supports 'RecalibrationintheLarge' and/or 'weakRecalibration')</p></td>
-    </tr>
-    <tr>
-      <th>saveDirectory</th>
-      <td><p>The location to save to validation results</p></td>
-    </tr>
-    </table>
-
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">validateMultiplePlp</span><span class="op">(</span></span>
+<span>  <span class="va">analysesLocation</span>,</span>
+<span>  <span class="va">validationDatabaseDetails</span>,</span>
+<span>  validationRestrictPlpDataSettings <span class="op">=</span> <span class="fu"><a href="createRestrictPlpDataSettings.html">createRestrictPlpDataSettings</a></span><span class="op">(</span><span class="op">)</span>,</span>
+<span>  recalibrate <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  cohortDefinitions <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  saveDirectory <span class="op">=</span> <span class="cn">NULL</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>analysesLocation</dt>
+<dd><p>The location where the multiple plp analyses are</p></dd>
 
+
+<dt>validationDatabaseDetails</dt>
+<dd><p>A single or list of validation database settings created using <code><a href="createDatabaseDetails.html">createDatabaseDetails()</a></code></p></dd>
+
+
+<dt>validationRestrictPlpDataSettings</dt>
+<dd><p>The settings specifying the extra restriction settings when extracting the data created using <code><a href="createRestrictPlpDataSettings.html">createRestrictPlpDataSettings()</a></code>.</p></dd>
+
+
+<dt>recalibrate</dt>
+<dd><p>A vector of recalibration methods (currently supports 'RecalibrationintheLarge' and/or 'weakRecalibration')</p></dd>
+
+
+<dt>cohortDefinitions</dt>
+<dd><p>A list of cohortDefinitions</p></dd>
+
+
+<dt>saveDirectory</dt>
+<dd><p>The location to save to validation results</p></dd>
+
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Users need to input a location where the results of the multiple plp analyses
 are found and the connection and database settings for the new data</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/viewDatabaseResultPlp.html b/docs/reference/viewDatabaseResultPlp.html
index 0b4be6fe9..6ff722a2e 100644
--- a/docs/reference/viewDatabaseResultPlp.html
+++ b/docs/reference/viewDatabaseResultPlp.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>open a local shiny app for viewing the result of a PLP analyses from a database — viewDatabaseResultPlp • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="open a local shiny app for viewing the result of a PLP analyses from a database — viewDatabaseResultPlp" />
-<meta property="og:description" content="open a local shiny app for viewing the result of a PLP analyses from a database" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>open a local shiny app for viewing the result of a PLP analyses from a database — viewDatabaseResultPlp • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="open a local shiny app for viewing the result of a PLP analyses from a database — viewDatabaseResultPlp"><meta property="og:description" content="open a local shiny app for viewing the result of a PLP analyses from a database"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
+</li>
+<li>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>open a local shiny app for viewing the result of a PLP analyses from a database</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ViewShinyPlp.R'><code>R/ViewShinyPlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ViewShinyPlp.R" class="external-link"><code>R/ViewShinyPlp.R</code></a></small>
     <div class="hidden name"><code>viewDatabaseResultPlp.Rd</code></div>
     </div>
 
@@ -183,78 +131,74 @@ <h1>open a local shiny app for viewing the result of a PLP analyses from a datab
     <p>open a local shiny app for viewing the result of a PLP analyses from a database</p>
     </div>
 
-    <pre class="usage"><span class='fu'>viewDatabaseResultPlp</span><span class='op'>(</span>
-  <span class='va'>mySchema</span>,
-  <span class='va'>myServer</span>,
-  <span class='va'>myUser</span>,
-  <span class='va'>myPassword</span>,
-  <span class='va'>myDbms</span>,
-  myPort <span class='op'>=</span> <span class='cn'>NULL</span>,
-  <span class='va'>myTableAppend</span>
-<span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">viewDatabaseResultPlp</span><span class="op">(</span></span>
+<span>  <span class="va">mySchema</span>,</span>
+<span>  <span class="va">myServer</span>,</span>
+<span>  <span class="va">myUser</span>,</span>
+<span>  <span class="va">myPassword</span>,</span>
+<span>  <span class="va">myDbms</span>,</span>
+<span>  myPort <span class="op">=</span> <span class="cn">NULL</span>,</span>
+<span>  <span class="va">myTableAppend</span></span>
+<span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>mySchema</dt>
+<dd><p>Database result schema containing the result tables</p></dd>
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>mySchema</th>
-      <td><p>Database result schema containing the result tables</p></td>
-    </tr>
-    <tr>
-      <th>myServer</th>
-      <td><p>server with the result database</p></td>
-    </tr>
-    <tr>
-      <th>myUser</th>
-      <td><p>Username for the connection to the result database</p></td>
-    </tr>
-    <tr>
-      <th>myPassword</th>
-      <td><p>Password for the connection to the result database</p></td>
-    </tr>
-    <tr>
-      <th>myDbms</th>
-      <td><p>database management system for the result database</p></td>
-    </tr>
-    <tr>
-      <th>myPort</th>
-      <td><p>Port for the connection to the result database</p></td>
-    </tr>
-    <tr>
-      <th>myTableAppend</th>
-      <td><p>A string appended to the results tables (optional)</p></td>
-    </tr>
-    </table>
 
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<dt>myServer</dt>
+<dd><p>server with the result database</p></dd>
 
+
+<dt>myUser</dt>
+<dd><p>Username for the connection to the result database</p></dd>
+
+
+<dt>myPassword</dt>
+<dd><p>Password for the connection to the result database</p></dd>
+
+
+<dt>myDbms</dt>
+<dd><p>database management system for the result database</p></dd>
+
+
+<dt>myPort</dt>
+<dd><p>Port for the connection to the result database</p></dd>
+
+
+<dt>myTableAppend</dt>
+<dd><p>A string appended to the results tables (optional)</p></dd>
+
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Opens a shiny app for viewing the results of the models from a database</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/viewMultiplePlp.html b/docs/reference/viewMultiplePlp.html
index 89200fb9e..60cbf1a87 100644
--- a/docs/reference/viewMultiplePlp.html
+++ b/docs/reference/viewMultiplePlp.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>open a local shiny app for viewing the result of a multiple PLP analyses — viewMultiplePlp • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="open a local shiny app for viewing the result of a multiple PLP analyses — viewMultiplePlp" />
-<meta property="og:description" content="open a local shiny app for viewing the result of a multiple PLP analyses" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>open a local shiny app for viewing the result of a multiple PLP analyses — viewMultiplePlp • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="open a local shiny app for viewing the result of a multiple PLP analyses — viewMultiplePlp"><meta property="og:description" content="open a local shiny app for viewing the result of a multiple PLP analyses"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
-
-  </head>
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>open a local shiny app for viewing the result of a multiple PLP analyses</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ViewShinyPlp.R'><code>R/ViewShinyPlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ViewShinyPlp.R" class="external-link"><code>R/ViewShinyPlp.R</code></a></small>
     <div class="hidden name"><code>viewMultiplePlp.Rd</code></div>
     </div>
 
@@ -183,47 +131,43 @@ <h1>open a local shiny app for viewing the result of a multiple PLP analyses</h1
     <p>open a local shiny app for viewing the result of a multiple PLP analyses</p>
     </div>
 
-    <pre class="usage"><span class='fu'>viewMultiplePlp</span><span class='op'>(</span><span class='va'>analysesLocation</span><span class='op'>)</span></pre>
-
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>analysesLocation</th>
-      <td><p>The directory containing the results (with the analysis_x folders)</p></td>
-    </tr>
-    </table>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">viewMultiplePlp</span><span class="op">(</span><span class="va">analysesLocation</span><span class="op">)</span></span></code></pre></div>
+    </div>
 
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>analysesLocation</dt>
+<dd><p>The directory containing the results (with the analysis_x folders)</p></dd>
 
+</dl></div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Opens a shiny app for viewing the results of the models from various T,O, Tar and settings
 settings.</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/reference/viewPlp.html b/docs/reference/viewPlp.html
index 7ad734083..5dc710218 100644
--- a/docs/reference/viewPlp.html
+++ b/docs/reference/viewPlp.html
@@ -1,67 +1,12 @@
-<!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
-<html lang="en">
-  <head>
-  <meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-<title>viewPlp - Interactively view the performance and model settings — viewPlp • PatientLevelPrediction</title>
-
-
-<!-- jquery -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
-<!-- Bootstrap -->
-<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
-
-
-<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
-
-<!-- bootstrap-toc -->
-<link rel="stylesheet" href="../bootstrap-toc.css">
-<script src="../bootstrap-toc.js"></script>
-
-<!-- Font Awesome icons -->
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
-<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />
-
-<!-- clipboard.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
-
-<!-- headroom.js -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>
-
-<!-- pkgdown -->
-<link href="../pkgdown.css" rel="stylesheet">
-<script src="../pkgdown.js"></script>
-
-
-
-
-<meta property="og:title" content="viewPlp - Interactively view the performance and model settings — viewPlp" />
-<meta property="og:description" content="This is a shiny app for viewing interactive plots of the performance and the settings" />
-
-
-
-
-<!-- mathjax -->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>
-
-<!--[if lt IE 9]>
+<!-- Generated by pkgdown: do not edit by hand --><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>viewPlp - Interactively view the performance and model settings — viewPlp • PatientLevelPrediction</title><!-- jquery --><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script><!-- Bootstrap --><link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous"><script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script><!-- bootstrap-toc --><link rel="stylesheet" href="../bootstrap-toc.css"><script src="../bootstrap-toc.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous"><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous"><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- pkgdown --><link href="../pkgdown.css" rel="stylesheet"><script src="../pkgdown.js"></script><meta property="og:title" content="viewPlp - Interactively view the performance and model settings — viewPlp"><meta property="og:description" content="This is a shiny app for viewing interactive plots of the performance and the settings"><!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-<![endif]-->
-
-
+<![endif]--></head><body data-spy="scroll" data-target="#toc">
+    
 
-  </head>
-
-  <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
-      <header>
-      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+      <header><div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
@@ -72,13 +17,12 @@
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">PatientLevelPrediction</a>
-        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">5.0.3</span>
+        <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">6.3.9</span>
       </span>
     </div>
 
     <div id="navbar" class="navbar-collapse collapse">
-      <ul class="nav navbar-nav">
-        <li>
+      <ul class="nav navbar-nav"><li>
   <a href="../index.html">
     <span class="fa fa-home fa-lg"></span>
      
@@ -94,13 +38,12 @@
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
-  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
+  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" data-bs-toggle="dropdown" aria-expanded="false">
     Articles
      
     <span class="caret"></span>
   </a>
-  <ul class="dropdown-menu" role="menu">
-    <li>
+  <ul class="dropdown-menu" role="menu"><li>
       <a href="../articles/AddingCustomFeatureEngineering.html">Adding Custom Feature Engineering Functions</a>
     </li>
     <li>
@@ -113,10 +56,10 @@
       <a href="../articles/AddingCustomSplitting.html">Adding Custom Data Splitting Functions</a>
     </li>
     <li>
-      <a href="../articles/BestPractices.html">Best Practice Research</a>
+      <a href="../articles/BenchmarkTasks.html">Benchmark Tasks</a>
     </li>
     <li>
-      <a href="../articles/BuildingEnsembleModels.html">Building Ensemble Models</a>
+      <a href="../articles/BestPractices.html">Best Practice Research</a>
     </li>
     <li>
       <a href="../articles/BuildingMultiplePredictiveModels.html">Automatically Build Multiple Patient-Level Predictive Models</a>
@@ -125,13 +68,16 @@
       <a href="../articles/BuildingPredictiveModels.html">Building patient-level predictive models</a>
     </li>
     <li>
-      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
+      <a href="../articles/ClinicalModels.html">Clinical Models</a>
     </li>
     <li>
-      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
+      <a href="../articles/ConstrainedPredictors.html">Constrained predictors</a>
+    </li>
+    <li>
+      <a href="../articles/CreatingLearningCurves.html">Creating Learning Curves</a>
     </li>
     <li>
-      <a href="../articles/CreatingShinyApp.html">Creating Shiny App</a>
+      <a href="../articles/CreatingNetworkStudies.html">Making  patient-level predictive network study packages</a>
     </li>
     <li>
       <a href="../articles/InstallationGuide.html">Patient-Level Prediction Installation Guide</a>
@@ -139,43 +85,45 @@
     <li>
       <a href="../articles/Videos.html">Demo Videos</a>
     </li>
-  </ul>
+  </ul></li>
+<li>
+  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/" class="external-link">Tutorial</a>
 </li>
 <li>
-  <a href="https://www.ohdsi.org/past-events/patient-level-prediction/">Tutorial</a>
+  <a href="../articles/BenchmarkTasks.html">Benchmarks</a>
+</li>
+<li>
+  <a href="../articles/ConstrainedPredictors.html">Predictors</a>
 </li>
 <li>
   <a href="../articles/BestPractices.html">Best Practices</a>
 </li>
+<li>
+  <a href="../articles/ClinicalModels.html">Clinical Models</a>
+</li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
-      </ul>
-      <ul class="nav navbar-nav navbar-right">
-        <li>
-  <a href="https://ohdsi.github.io/Hades"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
+      </ul><ul class="nav navbar-nav navbar-right"><li>
+  <a href="https://ohdsi.github.io/Hades" class="external-link"><img src='https://ohdsi.github.io/Hades/images/hadesMini.png' width=80 height=17 style='vertical-align: top;'></a>
 </li>
 <li>
-  <a href="https://github.com/OHDSI/PatientLevelPrediction">
+  <a href="https://github.com/OHDSI/PatientLevelPrediction" class="external-link">
     <span class="fa fa-github fa-lg"></span>
      
   </a>
 </li>
-      </ul>
-      
-    </div><!--/.nav-collapse -->
+      </ul></div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->
 
       
 
-      </header>
-
-<div class="row">
+      </header><div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>viewPlp - Interactively view the performance and model settings</h1>
-    <small class="dont-index">Source: <a href='https://github.com/OHDSI/PatientLevelPrediction/blob/master/R/ViewShinyPlp.R'><code>R/ViewShinyPlp.R</code></a></small>
+    <small class="dont-index">Source: <a href="https://github.com/OHDSI/PatientLevelPrediction/blob/HEAD/R/ViewShinyPlp.R" class="external-link"><code>R/ViewShinyPlp.R</code></a></small>
     <div class="hidden name"><code>viewPlp.Rd</code></div>
     </div>
 
@@ -183,53 +131,56 @@ <h1>viewPlp - Interactively view the performance and model settings</h1>
     <p>This is a shiny app for viewing interactive plots of the performance and the settings</p>
     </div>
 
-    <pre class="usage"><span class='fu'>viewPlp</span><span class='op'>(</span><span class='va'>runPlp</span>, validatePlp <span class='op'>=</span> <span class='cn'>NULL</span><span class='op'>)</span></pre>
+    <div id="ref-usage">
+    <div class="sourceCode"><pre class="sourceCode r"><code><span><span class="fu">viewPlp</span><span class="op">(</span><span class="va">runPlp</span>, validatePlp <span class="op">=</span> <span class="cn">NULL</span>, diagnosePlp <span class="op">=</span> <span class="cn">NULL</span><span class="op">)</span></span></code></pre></div>
+    </div>
+
+    <div id="arguments">
+    <h2>Arguments</h2>
+    <dl><dt>runPlp</dt>
+<dd><p>The output of runPlp() (an object of class 'runPlp')</p></dd>
+
 
-    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
-    <table class="ref-arguments">
-    <colgroup><col class="name" /><col class="desc" /></colgroup>
-    <tr>
-      <th>runPlp</th>
-      <td><p>The output of runPlp() (an object of class 'runPlp')</p></td>
-    </tr>
-    <tr>
-      <th>validatePlp</th>
-      <td><p>The output of externalValidatePlp (on object of class 'validatePlp')</p></td>
-    </tr>
-    </table>
+<dt>validatePlp</dt>
+<dd><p>The output of externalValidatePlp (on object of class 'validatePlp')</p></dd>
 
-    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>
 
-    <p>Opens a shiny app for interactively viewing the results</p>
-    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>
+<dt>diagnosePlp</dt>
+<dd><p>The output of diagnosePlp()</p></dd>
 
+</dl></div>
+    <div id="value">
+    <h2>Value</h2>
+    
+
+<p>Opens a shiny app for interactively viewing the results</p>
+    </div>
+    <div id="details">
+    <h2>Details</h2>
     <p>Either the result of runPlp and view the plots</p>
+    </div>
 
   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
-    <nav id="toc" data-toggle="toc" class="sticky-top">
-      <h2 data-toc-skip>Contents</h2>
-    </nav>
-  </div>
+    <nav id="toc" data-toggle="toc" class="sticky-top"><h2 data-toc-skip>Contents</h2>
+    </nav></div>
 </div>
 
 
-      <footer>
-      <div class="copyright">
-  <p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek.</p>
+      <footer><div class="copyright">
+  <p></p><p>Developed by Jenna Reps, Martijn Schuemie, Marc Suchard, Patrick Ryan, Peter Rijnbeek, Egill Fridgeirsson.</p>
 </div>
 
 <div class="pkgdown">
-  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
+  <p></p><p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.7.</p>
 </div>
 
-      </footer>
-   </div>
+      </footer></div>
 
   
 
 
-  </body>
-</html>
+  
 
+  </body></html>
 
diff --git a/docs/sitemap.xml b/docs/sitemap.xml
new file mode 100644
index 000000000..5c677f919
--- /dev/null
+++ b/docs/sitemap.xml
@@ -0,0 +1,456 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url>
+    <loc>/404.html</loc>
+  </url>
+  <url>
+    <loc>/articles/AddingCustomFeatureEngineering.html</loc>
+  </url>
+  <url>
+    <loc>/articles/AddingCustomModels.html</loc>
+  </url>
+  <url>
+    <loc>/articles/AddingCustomSamples.html</loc>
+  </url>
+  <url>
+    <loc>/articles/AddingCustomSplitting.html</loc>
+  </url>
+  <url>
+    <loc>/articles/BenchmarkTasks.html</loc>
+  </url>
+  <url>
+    <loc>/articles/BestPractices.html</loc>
+  </url>
+  <url>
+    <loc>/articles/BuildingMultiplePredictiveModels.html</loc>
+  </url>
+  <url>
+    <loc>/articles/BuildingPredictiveModels.html</loc>
+  </url>
+  <url>
+    <loc>/articles/ClinicalModels.html</loc>
+  </url>
+  <url>
+    <loc>/articles/ConstrainedPredictors.html</loc>
+  </url>
+  <url>
+    <loc>/articles/CreatingLearningCurves.html</loc>
+  </url>
+  <url>
+    <loc>/articles/CreatingNetworkStudies.html</loc>
+  </url>
+  <url>
+    <loc>/articles/InstallationGuide.html</loc>
+  </url>
+  <url>
+    <loc>/articles/Videos.html</loc>
+  </url>
+  <url>
+    <loc>/articles/index.html</loc>
+  </url>
+  <url>
+    <loc>/authors.html</loc>
+  </url>
+  <url>
+    <loc>/index.html</loc>
+  </url>
+  <url>
+    <loc>/news/index.html</loc>
+  </url>
+  <url>
+    <loc>/reference/MapIds.html</loc>
+  </url>
+  <url>
+    <loc>/reference/PatientLevelPrediction.html</loc>
+  </url>
+  <url>
+    <loc>/reference/accuracy.html</loc>
+  </url>
+  <url>
+    <loc>/reference/addDiagnosePlpToDatabase.html</loc>
+  </url>
+  <url>
+    <loc>/reference/addMultipleDiagnosePlpToDatabase.html</loc>
+  </url>
+  <url>
+    <loc>/reference/addMultipleRunPlpToDatabase.html</loc>
+  </url>
+  <url>
+    <loc>/reference/addRunPlpToDatabase.html</loc>
+  </url>
+  <url>
+    <loc>/reference/averagePrecision.html</loc>
+  </url>
+  <url>
+    <loc>/reference/brierScore.html</loc>
+  </url>
+  <url>
+    <loc>/reference/calibrationLine.html</loc>
+  </url>
+  <url>
+    <loc>/reference/computeAuc.html</loc>
+  </url>
+  <url>
+    <loc>/reference/computeGridPerformance.html</loc>
+  </url>
+  <url>
+    <loc>/reference/configurePython.html</loc>
+  </url>
+  <url>
+    <loc>/reference/covariateSummary.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createCohortCovariateSettings.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createDatabaseDetails.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createDatabaseList.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createDatabaseSchemaSettings.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createDefaultExecuteSettings.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createDefaultSplitSetting.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createExecuteSettings.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createFeatureEngineeringSettings.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createLearningCurve.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createLogSettings.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createModelDesign.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createPlpResultTables.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createPreprocessSettings.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createRandomForestFeatureSelection.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createRestrictPlpDataSettings.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createSampleSettings.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createSplineSettings.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createStratifiedImputationSettings.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createStudyPopulation.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createStudyPopulationSettings.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createTempModelLoc.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createUnivariateFeatureSelection.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createValidationDesign.html</loc>
+  </url>
+  <url>
+    <loc>/reference/createValidationSettings.html</loc>
+  </url>
+  <url>
+    <loc>/reference/diagnoseMultiplePlp.html</loc>
+  </url>
+  <url>
+    <loc>/reference/diagnosePlp.html</loc>
+  </url>
+  <url>
+    <loc>/reference/diagnosticOddsRatio.html</loc>
+  </url>
+  <url>
+    <loc>/reference/evaluatePlp.html</loc>
+  </url>
+  <url>
+    <loc>/reference/externalValidateDbPlp.html</loc>
+  </url>
+  <url>
+    <loc>/reference/extractDatabaseToCsv.html</loc>
+  </url>
+  <url>
+    <loc>/reference/f1Score.html</loc>
+  </url>
+  <url>
+    <loc>/reference/falseDiscoveryRate.html</loc>
+  </url>
+  <url>
+    <loc>/reference/falseNegativeRate.html</loc>
+  </url>
+  <url>
+    <loc>/reference/falseOmissionRate.html</loc>
+  </url>
+  <url>
+    <loc>/reference/falsePositiveRate.html</loc>
+  </url>
+  <url>
+    <loc>/reference/fitPlp.html</loc>
+  </url>
+  <url>
+    <loc>/reference/getCalibrationSummary.html</loc>
+  </url>
+  <url>
+    <loc>/reference/getCohortCovariateData.html</loc>
+  </url>
+  <url>
+    <loc>/reference/getDemographicSummary.html</loc>
+  </url>
+  <url>
+    <loc>/reference/getPlpData.html</loc>
+  </url>
+  <url>
+    <loc>/reference/getPredictionDistribution.html</loc>
+  </url>
+  <url>
+    <loc>/reference/getPredictionDistribution_binary.html</loc>
+  </url>
+  <url>
+    <loc>/reference/getThresholdSummary.html</loc>
+  </url>
+  <url>
+    <loc>/reference/getThresholdSummary_binary.html</loc>
+  </url>
+  <url>
+    <loc>/reference/ici.html</loc>
+  </url>
+  <url>
+    <loc>/reference/index.html</loc>
+  </url>
+  <url>
+    <loc>/reference/insertCsvToDatabase.html</loc>
+  </url>
+  <url>
+    <loc>/reference/insertModelDesignInDatabase.html</loc>
+  </url>
+  <url>
+    <loc>/reference/insertResultsToSqlite.html</loc>
+  </url>
+  <url>
+    <loc>/reference/listAppend.html</loc>
+  </url>
+  <url>
+    <loc>/reference/listCartesian.html</loc>
+  </url>
+  <url>
+    <loc>/reference/loadPlpAnalysesJson.html</loc>
+  </url>
+  <url>
+    <loc>/reference/loadPlpData.html</loc>
+  </url>
+  <url>
+    <loc>/reference/loadPlpModel.html</loc>
+  </url>
+  <url>
+    <loc>/reference/loadPlpResult.html</loc>
+  </url>
+  <url>
+    <loc>/reference/loadPlpShareable.html</loc>
+  </url>
+  <url>
+    <loc>/reference/loadPrediction.html</loc>
+  </url>
+  <url>
+    <loc>/reference/migrateDataModel.html</loc>
+  </url>
+  <url>
+    <loc>/reference/modelBasedConcordance.html</loc>
+  </url>
+  <url>
+    <loc>/reference/negativeLikelihoodRatio.html</loc>
+  </url>
+  <url>
+    <loc>/reference/negativePredictiveValue.html</loc>
+  </url>
+  <url>
+    <loc>/reference/outcomeSurvivalPlot.html</loc>
+  </url>
+  <url>
+    <loc>/reference/pfi.html</loc>
+  </url>
+  <url>
+    <loc>/reference/plotDemographicSummary.html</loc>
+  </url>
+  <url>
+    <loc>/reference/plotF1Measure.html</loc>
+  </url>
+  <url>
+    <loc>/reference/plotGeneralizability.html</loc>
+  </url>
+  <url>
+    <loc>/reference/plotLearningCurve.html</loc>
+  </url>
+  <url>
+    <loc>/reference/plotPlp.html</loc>
+  </url>
+  <url>
+    <loc>/reference/plotPrecisionRecall.html</loc>
+  </url>
+  <url>
+    <loc>/reference/plotPredictedPDF.html</loc>
+  </url>
+  <url>
+    <loc>/reference/plotPredictionDistribution.html</loc>
+  </url>
+  <url>
+    <loc>/reference/plotPreferencePDF.html</loc>
+  </url>
+  <url>
+    <loc>/reference/plotSmoothCalibration.html</loc>
+  </url>
+  <url>
+    <loc>/reference/plotSparseCalibration.html</loc>
+  </url>
+  <url>
+    <loc>/reference/plotSparseCalibration2.html</loc>
+  </url>
+  <url>
+    <loc>/reference/plotSparseRoc.html</loc>
+  </url>
+  <url>
+    <loc>/reference/plotVariableScatterplot.html</loc>
+  </url>
+  <url>
+    <loc>/reference/plpDataSimulationProfile.html</loc>
+  </url>
+  <url>
+    <loc>/reference/positiveLikelihoodRatio.html</loc>
+  </url>
+  <url>
+    <loc>/reference/positivePredictiveValue.html</loc>
+  </url>
+  <url>
+    <loc>/reference/predictCyclops.html</loc>
+  </url>
+  <url>
+    <loc>/reference/predictPlp.html</loc>
+  </url>
+  <url>
+    <loc>/reference/preprocessData.html</loc>
+  </url>
+  <url>
+    <loc>/reference/recalibratePlp.html</loc>
+  </url>
+  <url>
+    <loc>/reference/recalibratePlpRefit.html</loc>
+  </url>
+  <url>
+    <loc>/reference/runMultiplePlp.html</loc>
+  </url>
+  <url>
+    <loc>/reference/runPlp.html</loc>
+  </url>
+  <url>
+    <loc>/reference/savePlpAnalysesJson.html</loc>
+  </url>
+  <url>
+    <loc>/reference/savePlpData.html</loc>
+  </url>
+  <url>
+    <loc>/reference/savePlpModel.html</loc>
+  </url>
+  <url>
+    <loc>/reference/savePlpResult.html</loc>
+  </url>
+  <url>
+    <loc>/reference/savePlpShareable.html</loc>
+  </url>
+  <url>
+    <loc>/reference/savePrediction.html</loc>
+  </url>
+  <url>
+    <loc>/reference/sensitivity.html</loc>
+  </url>
+  <url>
+    <loc>/reference/setAdaBoost.html</loc>
+  </url>
+  <url>
+    <loc>/reference/setCoxModel.html</loc>
+  </url>
+  <url>
+    <loc>/reference/setDecisionTree.html</loc>
+  </url>
+  <url>
+    <loc>/reference/setGradientBoostingMachine.html</loc>
+  </url>
+  <url>
+    <loc>/reference/setIterativeHardThresholding.html</loc>
+  </url>
+  <url>
+    <loc>/reference/setKNN.html</loc>
+  </url>
+  <url>
+    <loc>/reference/setLassoLogisticRegression.html</loc>
+  </url>
+  <url>
+    <loc>/reference/setLightGBM.html</loc>
+  </url>
+  <url>
+    <loc>/reference/setMLP.html</loc>
+  </url>
+  <url>
+    <loc>/reference/setNaiveBayes.html</loc>
+  </url>
+  <url>
+    <loc>/reference/setPythonEnvironment.html</loc>
+  </url>
+  <url>
+    <loc>/reference/setRandomForest.html</loc>
+  </url>
+  <url>
+    <loc>/reference/setSVM.html</loc>
+  </url>
+  <url>
+    <loc>/reference/simulatePlpData.html</loc>
+  </url>
+  <url>
+    <loc>/reference/sklearnFromJson.html</loc>
+  </url>
+  <url>
+    <loc>/reference/sklearnToJson.html</loc>
+  </url>
+  <url>
+    <loc>/reference/specificity.html</loc>
+  </url>
+  <url>
+    <loc>/reference/splitData.html</loc>
+  </url>
+  <url>
+    <loc>/reference/toSparseM.html</loc>
+  </url>
+  <url>
+    <loc>/reference/validateExternal.html</loc>
+  </url>
+  <url>
+    <loc>/reference/validateMultiplePlp.html</loc>
+  </url>
+  <url>
+    <loc>/reference/viewDatabaseResultPlp.html</loc>
+  </url>
+  <url>
+    <loc>/reference/viewMultiplePlp.html</loc>
+  </url>
+  <url>
+    <loc>/reference/viewPlp.html</loc>
+  </url>
+</urlset>
diff --git a/extras/PatientLevelPrediction.pdf b/extras/PatientLevelPrediction.pdf
new file mode 100644
index 000000000..836c33b35
Binary files /dev/null and b/extras/PatientLevelPrediction.pdf differ
diff --git a/extras/checkPlpInstallation.R b/extras/checkPlpInstallation.R
index 5f0409c02..55de984f3 100644
--- a/extras/checkPlpInstallation.R
+++ b/extras/checkPlpInstallation.R
@@ -30,7 +30,7 @@ checkPlpInstallation <- function(connectionDetails=NULL, python=T) {
   sampleSize <- 2000
   plpData <- simulatePlpData(plpDataSimulationProfile, n = sampleSize)
   
-  popSettings <- createStudyPopulationSettings(
+  populationSettings <- createStudyPopulationSettings(
     firstExposureOnly = FALSE,
     washoutPeriod = 0,
     removeSubjectsWithPriorOutcome = FALSE,
@@ -47,7 +47,7 @@ checkPlpInstallation <- function(connectionDetails=NULL, python=T) {
   population <- tryCatch({createStudyPopulation(
     plpData = plpData,
     outcomeId = 2,
-    populationSettings = popSettings
+    populationSettings = populationSettings
   )},
     error = function(e) {
       return(0)
diff --git a/extras/testAllClassifiers.R b/extras/testAllClassifiers.R
index dcc6ac44c..682232c49 100644
--- a/extras/testAllClassifiers.R
+++ b/extras/testAllClassifiers.R
@@ -1,3 +1,4 @@
+library(PatientLevelPrediction)
 connectionDetails <- Eunomia::getEunomiaConnectionDetails()
 Eunomia::createCohorts(connectionDetails)
 
@@ -20,7 +21,7 @@ databaseDetails <- createDatabaseDetails(
   cohortTable = "cohort", 
   outcomeDatabaseSchema = "main", 
   outcomeTable =  "cohort",
-  cohortId = 1, 
+  targetId = 1, 
   outcomeIds = 3, #make this ids
   cdmVersion = 5)
 
@@ -67,7 +68,7 @@ plpResultEunomia9 <- PatientLevelPrediction::runPlp(
   modelSettings = setKNN(), 
   logSettings = createLogSettings(), 
   executeSettings = createDefaultExecuteSettings(), 
-  saveDirectory = file.path(tempdir(), 'EunomiaTest8')
+  saveDirectory = file.path(tempdir(), 'EunomiaTest9')
 )
 
 plpResultEunomia8 <- PatientLevelPrediction::runPlp(
@@ -83,8 +84,9 @@ plpResultEunomia8 <- PatientLevelPrediction::runPlp(
   modelSettings = setSVM(), 
   logSettings = createLogSettings(), 
   executeSettings = createDefaultExecuteSettings(), 
-  saveDirectory = file.path(tempdir(), 'EunomiaTest7')
+  saveDirectory = file.path(tempdir(), 'EunomiaTest8')
 )
+# issue with loading json - fixed by saving as pickle
 
 plpResultEunomia7 <- PatientLevelPrediction::runPlp(
   plpData = plpDataEunomia, 
@@ -99,7 +101,7 @@ plpResultEunomia7 <- PatientLevelPrediction::runPlp(
   modelSettings = setRandomForest(), 
   logSettings = createLogSettings(), 
   executeSettings = createDefaultExecuteSettings(), 
-  saveDirectory = file.path(tempdir(), 'EunomiaTest6')
+  saveDirectory = file.path(tempdir(), 'EunomiaTest7')
 )
 
 plpResultEunomia6 <- PatientLevelPrediction::runPlp(
@@ -112,11 +114,12 @@ plpResultEunomia6 <- PatientLevelPrediction::runPlp(
   sampleSettings = createSampleSettings(), 
   featureEngineeringSettings = createFeatureEngineeringSettings(), 
   preprocessSettings = createPreprocessSettings(), 
-  modelSettings = setMLP(), 
+  modelSettings = setMLP(hiddenLayerSizes = list(c(10))), 
   logSettings = createLogSettings(), 
   executeSettings = createDefaultExecuteSettings(), 
-  saveDirectory = file.path(tempdir(), 'EunomiaTest5')
+  saveDirectory = file.path(tempdir(), 'EunomiaTest6')
 )
+# invalid hiddenLayerSizes can cause error 
 
 plpResultEunomia5 <- PatientLevelPrediction::runPlp(
   plpData = plpDataEunomia, 
@@ -131,9 +134,9 @@ plpResultEunomia5 <- PatientLevelPrediction::runPlp(
   modelSettings = setNaiveBayes(), 
   logSettings = createLogSettings(), 
   executeSettings = createDefaultExecuteSettings(), 
-  saveDirectory = file.path(tempdir(), 'EunomiaTest4')
+  saveDirectory = file.path(tempdir(), 'EunomiaTest5')
 )
-
+# worked
 
 plpResultEunomia3 <- PatientLevelPrediction::runPlp(
   plpData = plpDataEunomia, 
@@ -148,8 +151,9 @@ plpResultEunomia3 <- PatientLevelPrediction::runPlp(
   modelSettings = setAdaBoost(), 
   logSettings = createLogSettings(), 
   executeSettings = createDefaultExecuteSettings(), 
-  saveDirectory = file.path(tempdir(), 'EunomiaTest2')
+  saveDirectory = file.path(tempdir(), 'EunomiaTest3')
 )
+# worked
 
 plpResultEunomia4 <- PatientLevelPrediction::runPlp(
   plpData = plpDataEunomia, 
@@ -161,12 +165,12 @@ plpResultEunomia4 <- PatientLevelPrediction::runPlp(
   sampleSettings = createSampleSettings(), 
   featureEngineeringSettings = createFeatureEngineeringSettings(), 
   preprocessSettings = createPreprocessSettings(), 
-  modelSettings = setDecisionTree(maxFeatures = list(50,'auto', NULL)), 
+  modelSettings = setDecisionTree(maxFeatures = list(50,'sqrt', NULL)), 
   logSettings = createLogSettings(), 
   executeSettings = createDefaultExecuteSettings(), 
-  saveDirectory = file.path(tempdir(), 'EunomiaTest3')
+  saveDirectory = file.path(tempdir(), 'EunomiaTest4')
 )
-
+# DT error!
 
 
 plpResultEunomia2 <- PatientLevelPrediction::runPlp(
@@ -184,10 +188,10 @@ plpResultEunomia2 <- PatientLevelPrediction::runPlp(
     nthread = c(10),
     earlyStopRound = c(25), 
     maxDepth = c(4), 
-    minRows = c(5), 
     learnRate = c(0.2) 
   ), 
   logSettings = createLogSettings(), 
   executeSettings = createDefaultExecuteSettings(), 
-  saveDirectory = file.path(tempdir(), 'EunomiaTest')
+  saveDirectory = file.path(tempdir(), 'EunomiaTest2')
 )
+# worked
\ No newline at end of file
diff --git a/inst/doc/AddingCustomAlgorithms.tex b/inst/doc/AddingCustomAlgorithms.tex
deleted file mode 100644
index 125ff3257..000000000
--- a/inst/doc/AddingCustomAlgorithms.tex
+++ /dev/null
@@ -1,617 +0,0 @@
-% Options for packages loaded elsewhere
-\PassOptionsToPackage{unicode}{hyperref}
-\PassOptionsToPackage{hyphens}{url}
-%
-\documentclass[
-]{article}
-\usepackage{lmodern}
-\usepackage{amssymb,amsmath}
-\usepackage{ifxetex,ifluatex}
-\ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex
-  \usepackage[T1]{fontenc}
-  \usepackage[utf8]{inputenc}
-  \usepackage{textcomp} % provide euro and other symbols
-\else % if luatex or xetex
-  \usepackage{unicode-math}
-  \defaultfontfeatures{Scale=MatchLowercase}
-  \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
-\fi
-% Use upquote if available, for straight quotes in verbatim environments
-\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
-\IfFileExists{microtype.sty}{% use microtype if available
-  \usepackage[]{microtype}
-  \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
-}{}
-\makeatletter
-\@ifundefined{KOMAClassName}{% if non-KOMA class
-  \IfFileExists{parskip.sty}{%
-    \usepackage{parskip}
-  }{% else
-    \setlength{\parindent}{0pt}
-    \setlength{\parskip}{6pt plus 2pt minus 1pt}}
-}{% if KOMA class
-  \KOMAoptions{parskip=half}}
-\makeatother
-\usepackage{xcolor}
-\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
-\IfFileExists{bookmark.sty}{\usepackage{bookmark}}{\usepackage{hyperref}}
-\hypersetup{
-  pdftitle={Adding Custom Patient-Level Prediction Algorithms},
-  pdfauthor={Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek},
-  hidelinks,
-  pdfcreator={LaTeX via pandoc}}
-\urlstyle{same} % disable monospaced font for URLs
-\usepackage[margin=1in]{geometry}
-\usepackage{color}
-\usepackage{fancyvrb}
-\newcommand{\VerbBar}{|}
-\newcommand{\VERB}{\Verb[commandchars=\\\{\}]}
-\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
-% Add ',fontsize=\small' for more characters per line
-\usepackage{framed}
-\definecolor{shadecolor}{RGB}{248,248,248}
-\newenvironment{Shaded}{\begin{snugshade}}{\end{snugshade}}
-\newcommand{\AlertTok}[1]{\textcolor[rgb]{0.94,0.16,0.16}{#1}}
-\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.77,0.63,0.00}{#1}}
-\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\BuiltInTok}[1]{#1}
-\newcommand{\CharTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
-\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
-\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{#1}}
-\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\ErrorTok}[1]{\textcolor[rgb]{0.64,0.00,0.00}{\textbf{#1}}}
-\newcommand{\ExtensionTok}[1]{#1}
-\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\ImportTok}[1]{#1}
-\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
-\newcommand{\NormalTok}[1]{#1}
-\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.81,0.36,0.00}{\textbf{#1}}}
-\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{#1}}
-\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
-\newcommand{\RegionMarkerTok}[1]{#1}
-\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\StringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\usepackage{graphicx,grffile}
-\makeatletter
-\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
-\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
-\makeatother
-% Scale images if necessary, so that they will not overflow the page
-% margins by default, and it is still possible to overwrite the defaults
-% using explicit options in \includegraphics[width, height, ...]{}
-\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
-% Set default figure placement to htbp
-\makeatletter
-\def\fps@figure{htbp}
-\makeatother
-\setlength{\emergencystretch}{3em} % prevent overfull lines
-\providecommand{\tightlist}{%
-  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
-\setcounter{secnumdepth}{5}
-\usepackage{fancyhdr}
-\pagestyle{fancy}
-\fancyhead{}
-\fancyhead[CO,CE]{Custom Patient-Level Prediction Algorithms}
-\fancyfoot[CO,CE]{PatientLevelPrediction Package Version 3.1.0}
-\fancyfoot[LE,RO]{\thepage}
-\renewcommand{\headrulewidth}{0.4pt}
-\renewcommand{\footrulewidth}{0.4pt}
-
-\title{Adding Custom Patient-Level Prediction Algorithms}
-\author{Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek}
-\date{2020-06-03}
-
-\begin{document}
-\maketitle
-
-{
-\setcounter{tocdepth}{2}
-\tableofcontents
-}
-\hypertarget{introduction}{%
-\section{Introduction}\label{introduction}}
-
-This vignette describes how you can add your own custom algorithms in
-the Observational Health Data Sciencs and Informatics (OHDSI)
-\href{http://github.com/OHDSI/PatientLevelPrediction}{\texttt{PatientLevelPrediction}}
-package. This allows you to fully leverage the OHDSI
-PatientLevelPrediction framework for model development and validation.
-This vignette assumes you have read and are comfortable with building
-single patient level prediction models as described in the
-\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf}{\texttt{BuildingPredictiveModels}
-vignette}.
-
-\textbf{We invite you to share your new algorithms with the OHDSI
-community through our
-\href{http://github.com/OHDSI/PatientLevelPrediction}{GitHub
-repository}.}
-
-\hypertarget{algorithm-code-structure}{%
-\section{Algorithm Code Structure}\label{algorithm-code-structure}}
-
-Each algorithm in the package should be implemented in its own
-\textless Name\textgreater.R file, e.g.~KNN.R, containing a
-set\textless Name\textgreater{} function and a
-fit\textless Name\textgreater{} function. Furthermore, a corresponding
-predict function in predict.R is needed (if there isn't one available
-that would work, see example at the end of the document). We will now
-describe each of these functions in more detail below.
-
-\hypertarget{set}{%
-\subsection{Set}\label{set}}
-
-The set\textless Name\textgreater{} is a function that takes as input
-the different hyper-parameter values to do a grid search when training.
-The output of the functions needs to be a list as class
-\texttt{modelSettings} containing:
-
-\begin{itemize}
-\tightlist
-\item
-  param - all the combinations of the hyper-parameter values input
-\item
-  model - a string specifying what function to call to fit the model
-\item
-  name - a string containing the name of the model.
-\end{itemize}
-
-For example, if you were adding a model called madeUp that has two
-hyper-parameters then the set function should be:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{setMadeUp <-}\StringTok{ }\ControlFlowTok{function}\NormalTok{(}\DataTypeTok{a=}\DecValTok{1}\NormalTok{, }\DataTypeTok{b=}\DecValTok{2}\NormalTok{, }\DataTypeTok{seed=}\OtherTok{NULL}\NormalTok{)\{}
-  \CommentTok{# add input checks here...}
-  
-  \CommentTok{# now create list of all combinations:}
-\NormalTok{  result <-}\StringTok{ }\KeywordTok{list}\NormalTok{(}\DataTypeTok{model=}\StringTok{'fitMadeUp'}\NormalTok{, }\CommentTok{# this will be called to train the made up model}
-                 \DataTypeTok{param=} \KeywordTok{split}\NormalTok{(}\KeywordTok{expand.grid}\NormalTok{(}\DataTypeTok{a=}\NormalTok{a, }
-                                          \DataTypeTok{b=}\NormalTok{b,}
-                                          \DataTypeTok{seed=}\KeywordTok{ifelse}\NormalTok{(}\KeywordTok{is.null}\NormalTok{(seed),}\StringTok{'NULL'}\NormalTok{, seed)),}
-                              \DecValTok{1}\OperatorTok{:}\NormalTok{(}\KeywordTok{length}\NormalTok{(a)}\OperatorTok{*}\KeywordTok{length}\NormalTok{(b)  )),}
-                 \DataTypeTok{name=}\StringTok{'Made Up Algorithm'}
-\NormalTok{  )}
-  \KeywordTok{class}\NormalTok{(result) <-}\StringTok{ 'modelSettings'} 
-  
-  \KeywordTok{return}\NormalTok{(result)}
-\NormalTok{\}}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{fit}{%
-\subsection{Fit}\label{fit}}
-
-This function should train your custom model for each parameter entry,
-pick the best parameters and train a final model for that setting.
-
-The fit\textless Model\textgreater{} should have as inputs:
-
-\begin{itemize}
-\tightlist
-\item
-  population - the study popualation the model is being developed on
-\item
-  plpData - the plpData object
-\item
-  param - the hyper-parameters as a list of all combinations
-\item
-  quiet - T or F indicating whether to output progress
-\item
-  outcomeId - the outcome id
-\item
-  cohortId - the target population id
-\end{itemize}
-
-The fit function should return a list of class \texttt{plpModel} with
-the following objects:
-
-\begin{itemize}
-\tightlist
-\item
-  model - a trained model
-\item
-  modelSettings - a list containing the model and input param
-\item
-  trainCVAuc - a value with the train AUC value
-\item
-  hyperParamSearch - a dataframe with the hyperparameter grid and
-  corresponding AUCs
-\item
-  metaData - the metaData from the plpData object
-\item
-  populationSettings - the settings used to create the population and
-  define the time-at-risk
-\item
-  outcomeId - the outcomeId being predicted
-\item
-  cohortId - the cohortId corresponding to the target cohort
-\item
-  varImp - a dataframe with the covaraites and a measure of importance
-\item
-  trainingTime - how long it took to develop/evaluate the model
-\item
-  covariateMap - if the plpData are converted to a matrix for model
-  compatibility this tells us what covariate each row in the matrix
-  correpsonds to and is need when implementing the model on new data
-\end{itemize}
-
-The plpModel returned by fit also has a type attribute, this points to
-the predict function, for example
-\texttt{attr(result,\ \textquotesingle{}type\textquotesingle{})\ \textless{}-\ \textquotesingle{}madeup\textquotesingle{}}
-means when the model is applied to new data, the `predict.madeup'
-function in Predict.R is called. if this doesnt exist, then the model
-will fail. Another attribute is the predictionType
-\texttt{attr(result,\ \textquotesingle{}predictionType\textquotesingle{})\ \textless{}-\ \textquotesingle{}binary\textquotesingle{}}
-this is currently not needed but may be important in the future when we
-expand to regression or multiclass classification.
-
-For example:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{fitMadeUp <-}\StringTok{ }\ControlFlowTok{function}\NormalTok{(population, plpData, param, }\DataTypeTok{quiet=}\NormalTok{F,}
-\NormalTok{                        outcomeId, cohortId, ...)\{}
-  
-  \CommentTok{# **************** code to train the model here}
-  \CommentTok{# trainedModel <- this code should apply each hyper-parameter using the cross validation}
-  \CommentTok{#                 then pick out the best hyper-parameter setting}
-  \CommentTok{#                 and finally fit a model on the whole train data using the }
-  \CommentTok{#                 optimal hyper-parameter settings}
-  \CommentTok{# ****************}
-  
-  \CommentTok{# construct the standard output for a model:}
-\NormalTok{  result <-}\StringTok{ }\KeywordTok{list}\NormalTok{(}\DataTypeTok{model =}\NormalTok{ trainedModel,}
-                 \DataTypeTok{modelSettings =} \KeywordTok{list}\NormalTok{(}\DataTypeTok{model=}\StringTok{'made_up'}\NormalTok{, }\DataTypeTok{modelParameters=}\NormalTok{param),}
-                 \DataTypeTok{trainCVAuc =} \OtherTok{NULL}\NormalTok{,}
-                 \DataTypeTok{hyperParamSearch =}\NormalTok{ hyperSummary,}
-                 \DataTypeTok{metaData =}\NormalTok{ plpData}\OperatorTok{$}\NormalTok{metaData,}
-                 \DataTypeTok{populationSettings =} \KeywordTok{attr}\NormalTok{(population, }\StringTok{'metaData'}\NormalTok{),}
-                 \DataTypeTok{outcomeId=}\NormalTok{outcomeId,}\CommentTok{# can use populationSettings$outcomeId?}
-                 \DataTypeTok{cohortId=}\NormalTok{cohortId,}
-                 \DataTypeTok{varImp =} \OtherTok{NULL}\NormalTok{,}
-                 \DataTypeTok{trainingTime=}\NormalTok{comp,}
-                 \DataTypeTok{covariateMap=}\NormalTok{result}\OperatorTok{$}\NormalTok{map}
-\NormalTok{  )}
-  \KeywordTok{class}\NormalTok{(result) <-}\StringTok{ 'plpModel'}
-  \KeywordTok{attr}\NormalTok{(result, }\StringTok{'type'}\NormalTok{) <-}\StringTok{ 'madeup'}
-  \KeywordTok{attr}\NormalTok{(result, }\StringTok{'predictionType'}\NormalTok{) <-}\StringTok{ 'binary'}
-  \KeywordTok{return}\NormalTok{(result)}
-    
-\NormalTok{\}}
-\end{Highlighting}
-\end{Shaded}
-
-You could make the fitMadeUp function cleaner by adding helper function
-in the MadeUp.R file that are called by the fit function. As the end of
-the fit function specified
-\texttt{attr(result,\ \textquotesingle{}type\textquotesingle{})\ \textless{}-\ \textquotesingle{}madeup\textquotesingle{}}
-we also need to make sure there is a \texttt{predict.madeup} function in
-Predict.R:
-
-\hypertarget{predict}{%
-\subsection{Predict}\label{predict}}
-
-The prediction function takes as input the plpModel returned by fit, a
-population and corresponding plpData. It returns a data.frame with the
-columns:
-
-\begin{itemize}
-\tightlist
-\item
-  rowId - the id for each person in the population
-\item
-  value - the predicted risk from the plpModel
-\end{itemize}
-
-If the population contains the columns outcomeCount and indexes, then
-these are also in the output.
-
-For example:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{predict.madeup <-}\StringTok{ }\ControlFlowTok{function}\NormalTok{(plpModel,population, plpData, ...)\{ }
-
-  \CommentTok{# ************* code to do prediction for each rowId in population}
-  \CommentTok{# prediction <- code to do prediction here returning columns: rowId }
-  \CommentTok{#               and value (predicted risk)}
-  \CommentTok{#**************}
-  
-\NormalTok{  prediction <-}\StringTok{ }\KeywordTok{merge}\NormalTok{(population, prediction, }\DataTypeTok{by=}\StringTok{'rowId'}\NormalTok{)}
-\NormalTok{  prediction <-}\StringTok{ }\NormalTok{prediction[,}\KeywordTok{colnames}\NormalTok{(prediction)}\OperatorTok{%in%}\KeywordTok{c}\NormalTok{(}\StringTok{'rowId'}\NormalTok{,}\StringTok{'outcomeCount'}\NormalTok{,}
-                                                      \StringTok{'indexes'}\NormalTok{, }\StringTok{'value'}\NormalTok{)] }
-  \KeywordTok{attr}\NormalTok{(prediction, }\StringTok{"metaData"}\NormalTok{) <-}\StringTok{ }\KeywordTok{list}\NormalTok{(}\DataTypeTok{predictionType =} \StringTok{"binary"}\NormalTok{) }
-  \KeywordTok{return}\NormalTok{(prediction)}
-  
-\NormalTok{\}}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{algorithm-example}{%
-\section{Algorithm Example}\label{algorithm-example}}
-
-Below a fully functional algorithm example is given, however we highly
-recommend you to have a look at the available algorithms in the package.
-
-\hypertarget{set-1}{%
-\subsection{Set}\label{set-1}}
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{setMadeUp <-}\StringTok{ }\ControlFlowTok{function}\NormalTok{(}\DataTypeTok{a=}\DecValTok{1}\NormalTok{, }\DataTypeTok{b=}\DecValTok{2}\NormalTok{, }\DataTypeTok{seed=}\OtherTok{NULL}\NormalTok{)\{}
-  \CommentTok{# check a is valid positive value}
-  \ControlFlowTok{if}\NormalTok{(}\KeywordTok{missing}\NormalTok{(a))\{}
-    \KeywordTok{stop}\NormalTok{(}\StringTok{'a must be input'}\NormalTok{)}
-\NormalTok{  \}}
-  \ControlFlowTok{if}\NormalTok{(}\OperatorTok{!}\KeywordTok{class}\NormalTok{(a)}\OperatorTok{%in%}\KeywordTok{c}\NormalTok{(}\StringTok{'numeric'}\NormalTok{,}\StringTok{'integer'}\NormalTok{)\{}
-    \KeywordTok{stop}\NormalTok{(}\StringTok{'a must be numeric'}\NormalTok{)}
-\NormalTok{  \}}
-  \ControlFlowTok{if}\NormalTok{(a }\OperatorTok{<}\StringTok{ }\DecValTok{0}\NormalTok{)\{}
-    \KeywordTok{stop}\NormalTok{(}\StringTok{'a must be positive'}\NormalTok{)}
-\NormalTok{  \}}
-  \CommentTok{# check b is numeric}
-  \ControlFlowTok{if}\NormalTok{(}\KeywordTok{missing}\NormalTok{(b))\{}
-    \KeywordTok{stop}\NormalTok{(}\StringTok{'b must be input'}\NormalTok{)}
-\NormalTok{  \}}
-  \ControlFlowTok{if}\NormalTok{(}\OperatorTok{!}\KeywordTok{class}\NormalTok{(b)}\OperatorTok{%in%}\KeywordTok{c}\NormalTok{(}\StringTok{'numeric'}\NormalTok{,}\StringTok{'integer'}\NormalTok{)\{}
-    \KeywordTok{stop}\NormalTok{(}\StringTok{'b must be numeric'}\NormalTok{)}
-\NormalTok{  \}}
-  
-  \CommentTok{# now create list of all combinations:}
-\NormalTok{  result <-}\StringTok{ }\KeywordTok{list}\NormalTok{(}\DataTypeTok{model=}\StringTok{'fitMadeUp'}\NormalTok{, }
-                 \DataTypeTok{param=} \KeywordTok{split}\NormalTok{(}\KeywordTok{expand.grid}\NormalTok{(}\DataTypeTok{a=}\NormalTok{a, }
-                                          \DataTypeTok{b=}\NormalTok{b,}
-                                          \DataTypeTok{seed=}\KeywordTok{ifelse}\NormalTok{(}\KeywordTok{is.null}\NormalTok{(seed),}\StringTok{'NULL'}\NormalTok{, seed)),}
-                              \DecValTok{1}\OperatorTok{:}\NormalTok{(}\KeywordTok{length}\NormalTok{(a)}\OperatorTok{*}\KeywordTok{length}\NormalTok{(b)  )),}
-                 \DataTypeTok{name=}\StringTok{'Made Up Algorithm'}
-\NormalTok{  )}
-  \KeywordTok{class}\NormalTok{(result) <-}\StringTok{ 'modelSettings'} 
-  
-  \KeywordTok{return}\NormalTok{(result)}
-    
-  
-\ErrorTok{\}}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{fit-1}{%
-\subsection{Fit}\label{fit-1}}
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{fitMadeUp <-}\StringTok{ }\ControlFlowTok{function}\NormalTok{(population, plpData, param, }\DataTypeTok{quiet=}\NormalTok{F,}
-\NormalTok{                        outcomeId, cohortId, ...)\{}
-    \ControlFlowTok{if}\NormalTok{(}\OperatorTok{!}\NormalTok{quiet)}
-    \KeywordTok{writeLines}\NormalTok{(}\StringTok{'Training Made Up model'}\NormalTok{)}
-  
-  \ControlFlowTok{if}\NormalTok{(param[[}\DecValTok{1}\NormalTok{]]}\OperatorTok{$}\NormalTok{seed}\OperatorTok{!=}\StringTok{'NULL'}\NormalTok{)}
-    \KeywordTok{set.seed}\NormalTok{(param[[}\DecValTok{1}\NormalTok{]]}\OperatorTok{$}\NormalTok{seed)}
-  
-    \CommentTok{# check plpData is coo format:}
-  \ControlFlowTok{if}\NormalTok{(}\OperatorTok{!}\StringTok{'ffdf'}\OperatorTok{%in%}\KeywordTok{class}\NormalTok{(plpData}\OperatorTok{$}\NormalTok{covariates) )}
-    \KeywordTok{stop}\NormalTok{(}\StringTok{'This algorithm requires plpData in coo format'}\NormalTok{)}
-  
-\NormalTok{  metaData <-}\StringTok{ }\KeywordTok{attr}\NormalTok{(population, }\StringTok{'metaData'}\NormalTok{)}
-  \ControlFlowTok{if}\NormalTok{(}\OperatorTok{!}\KeywordTok{is.null}\NormalTok{(population}\OperatorTok{$}\NormalTok{indexes))}
-\NormalTok{    population <-}\StringTok{ }\NormalTok{population[population}\OperatorTok{$}\NormalTok{indexes}\OperatorTok{>}\DecValTok{0}\NormalTok{,]}
-  \KeywordTok{attr}\NormalTok{(population, }\StringTok{'metaData'}\NormalTok{) <-}\StringTok{ }\NormalTok{metaData}
-  
-  \CommentTok{# convert data into sparse R Matrix:}
-\NormalTok{  result <-}\StringTok{ }\KeywordTok{toSparseM}\NormalTok{(plpData,population,}\DataTypeTok{map=}\OtherTok{NULL}\NormalTok{)}
-\NormalTok{  data <-}\StringTok{ }\NormalTok{result}\OperatorTok{$}\NormalTok{data}
-  
-\NormalTok{  data <-}\StringTok{ }\NormalTok{data[population}\OperatorTok{$}\NormalTok{rowId,]}
-  
-  \CommentTok{# set test/train sets (for printing performance as it trains)}
-  \ControlFlowTok{if}\NormalTok{(}\OperatorTok{!}\NormalTok{quiet)}
-    \KeywordTok{writeLines}\NormalTok{(}\KeywordTok{paste0}\NormalTok{(}\StringTok{'Training made up model on train set containing '}\NormalTok{, }\KeywordTok{nrow}\NormalTok{(population), }
-                      \StringTok{' people with '}\NormalTok{,}\KeywordTok{sum}\NormalTok{(population}\OperatorTok{$}\NormalTok{outcomeCount}\OperatorTok{>}\DecValTok{0}\NormalTok{), }\StringTok{' outcomes'}\NormalTok{))}
-\NormalTok{  start <-}\StringTok{ }\KeywordTok{Sys.time}\NormalTok{()}
-  
-  \CommentTok{#============= STEP 1 ======================================}
-  \CommentTok{# pick the best hyper-params and then do final training on all data...}
-  \KeywordTok{writeLines}\NormalTok{(}\StringTok{'train'}\NormalTok{)}
-\NormalTok{  datas <-}\StringTok{ }\KeywordTok{list}\NormalTok{(}\DataTypeTok{population=}\NormalTok{population, }\DataTypeTok{data=}\NormalTok{data)}
-\NormalTok{  param.sel <-}\StringTok{ }\KeywordTok{lapply}\NormalTok{(param, }\ControlFlowTok{function}\NormalTok{(x) }\KeywordTok{do.call}\NormalTok{(made_up_model, }\KeywordTok{c}\NormalTok{(x,datas)  ))}
-\NormalTok{  hyperSummary <-}\StringTok{ }\KeywordTok{do.call}\NormalTok{(rbind, }\KeywordTok{lapply}\NormalTok{(param.sel, }\ControlFlowTok{function}\NormalTok{(x) x}\OperatorTok{$}\NormalTok{hyperSum))}
-\NormalTok{  hyperSummary <-}\StringTok{ }\KeywordTok{as.data.frame}\NormalTok{(hyperSummary)}
-\NormalTok{  hyperSummary}\OperatorTok{$}\NormalTok{auc <-}\StringTok{ }\KeywordTok{unlist}\NormalTok{(}\KeywordTok{lapply}\NormalTok{(param.sel, }\ControlFlowTok{function}\NormalTok{(x) x}\OperatorTok{$}\NormalTok{auc)) }
-\NormalTok{  param.sel <-}\StringTok{ }\KeywordTok{unlist}\NormalTok{(}\KeywordTok{lapply}\NormalTok{(param.sel, }\ControlFlowTok{function}\NormalTok{(x) x}\OperatorTok{$}\NormalTok{auc))}
-\NormalTok{  param <-}\StringTok{ }\NormalTok{param[[}\KeywordTok{which.max}\NormalTok{(param.sel)]]}
-  
-  \CommentTok{# set this so you do a final model train }
-\NormalTok{  param}\OperatorTok{$}\NormalTok{final=T}
-  
-  \KeywordTok{writeLines}\NormalTok{(}\StringTok{'final train'}\NormalTok{)}
-\NormalTok{  trainedModel <-}\StringTok{ }\KeywordTok{do.call}\NormalTok{(made_up_model, }\KeywordTok{c}\NormalTok{(param,datas)  )}\OperatorTok{$}\NormalTok{model}
-  
-\NormalTok{  comp <-}\StringTok{ }\KeywordTok{Sys.time}\NormalTok{() }\OperatorTok{-}\StringTok{ }\NormalTok{start}
-  \ControlFlowTok{if}\NormalTok{(}\OperatorTok{!}\NormalTok{quiet)}
-    \KeywordTok{writeLines}\NormalTok{(}\KeywordTok{paste0}\NormalTok{(}\StringTok{'Model Made Up trained - took:'}\NormalTok{,  }\KeywordTok{format}\NormalTok{(comp, }\DataTypeTok{digits=}\DecValTok{3}\NormalTok{)))}
-  
-  \CommentTok{# construct the standard output for a model:}
-\NormalTok{  result <-}\StringTok{ }\KeywordTok{list}\NormalTok{(}\DataTypeTok{model =}\NormalTok{ trainedModel,}
-                 \DataTypeTok{modelSettings =} \KeywordTok{list}\NormalTok{(}\DataTypeTok{model=}\StringTok{'made_up'}\NormalTok{, }\DataTypeTok{modelParameters=}\NormalTok{param),}
-                 \DataTypeTok{trainCVAuc =} \OtherTok{NULL}\NormalTok{,}
-                 \DataTypeTok{hyperParamSearch =}\NormalTok{ hyperSummary,}
-                 \DataTypeTok{metaData =}\NormalTok{ plpData}\OperatorTok{$}\NormalTok{metaData,}
-                 \DataTypeTok{populationSettings =} \KeywordTok{attr}\NormalTok{(population, }\StringTok{'metaData'}\NormalTok{),}
-                 \DataTypeTok{outcomeId=}\NormalTok{outcomeId,}\CommentTok{# can use populationSettings$outcomeId?}
-                 \DataTypeTok{cohortId=}\NormalTok{cohortId,}
-                 \DataTypeTok{varImp =} \OtherTok{NULL}\NormalTok{,}
-                 \DataTypeTok{trainingTime=}\NormalTok{comp,}
-                 \DataTypeTok{covariateMap=}\NormalTok{result}\OperatorTok{$}\NormalTok{map}
-\NormalTok{  )}
-  \KeywordTok{class}\NormalTok{(result) <-}\StringTok{ 'plpModel'}
-  \KeywordTok{attr}\NormalTok{(result, }\StringTok{'type'}\NormalTok{) <-}\StringTok{ 'madeup'}
-  \KeywordTok{attr}\NormalTok{(result, }\StringTok{'predictionType'}\NormalTok{) <-}\StringTok{ 'binary'}
-  \KeywordTok{return}\NormalTok{(result)}
-    
-\NormalTok{\}}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{helpers}{%
-\subsection{Helpers}\label{helpers}}
-
-In the fit model a helper function \texttt{made\_up\_model} is called,
-this is the function that trains a model given the data and population
-(where the popualtion contains a column outcomeCount corresponding to
-the outcome). Both the data and population are ordered the same way:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{made_up_model <-}\StringTok{ }\ControlFlowTok{function}\NormalTok{(data, population,}
-                       \DataTypeTok{a=}\DecValTok{1}\NormalTok{,}\DataTypeTok{b=}\DecValTok{1}\NormalTok{, }\DataTypeTok{final=}\NormalTok{F, ...)\{}
-  
-  \KeywordTok{writeLines}\NormalTok{(}\KeywordTok{paste}\NormalTok{(}\StringTok{'Training Made Up model with '}\NormalTok{,}\KeywordTok{length}\NormalTok{(}\KeywordTok{unique}\NormalTok{(population}\OperatorTok{$}\NormalTok{indexes)),}
-                   \StringTok{' fold CV'}\NormalTok{))}
-  \ControlFlowTok{if}\NormalTok{(}\OperatorTok{!}\KeywordTok{is.null}\NormalTok{(population}\OperatorTok{$}\NormalTok{indexes) }\OperatorTok{&&}\StringTok{ }\NormalTok{final}\OperatorTok{==}\NormalTok{F)\{}
-\NormalTok{    index_vect <-}\StringTok{ }\KeywordTok{unique}\NormalTok{(population}\OperatorTok{$}\NormalTok{indexes)}
-\NormalTok{    perform <-}\StringTok{ }\KeywordTok{c}\NormalTok{()}
-    
-    \CommentTok{# create prediction matrix to store all predictions}
-\NormalTok{    predictionMat <-}\StringTok{ }\NormalTok{population}
-\NormalTok{    predictionMat}\OperatorTok{$}\NormalTok{value <-}\StringTok{ }\DecValTok{0}
-    \KeywordTok{attr}\NormalTok{(predictionMat, }\StringTok{"metaData"}\NormalTok{) <-}\StringTok{ }\KeywordTok{list}\NormalTok{(}\DataTypeTok{predictionType =} \StringTok{"binary"}\NormalTok{)}
-    
-    \ControlFlowTok{for}\NormalTok{(index }\ControlFlowTok{in} \DecValTok{1}\OperatorTok{:}\KeywordTok{length}\NormalTok{(index_vect ))\{}
-      \KeywordTok{writeLines}\NormalTok{(}\KeywordTok{paste}\NormalTok{(}\StringTok{'Fold '}\NormalTok{,index, }\StringTok{' -- with '}\NormalTok{, }\KeywordTok{sum}\NormalTok{(population}\OperatorTok{$}\NormalTok{indexes}\OperatorTok{!=}\NormalTok{index),}
-                       \StringTok{'train rows'}\NormalTok{))}
-\NormalTok{      model <-}\StringTok{ }\NormalTok{madeup}\OperatorTok{::}\KeywordTok{model}\NormalTok{(}\DataTypeTok{x =}\NormalTok{ data[population}\OperatorTok{$}\NormalTok{indexes}\OperatorTok{!=}\NormalTok{index,],}
-                             \DataTypeTok{y=}\NormalTok{ population}\OperatorTok{$}\NormalTok{outcomeCount[population}\OperatorTok{$}\NormalTok{indexes}\OperatorTok{!=}\NormalTok{index],}
-                                  \DataTypeTok{a=}\NormalTok{a, }\DataTypeTok{b=}\NormalTok{b)}
-      
-\NormalTok{      pred <-}\StringTok{ }\NormalTok{stats}\OperatorTok{::}\KeywordTok{predict}\NormalTok{(model, data[population}\OperatorTok{$}\NormalTok{indexes}\OperatorTok{==}\NormalTok{index,])}
-\NormalTok{      prediction <-}\StringTok{ }\NormalTok{population[population}\OperatorTok{$}\NormalTok{indexes}\OperatorTok{==}\NormalTok{index,]}
-\NormalTok{      prediction}\OperatorTok{$}\NormalTok{value <-}\StringTok{ }\NormalTok{pred}
-      \KeywordTok{attr}\NormalTok{(prediction, }\StringTok{"metaData"}\NormalTok{) <-}\StringTok{ }\KeywordTok{list}\NormalTok{(}\DataTypeTok{predictionType =} \StringTok{"binary"}\NormalTok{)}
-\NormalTok{      aucVal <-}\StringTok{ }\KeywordTok{computeAuc}\NormalTok{(prediction)}
-\NormalTok{      perform <-}\StringTok{ }\KeywordTok{c}\NormalTok{(perform,aucVal)}
-      
-      \CommentTok{# add the fold predictions and compute AUC after loop}
-\NormalTok{      predictionMat}\OperatorTok{$}\NormalTok{value[population}\OperatorTok{$}\NormalTok{indexes}\OperatorTok{==}\NormalTok{index] <-}\StringTok{ }\NormalTok{pred}
-      
-\NormalTok{     \}}
-    \CommentTok{##auc <- mean(perform) # want overal rather than mean}
-\NormalTok{    auc <-}\StringTok{ }\KeywordTok{computeAuc}\NormalTok{(predictionMat)}
-    
-\NormalTok{    foldPerm <-}\StringTok{ }\NormalTok{perform}
-\NormalTok{  \} }\ControlFlowTok{else}\NormalTok{ \{}
-\NormalTok{    model <-}\StringTok{ }\NormalTok{madeup}\OperatorTok{::}\KeywordTok{model}\NormalTok{(}\DataTypeTok{x=}\NormalTok{ data, }
-                                \DataTypeTok{y=}\NormalTok{ population}\OperatorTok{$}\NormalTok{outcomeCount,}
-                                \DataTypeTok{a=}\NormalTok{a,}\DataTypeTok{b=}\NormalTok{b)}
-    
-\NormalTok{    pred <-}\StringTok{ }\NormalTok{stats}\OperatorTok{::}\KeywordTok{predict}\NormalTok{(model, data)}
-\NormalTok{    prediction <-}\StringTok{ }\NormalTok{population}
-\NormalTok{    prediction}\OperatorTok{$}\NormalTok{value <-}\StringTok{ }\NormalTok{pred}
-    \KeywordTok{attr}\NormalTok{(prediction, }\StringTok{"metaData"}\NormalTok{) <-}\StringTok{ }\KeywordTok{list}\NormalTok{(}\DataTypeTok{predictionType =} \StringTok{"binary"}\NormalTok{) }
-\NormalTok{    auc <-}\StringTok{ }\KeywordTok{computeAuc}\NormalTok{(prediction)}
-\NormalTok{    foldPerm <-}\StringTok{ }\NormalTok{auc}
-\NormalTok{  \}}
-  
-\NormalTok{  result <-}\StringTok{ }\KeywordTok{list}\NormalTok{(}\DataTypeTok{model=}\NormalTok{model,}
-                 \DataTypeTok{auc=}\NormalTok{auc,}
-                 \DataTypeTok{hyperSum =} \KeywordTok{unlist}\NormalTok{(}\KeywordTok{list}\NormalTok{(}\DataTypeTok{a =}\NormalTok{ a, }\DataTypeTok{b =}\NormalTok{ b, }\DataTypeTok{fold_auc=}\NormalTok{foldPerm))}
-\NormalTok{  )}
-  \KeywordTok{return}\NormalTok{(result)}
-\NormalTok{\}}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{predict-1}{%
-\subsection{Predict}\label{predict-1}}
-
-The final step is to create a predict function for the model. This gets
-added to the predict.R file. In the example above the type
-\texttt{attr(result,\ \textquotesingle{}type\textquotesingle{})\ \textless{}-\ \textquotesingle{}madeup\textquotesingle{}}
-was madeup, so a \texttt{predict.madeup} function is required to be
-added into the predict.R. The predict function needs to take as input
-the plpModel returned by the fit function, the population to apply the
-model on and the plpData specifying the covariates of the population.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{predict.madeup <-}\StringTok{ }\ControlFlowTok{function}\NormalTok{(plpModel,population, plpData, ...)\{ }
-\NormalTok{  result <-}\StringTok{ }\KeywordTok{toSparseM}\NormalTok{(plpData, population, }\DataTypeTok{map=}\NormalTok{plpModel}\OperatorTok{$}\NormalTok{covariateMap)}
-\NormalTok{  data <-}\StringTok{ }\NormalTok{result}\OperatorTok{$}\NormalTok{data[population}\OperatorTok{$}\NormalTok{rowId,]}
-\NormalTok{  prediction <-}\StringTok{ }\KeywordTok{data.frame}\NormalTok{(}\DataTypeTok{rowId=}\NormalTok{population}\OperatorTok{$}\NormalTok{rowId, }
-                           \DataTypeTok{value=}\NormalTok{stats}\OperatorTok{::}\KeywordTok{predict}\NormalTok{(plpModel}\OperatorTok{$}\NormalTok{model, data)}
-\NormalTok{                           )}
-  
-\NormalTok{  prediction <-}\StringTok{ }\KeywordTok{merge}\NormalTok{(population, prediction, }\DataTypeTok{by=}\StringTok{'rowId'}\NormalTok{)}
-\NormalTok{  prediction <-}\StringTok{ }\NormalTok{prediction[,}\KeywordTok{colnames}\NormalTok{(prediction)}\OperatorTok{%in%}
-\StringTok{                           }\KeywordTok{c}\NormalTok{(}\StringTok{'rowId'}\NormalTok{,}\StringTok{'outcomeCount'}\NormalTok{,}\StringTok{'indexes'}\NormalTok{, }\StringTok{'value'}\NormalTok{)] }\CommentTok{# need to fix no index issue}
-  \KeywordTok{attr}\NormalTok{(prediction, }\StringTok{"metaData"}\NormalTok{) <-}\StringTok{ }\KeywordTok{list}\NormalTok{(}\DataTypeTok{predictionType =} \StringTok{"binary"}\NormalTok{) }
-  \KeywordTok{return}\NormalTok{(prediction)}
-  
-\NormalTok{\}}
-\end{Highlighting}
-\end{Shaded}
-
-As the madeup model uses the standard R prediction, it has the same
-prediction function as xgboost, so we could have not added a new
-prediction function and instead made the type of the result returned by
-fitMadeUpModel to
-\texttt{attr(result,\ \textquotesingle{}type\textquotesingle{})\ \textless{}-\ \textquotesingle{}xgboost\textquotesingle{}}.
-
-\hypertarget{acknowledgments}{%
-\section{Acknowledgments}\label{acknowledgments}}
-
-Considerable work has been dedicated to provide the
-\texttt{PatientLevelPrediction} package.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{citation}\NormalTok{(}\StringTok{"PatientLevelPrediction"}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-\begin{verbatim}
-## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design and
-## implementation of a standardized framework to generate and evaluate patient-level
-## prediction models using observational healthcare data." _Journal of the American
-## Medical Informatics Association_, *25*(8), 969-975. <URL:
-## https://doi.org/10.1093/jamia/ocy032>.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }
-\end{verbatim}
-
-\textbf{Please reference this paper if you use the PLP Package in your
-work:}
-
-\href{http://dx.doi.org/10.1093/jamia/ocy032}{Reps JM, Schuemie MJ,
-Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a
-standardized framework to generate and evaluate patient-level prediction
-models using observational healthcare data. J Am Med Inform Assoc.
-2018;25(8):969-975.}
-
-This work is supported in part through the National Science Foundation
-grant IIS 1251151.
-
-\end{document}
diff --git a/inst/doc/AddingCustomFeatureEngineering.pdf b/inst/doc/AddingCustomFeatureEngineering.pdf
index 4d97db3c5..e4e8220ce 100644
Binary files a/inst/doc/AddingCustomFeatureEngineering.pdf and b/inst/doc/AddingCustomFeatureEngineering.pdf differ
diff --git a/inst/doc/BuildingDeepLearningModels.pdf b/inst/doc/BuildingDeepLearningModels.pdf
deleted file mode 100644
index a496daccb..000000000
Binary files a/inst/doc/BuildingDeepLearningModels.pdf and /dev/null differ
diff --git a/inst/doc/BuildingDeepLearningModels.tex b/inst/doc/BuildingDeepLearningModels.tex
deleted file mode 100644
index 8917b0849..000000000
--- a/inst/doc/BuildingDeepLearningModels.tex
+++ /dev/null
@@ -1,741 +0,0 @@
-% Options for packages loaded elsewhere
-\PassOptionsToPackage{unicode}{hyperref}
-\PassOptionsToPackage{hyphens}{url}
-%
-\documentclass[
-]{article}
-\usepackage{lmodern}
-\usepackage{amssymb,amsmath}
-\usepackage{ifxetex,ifluatex}
-\ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex
-  \usepackage[T1]{fontenc}
-  \usepackage[utf8]{inputenc}
-  \usepackage{textcomp} % provide euro and other symbols
-\else % if luatex or xetex
-  \usepackage{unicode-math}
-  \defaultfontfeatures{Scale=MatchLowercase}
-  \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
-\fi
-% Use upquote if available, for straight quotes in verbatim environments
-\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
-\IfFileExists{microtype.sty}{% use microtype if available
-  \usepackage[]{microtype}
-  \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
-}{}
-\makeatletter
-\@ifundefined{KOMAClassName}{% if non-KOMA class
-  \IfFileExists{parskip.sty}{%
-    \usepackage{parskip}
-  }{% else
-    \setlength{\parindent}{0pt}
-    \setlength{\parskip}{6pt plus 2pt minus 1pt}}
-}{% if KOMA class
-  \KOMAoptions{parskip=half}}
-\makeatother
-\usepackage{xcolor}
-\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
-\IfFileExists{bookmark.sty}{\usepackage{bookmark}}{\usepackage{hyperref}}
-\hypersetup{
-  pdftitle={Building Deep Learning Models},
-  pdfauthor={Peter R. Rijnbeek, Seng Chan You, Xiaoyong Pan, Jenna Reps},
-  hidelinks,
-  pdfcreator={LaTeX via pandoc}}
-\urlstyle{same} % disable monospaced font for URLs
-\usepackage[margin=1in]{geometry}
-\usepackage{color}
-\usepackage{fancyvrb}
-\newcommand{\VerbBar}{|}
-\newcommand{\VERB}{\Verb[commandchars=\\\{\}]}
-\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
-% Add ',fontsize=\small' for more characters per line
-\usepackage{framed}
-\definecolor{shadecolor}{RGB}{248,248,248}
-\newenvironment{Shaded}{\begin{snugshade}}{\end{snugshade}}
-\newcommand{\AlertTok}[1]{\textcolor[rgb]{0.94,0.16,0.16}{#1}}
-\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.77,0.63,0.00}{#1}}
-\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\BuiltInTok}[1]{#1}
-\newcommand{\CharTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
-\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
-\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{#1}}
-\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\ErrorTok}[1]{\textcolor[rgb]{0.64,0.00,0.00}{\textbf{#1}}}
-\newcommand{\ExtensionTok}[1]{#1}
-\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\ImportTok}[1]{#1}
-\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
-\newcommand{\NormalTok}[1]{#1}
-\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.81,0.36,0.00}{\textbf{#1}}}
-\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{#1}}
-\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
-\newcommand{\RegionMarkerTok}[1]{#1}
-\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\StringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\usepackage{longtable,booktabs}
-% Correct order of tables after \paragraph or \subparagraph
-\usepackage{etoolbox}
-\makeatletter
-\patchcmd\longtable{\par}{\if@noskipsec\mbox{}\fi\par}{}{}
-\makeatother
-% Allow footnotes in longtable head/foot
-\IfFileExists{footnotehyper.sty}{\usepackage{footnotehyper}}{\usepackage{footnote}}
-\makesavenoteenv{longtable}
-\usepackage{graphicx,grffile}
-\makeatletter
-\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
-\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
-\makeatother
-% Scale images if necessary, so that they will not overflow the page
-% margins by default, and it is still possible to overwrite the defaults
-% using explicit options in \includegraphics[width, height, ...]{}
-\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
-% Set default figure placement to htbp
-\makeatletter
-\def\fps@figure{htbp}
-\makeatother
-\setlength{\emergencystretch}{3em} % prevent overfull lines
-\providecommand{\tightlist}{%
-  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
-\setcounter{secnumdepth}{5}
-\usepackage{fancyhdr}
-\pagestyle{fancy}
-\fancyhead{}
-\fancyhead[CO,CE]{Building Deep Learning Models}
-\fancyfoot[CO,CE]{PatientLevelPrediction Package Version 3.1.0}
-\fancyfoot[LE,RO]{\thepage}
-\renewcommand{\headrulewidth}{0.4pt}
-\renewcommand{\footrulewidth}{0.4pt}
-
-\title{Building Deep Learning Models}
-\author{Peter R. Rijnbeek, Seng Chan You, Xiaoyong Pan, Jenna Reps}
-\date{2020-06-03}
-
-\begin{document}
-\maketitle
-
-{
-\setcounter{tocdepth}{2}
-\tableofcontents
-}
-\hypertarget{introduction}{%
-\section{Introduction}\label{introduction}}
-
-Electronic Health Records (EHR) data is high dimensional, heterogeneous,
-and sparse, which makes predictive modelling a challenge. In the early
-days, the machine learning community mainly focused on algorithm
-development, currently there is a shift to more powerful feature
-engineering. Deep Learning models are widely used to automatically learn
-high-level feature representations from the data, and have achieved
-remarkable results in image processing, speech recognition and
-computational biology. Recently, interesting results have been shown
-using EHRs, but more extensive research is needed to assess the power of
-Deep Learning in this domain.
-
-This vignette describes how you can use the Observational Health Data
-Sciences and Informatics (OHDSI)
-\href{http://github.com/OHDSI/PatientLevelPrediction}{\texttt{PatientLevelPrediction}}
-package to build Deep Learning models. This vignette assumes you have
-read and are comfortable with building patient level prediction models
-as described in the
-\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf}{\texttt{BuildingPredictiveModels}
-vignette}. Furthermore, this vignette assumes you are familiar with Deep
-Learning methods.
-
-\hypertarget{background}{%
-\section{Background}\label{background}}
-
-Deep Learning models are build by stacking an often large number of
-neural network layers that perform feature engineering steps, e.g
-embedding, and are collapsed in a final softmax layer (basically a
-logistic regression layer). These algorithms need a lot of data to
-converge to a good representation, but currently the sizes of the EHR
-databases are growing fast which would make Deep Learning an interesting
-approach to test within OHDSI's
-\href{https://academic.oup.com/jamia/article/25/8/969/4989437}{Patient-Level
-Prediction Framework}. The current implementation allows us to perform
-research at scale on the value and limitations of Deep Learning using
-EHR data. For relatively small Target and Outcome cohorts, Deep Learning
-is most probably not the best choice.
-
-Most current Deep Learning research is performed in python and we have
-developed a pipeline to interact with python. Multiple Deep Learning
-backends have been developed, e.g.~Tensorflow, PyTorch, Keras (recently
-also available in R) etc. In the package we have implemented interaction
-with Keras in R and PyTorch in Python but we invite the community to add
-other backends.
-
-Many network architectures have recently been proposed and we have
-implemented a number of them, however, this list will grow in the near
-future. It is important to understand that some of these architectures
-require a 2D data matrix,
-i.e.~\textbar patient\textbar x\textbar feature\textbar, and others use
-a 3D data matrix
-\textbar patient\textbar x\textbar feature\textbar x\textbar time\textbar.
-The \href{www.github.com/ohdsi/FeatureExtraction}{FeatureExtraction
-Package} has been extended to enable the extraction of both data formats
-as will be described with examples below.
-
-Note that training Deep Learning models is computationally intensive,
-our implementation therefore supports both GPU and CPU. It will
-automatically check whether there is GPU or not in your computer. A GPU
-is highly recommended for Deep Learning!
-
-\hypertarget{non-temporal-architectures}{%
-\section{Non-Temporal Architectures}\label{non-temporal-architectures}}
-
-We implemented the following non-temporal (2D data matrix) architectures
-using PyTorch:
-
-\begin{verbatim}
-1) Logistics regression (LRTorch)
-   A simple softmax layer with l2 regularization
-
-2) Feed forward network (MLPTorch) 
-   Supports multilayer perceptron (mlp_type = MLP) and 
-   Self-Normalizing Neural Networks (mlp_type = SNN)
-   Reference: https://arxiv.org/abs/1706.02515
-\end{verbatim}
-
-For the above two methods, we implemented support for a stacked
-autoencoder and a variational autoencoder to reduce the feature
-dimension as a first step. These autoencoders learn efficient data
-encodings in an unsupervised manner by stacking multiple layers in a
-neural network. Compared to the standard implementations of LR and MLP
-these implementations can use the GPU power to speed up the gradient
-descent approach in the back propagation to optimize the weights of the
-classifier.
-
-Table 1: Non-Temporal Deep Learning Models Hyper-Parameters
-
-\begin{longtable}[]{@{}lll@{}}
-\toprule
-\begin{minipage}[b]{0.10\columnwidth}\raggedright
-Name\strut
-\end{minipage} & \begin{minipage}[b]{0.34\columnwidth}\raggedright
-Description\strut
-\end{minipage} & \begin{minipage}[b]{0.47\columnwidth}\raggedright
-Hyper-parameters\strut
-\end{minipage}\tabularnewline
-\midrule
-\endhead
-\begin{minipage}[t]{0.10\columnwidth}\raggedright
-LRTorch\strut
-\end{minipage} & \begin{minipage}[t]{0.34\columnwidth}\raggedright
-Logistic Regression Model\strut
-\end{minipage} & \begin{minipage}[t]{0.47\columnwidth}\raggedright
-w\_decay (l2 regularization), epochs (number of epochs), class\_weight
-(0 = inverse ratio between number of positive and negative examples, -1
-= focal loss (\url{https://arxiv.org/abs/1708.02002}), or other),
-autoencoder (apply stacked autoencoder?, vae (apply variational
-autoencoder)\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.10\columnwidth}\raggedright
-MLPTorch\strut
-\end{minipage} & \begin{minipage}[t]{0.34\columnwidth}\raggedright
-Multi-Layer Perceptron Model\strut
-\end{minipage} & \begin{minipage}[t]{0.47\columnwidth}\raggedright
-mlp\_type (MLP = default, SNN = self-normalizing neural network), size
-(number of hidden nodes), w\_decay (l2 regularization), epochs (number
-of epochs), class\_weight(0 = inverse ratio between number of positive
-and negative examples, -1 = focal loss, or other), autoencoder (apply
-stacked autoencoder), vae (apply variational autoencoder?)\strut
-\end{minipage}\tabularnewline
-\bottomrule
-\end{longtable}
-
-\#\#Example The approach for logistic regression (LRTorch) and the
-Multi-Layer Perceptron (MLPTorch) is identical. Here we will take
-LRTorch as an example.
-
-You need to generate a \texttt{population} and \texttt{plpData} object
-as described in more detail in
-\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf}{\texttt{BuildingPredictiveModels}
-vignette}.
-
-Alternatively, you can make use of the data simulator. The following
-code snippet creates a population of 12000 patients.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{set.seed}\NormalTok{(}\DecValTok{1234}\NormalTok{)}
-\KeywordTok{data}\NormalTok{(plpDataSimulationProfile)}
-\NormalTok{sampleSize <-}\StringTok{ }\DecValTok{12000}
-\NormalTok{plpData <-}\StringTok{ }\KeywordTok{simulatePlpData}\NormalTok{(}
-\NormalTok{  plpDataSimulationProfile,}
-  \DataTypeTok{n =}\NormalTok{ sampleSize}
-\NormalTok{)}
-
-\NormalTok{population <-}\StringTok{ }\KeywordTok{createStudyPopulation}\NormalTok{(}
-\NormalTok{  plpData,}
-  \DataTypeTok{outcomeId =} \DecValTok{2}\NormalTok{,}
-  \DataTypeTok{binary =} \OtherTok{TRUE}\NormalTok{,}
-  \DataTypeTok{firstExposureOnly =} \OtherTok{FALSE}\NormalTok{,}
-  \DataTypeTok{washoutPeriod =} \DecValTok{0}\NormalTok{,}
-  \DataTypeTok{removeSubjectsWithPriorOutcome =} \OtherTok{FALSE}\NormalTok{,}
-  \DataTypeTok{priorOutcomeLookback =} \DecValTok{99999}\NormalTok{,}
-  \DataTypeTok{requireTimeAtRisk =} \OtherTok{FALSE}\NormalTok{,}
-  \DataTypeTok{minTimeAtRisk =} \DecValTok{0}\NormalTok{,}
-  \DataTypeTok{riskWindowStart =} \DecValTok{0}\NormalTok{,}
-  \DataTypeTok{addExposureDaysToStart =} \OtherTok{FALSE}\NormalTok{,}
-  \DataTypeTok{riskWindowEnd =} \DecValTok{365}\NormalTok{,}
-  \DataTypeTok{addExposureDaysToEnd =} \OtherTok{FALSE}\NormalTok{,}
-  \DataTypeTok{verbosity =} \StringTok{"INFO"}
-\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-As an example we will build a LRTorch model. We could specify the
-stacked autoencoder or the variational autoencoder to be used for
-reducing the feature dimension as an initial layer, but for this example
-we do not.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{autoencoder <-}\StringTok{ }\OtherTok{FALSE}
-\NormalTok{vae <-}\StringTok{ }\OtherTok{FALSE}
-\end{Highlighting}
-\end{Shaded}
-
-We added a class\_weight for imbalanced data, the default value 0 is the
-inverse ratio between negatives and positives,-1 applies focal loss.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{class_weight <-}\StringTok{ }\DecValTok{0}
-\end{Highlighting}
-\end{Shaded}
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\CommentTok{# Specify the settings for Logistics regression model using Torch in Python}
-\NormalTok{model <-}\StringTok{ }\KeywordTok{setLRTorch}\NormalTok{(}\DataTypeTok{autoencoder=}\NormalTok{autoencoder, }\DataTypeTok{vae=}\NormalTok{vae,  }\DataTypeTok{class_weight=}\NormalTok{class_weight)}
-\end{Highlighting}
-\end{Shaded}
-
-No we define our modelling parameters.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{testFraction <-}\StringTok{ }\FloatTok{0.2}
-\NormalTok{testSplit <-}\StringTok{ 'person'}
-\NormalTok{nfold <-}\StringTok{ }\DecValTok{3}
-\NormalTok{splitSeed <-}\StringTok{ }\DecValTok{1000}
-\end{Highlighting}
-\end{Shaded}
-
-And we train and internally validate the model.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{results <-}\StringTok{ }\NormalTok{PatientLevelPrediction}\OperatorTok{::}\KeywordTok{runPlp}\NormalTok{(}\DataTypeTok{population =}\NormalTok{ population, }
-                                          \DataTypeTok{plpData =}\NormalTok{ plpData, }
-                                          \DataTypeTok{modelSettings =}\NormalTok{ model,}
-                                          \DataTypeTok{testSplit=}\NormalTok{testSplit,}
-                                          \DataTypeTok{testFraction=}\NormalTok{testFraction,}
-                                          \DataTypeTok{nfold=}\NormalTok{nfold, }
-                                          \DataTypeTok{splitSeed=}\NormalTok{splitSeed) }
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{temporal-architectures}{%
-\section{Temporal Architectures}\label{temporal-architectures}}
-
-Several architectures are implemented that can handle temporal data in
-PyTorch and R Keras.
-
-\hypertarget{pytorch-cnn}{%
-\subsection{PyTorch CNN}\label{pytorch-cnn}}
-
-We implemented the following \textbf{convolutional} models described in
-\url{https://github.com/clinicalml/deepDiagnosis} in CNNTorch:
-
-\begin{enumerate}
-\def\labelenumi{\arabic{enumi})}
-\item
-  Temporal Convolutional neural network over a backward window (type =
-  cnn)
-
-  \includegraphics{arch1.png}
-\item
-  Convolutional neural network over input and time dimension (type =
-  mix)
-
-  \includegraphics{conv_arch2.png}
-\item
-  Multi-resolution temporal convolutional neural network (type = multi)
-
-  \includegraphics{conv_arch1.png}
-\end{enumerate}
-
-Furthermore, we added the following achitectures:
-
-\begin{enumerate}
-\def\labelenumi{\arabic{enumi})}
-\setcounter{enumi}{3}
-\item
-  CNN with filters with three different parallel kernel sizes (3,4,5)
-  and a fully connected layers (type = mlf)
-
-  \includegraphics{cnn_mlf2.png}
-\item
-  LSTM network over the backward window (type = lstm)
-
-  \includegraphics{cnn_lstm.png}
-\item
-  Residual Learning Network as described in:
-  \url{https://arxiv.org/abs/1512.03385} (type = resnet)
-
-  This a very big network, see the paper for the topology.
-\end{enumerate}
-
-\begin{longtable}[]{@{}ll@{}}
-\toprule
-\begin{minipage}[b]{0.26\columnwidth}\raggedright
-parameter\strut
-\end{minipage} & \begin{minipage}[b]{0.68\columnwidth}\raggedright
-description\strut
-\end{minipage}\tabularnewline
-\midrule
-\endhead
-\begin{minipage}[t]{0.26\columnwidth}\raggedright
-nbfilters\strut
-\end{minipage} & \begin{minipage}[t]{0.68\columnwidth}\raggedright
-The number of convolution filters\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.26\columnwidth}\raggedright
-epochs\strut
-\end{minipage} & \begin{minipage}[t]{0.68\columnwidth}\raggedright
-The number of epochs\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.26\columnwidth}\raggedright
-seed\strut
-\end{minipage} & \begin{minipage}[t]{0.68\columnwidth}\raggedright
-Random seed\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.26\columnwidth}\raggedright
-class\_weight\strut
-\end{minipage} & \begin{minipage}[t]{0.68\columnwidth}\raggedright
-The class weight used for imbalanced data (0: Inverse ratio between
-positives and negatives, -1: Focal loss, or number)\strut
-\end{minipage}\tabularnewline
-\bottomrule
-\end{longtable}
-
-\hypertarget{pytorch-rnn}{%
-\subsection{PyTorch RNN}\label{pytorch-rnn}}
-
-The following \textbf{recurrent neural network} models are implemented
-in RNNTorch:
-
-\begin{enumerate}
-\def\labelenumi{\arabic{enumi})}
-\item
-  RNN with one LSTM layer fed into one fully connected layer (type =
-  RNN)
-
-  \includegraphics{lstm_last.png}
-\item
-  RNN with one bidirectional LSTM layer fed into one fully connected
-  layer (type = BiRNN)
-
-  This network looks the same as above but then as a bi-directional
-  version
-\item
-  One Gated Recurrent Unit layer fed into one fully connected layers
-  (type = GRU)
-
-  This network looks the same as above but then implemented as GRU
-\end{enumerate}
-
-The following hyper-parameters can be set for these PyTorch models:
-
-\begin{longtable}[]{@{}ll@{}}
-\toprule
-\begin{minipage}[b]{0.26\columnwidth}\raggedright
-parameter\strut
-\end{minipage} & \begin{minipage}[b]{0.68\columnwidth}\raggedright
-description\strut
-\end{minipage}\tabularnewline
-\midrule
-\endhead
-\begin{minipage}[t]{0.26\columnwidth}\raggedright
-hidden\_size\strut
-\end{minipage} & \begin{minipage}[t]{0.68\columnwidth}\raggedright
-The number of features in hidden state\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.26\columnwidth}\raggedright
-epochs\strut
-\end{minipage} & \begin{minipage}[t]{0.68\columnwidth}\raggedright
-The number of epochs\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.26\columnwidth}\raggedright
-seed\strut
-\end{minipage} & \begin{minipage}[t]{0.68\columnwidth}\raggedright
-Random seed\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.26\columnwidth}\raggedright
-class\_weight\strut
-\end{minipage} & \begin{minipage}[t]{0.68\columnwidth}\raggedright
-The class weight used for imbalanced data (0: Inverse ratio between
-positives and negatives, -1: Focal loss, or number)\strut
-\end{minipage}\tabularnewline
-\bottomrule
-\end{longtable}
-
-\newpage
-
-\hypertarget{r-keras-cnn}{%
-\subsection{R Keras CNN}\label{r-keras-cnn}}
-
-The following temporal architectures as described in
-\url{https://arxiv.org/pdf/1608.00647.pdf} were implemented using R
-Keras:
-
-\begin{enumerate}
-\def\labelenumi{\arabic{enumi}.}
-\item
-  Multi-resolution CovNN model (CovNN.R)
-
-  \includegraphics{covcnn.png}
-\item
-  Convolution across data and time according(CovNN2.R)
-
-  \includegraphics{covcnn2.png}
-
-  \newpage
-\end{enumerate}
-
-Furthermore, a custom build RNN is added that uses a variational
-autoencoder.
-
-\begin{enumerate}
-\def\labelenumi{\arabic{enumi}.}
-\setcounter{enumi}{2}
-\item
-  Clinically Informing application based on Recurrent Neural Network
-  (CIReNN.R)
-
-  \includegraphics{cirenn.png}
-\end{enumerate}
-
-Table 2: Temporal Deep Learning Models
-
-\begin{longtable}[]{@{}ll@{}}
-\toprule
-\begin{minipage}[b]{0.11\columnwidth}\raggedright
-Model\strut
-\end{minipage} & \begin{minipage}[b]{0.83\columnwidth}\raggedright
-Hyper-parameters\strut
-\end{minipage}\tabularnewline
-\midrule
-\endhead
-\begin{minipage}[t]{0.11\columnwidth}\raggedright
-CovNN\strut
-\end{minipage} & \begin{minipage}[t]{0.83\columnwidth}\raggedright
-batchSize (The number of samples to used in each batch during model
-training), outcomeWeight (The weight assigned to the outcome), lr (The
-learning rate), decay (The decay of the learning rate), dropout
-({[}currently not used{]} the dropout rate for regularization), epochs
-(The number of times data is used to train the model, e.g., epoches=1
-means data only used once to train), filters (The number of columns
-output by each convolution), kernelSize (The number of time dimensions
-used for each convolution), loss (The loss function implemented), seed
-(The random seed)\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.11\columnwidth}\raggedright
-CovNN2\strut
-\end{minipage} & \begin{minipage}[t]{0.83\columnwidth}\raggedright
-batchSize (The number of samples to used in each batch during model
-training), outcomeWeight (The weight assigned to the outcome), lr (The
-learning rate), decay (The decay of the learning rate), dropout
-({[}currently not used{]} the dropout rate for regularization), epochs
-(The number of times data is used to train the model, e.g., epoches=1
-means data only used once to train), filters (The number of columns
-output by each convolution), kernelSize (The number of time dimensions
-used for each convolution), loss (The loss function implemented), seed
-(The random seed)\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.11\columnwidth}\raggedright
-CIReNN\strut
-\end{minipage} & \begin{minipage}[t]{0.83\columnwidth}\raggedright
-units (The number of units of RNN layer - as a list of vectors),
-recurrentDropout (The reccurrent dropout rate), layerDropout (The layer
-dropout rate), lr (Learning rate), decay (Learning rate decay over each
-update), outcomeWeight (The weight of the outcome class in the loss
-function), batchSize (The number of data points to use per training
-batch), epochs (Number of times to iterate over data set),
-earlyStoppingMinDelta (Minimum change in the monitored quantity to
-qualify as an improvement for early stopping, i.e.~an absolute change of
-less than min\_delta in loss of validation data, will count as no
-improvement), earlyStoppingPatience (Number of epochs with no
-improvement after which training will be stopped), seed (Random seed
-used by Deep Learning model)\strut
-\end{minipage}\tabularnewline
-\bottomrule
-\end{longtable}
-
-\hypertarget{example}{%
-\subsection{Example}\label{example}}
-
-We will now show how to use the temporal models by using CNNTorch as an
-example.
-
-You need to generate a \texttt{population} and \texttt{plpData} object
-as described in more detail in
-\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf}{\texttt{BuildingPredictiveModels}
-vignette}.
-
-Note that for these algorithms you need to extracted temporal data as
-described in the {[}FeatureExtraction vignette{]}
-(\url{https://github.com/OHDSI/FeatureExtraction/blob/master/inst/doc/UsingFeatureExtraction.pdf})
-as follows:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{settings <-}\StringTok{ }\KeywordTok{createTemporalCovariateSettings}\NormalTok{(}\DataTypeTok{useConditionEraStart =} \OtherTok{FALSE}\NormalTok{,}
-                                            \DataTypeTok{useConditionEraOverlap =} \OtherTok{FALSE}\NormalTok{,}
-                                            \DataTypeTok{useConditionOccurrence =} \OtherTok{FALSE}\NormalTok{,}
-                                            \DataTypeTok{useConditionEraGroupStart =} \OtherTok{FALSE}\NormalTok{,}
-                                            \DataTypeTok{useConditionEraGroupOverlap =} \OtherTok{FALSE}\NormalTok{,}
-                                            \DataTypeTok{useDrugExposure =} \OtherTok{FALSE}\NormalTok{,}
-                                            \DataTypeTok{useDrugEraStart =} \OtherTok{FALSE}\NormalTok{,}
-                                            \DataTypeTok{useDrugEraOverlap =} \OtherTok{FALSE}\NormalTok{,}
-                                            \DataTypeTok{useMeasurement =} \OtherTok{FALSE}\NormalTok{,}
-                                            \DataTypeTok{useMeasurementValue =} \OtherTok{TRUE}\NormalTok{,}
-                                            \DataTypeTok{useMeasurementRangeGroup =} \OtherTok{FALSE}\NormalTok{,}
-                                            \DataTypeTok{useProcedureOccurrence =} \OtherTok{FALSE}\NormalTok{,}
-                                            \DataTypeTok{useDeviceExposure =} \OtherTok{FALSE}\NormalTok{,}
-                                            \DataTypeTok{useObservation =} \OtherTok{FALSE}\NormalTok{,}
-                                            \DataTypeTok{excludedCovariateConceptIds =} \KeywordTok{c}\NormalTok{(}\DecValTok{316866}\NormalTok{),}
-                                            \DataTypeTok{addDescendantsToExclude =} \OtherTok{TRUE}\NormalTok{,}
-                                            \DataTypeTok{temporalStartDays =} \KeywordTok{seq}\NormalTok{(}\DataTypeTok{from =} \DecValTok{-365}\NormalTok{, }
-                                                                    \DataTypeTok{to =} \DecValTok{-1}\NormalTok{, }\DataTypeTok{by =} \DecValTok{12}\NormalTok{), }
-                                            \DataTypeTok{temporalEndDays =} \KeywordTok{c}\NormalTok{(}\KeywordTok{seq}\NormalTok{(}\DataTypeTok{from =} \DecValTok{-353}\NormalTok{, }
-                                                                    \DataTypeTok{to =} \DecValTok{0}\NormalTok{, }\DataTypeTok{by =} \DecValTok{12}\NormalTok{), }\DecValTok{0}\NormalTok{))}
-
-\NormalTok{plpData <-}\StringTok{ }\KeywordTok{getPlpData}\NormalTok{(}\DataTypeTok{connectionDetails =}\NormalTok{ connectionDetails,}
-                        \DataTypeTok{cdmDatabaseSchema =}\NormalTok{ cdmDatabaseSchema,}
-                        \DataTypeTok{cohortDatabaseSchema =} \StringTok{"results"}\NormalTok{,}
-                        \DataTypeTok{cohortTable =} \StringTok{"cohort"}\NormalTok{,}
-                        \DataTypeTok{cohortId =} \DecValTok{11}\NormalTok{,}
-                        \DataTypeTok{covariateSettings =}\NormalTok{ settings,}
-                        \DataTypeTok{outcomeDatabaseSchema =}\NormalTok{ resultsDatabaseSchema,}
-                        \DataTypeTok{outcomeTable =} \StringTok{"cohort"}\NormalTok{,}
-                        \DataTypeTok{outcomeIds =} \DecValTok{25}\NormalTok{,}
-                        \DataTypeTok{cdmVersion =} \DecValTok{5}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-Each CNN/RNN has several hyper-parameters that can be set as shown in
-the Tables above, but for this example we take the defaults.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\CommentTok{# specify the the CNN}
-\NormalTok{model <-}\StringTok{ }\KeywordTok{setCNNTorch}\NormalTok{(}\DataTypeTok{cnn_type=}\StringTok{'CNN'}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-Run the model training, for example with a testFraction = 0.2 and a
-split by person:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{results <-}\StringTok{ }\NormalTok{PatientLevelPrediction}\OperatorTok{::}\KeywordTok{runPlp}\NormalTok{(population, plpData, model,}
-                                          \DataTypeTok{testSplit=}\StringTok{'person'}\NormalTok{,}
-                                          \DataTypeTok{testFraction=}\FloatTok{0.2}\NormalTok{,}
-                                          \DataTypeTok{nfold=}\DecValTok{3}\NormalTok{, }
-                                          \DataTypeTok{splitSeed=}\DecValTok{1000}\NormalTok{) }
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{apply-the-trained-deep-learning-model}{%
-\section{Apply the trained Deep Learning
-model}\label{apply-the-trained-deep-learning-model}}
-
-Applying a Deep Learning is identical to the other models in the
-package:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\CommentTok{# load the trained model}
-\NormalTok{plpModel <-}\StringTok{ }\KeywordTok{loadPlpModel}\NormalTok{(}\KeywordTok{getwd}\NormalTok{(), }\StringTok{"<your model>"}\NormalTok{)}
-
-\CommentTok{# load the new plpData (should have the same temporal features!) and create the population}
-\NormalTok{plpData <-}\StringTok{ }\KeywordTok{loadPlpData}\NormalTok{(}\KeywordTok{getwd}\NormalTok{(), }\StringTok{"<your data>"}\NormalTok{)}
-
-\NormalTok{populationSettings <-}\StringTok{ }\NormalTok{plpModel}\OperatorTok{$}\NormalTok{populationSettings}
-\NormalTok{populationSettings}\OperatorTok{$}\NormalTok{plpData <-}\StringTok{ }\NormalTok{plpData}
-\NormalTok{population <-}\StringTok{ }\KeywordTok{do.call}\NormalTok{(createStudyPopulation, populationSettings)  }
-
-\CommentTok{# apply the trained model on the new data}
-\NormalTok{validationResults <-}\StringTok{ }\KeywordTok{applyModel}\NormalTok{(population, plpData, plpModel)}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{adding-new-architectures}{%
-\section{Adding new architectures}\label{adding-new-architectures}}
-
-It is possible to add new architectures in our framework using PyTorch
-or R Keras. We are happy to help you with this, please post your
-questions on the
-\href{www.github.com/OHDSI/PatientLevelPrediction/issues}{issue tracker}
-of the package.
-
-\hypertarget{acknowledgments}{%
-\section{Acknowledgments}\label{acknowledgments}}
-
-Considerable work has been dedicated to provide the
-\texttt{PatientLevelPrediction} package.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{citation}\NormalTok{(}\StringTok{"PatientLevelPrediction"}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-\begin{verbatim}
-## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design and
-## implementation of a standardized framework to generate and evaluate patient-level
-## prediction models using observational healthcare data." _Journal of the American
-## Medical Informatics Association_, *25*(8), 969-975. <URL:
-## https://doi.org/10.1093/jamia/ocy032>.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }
-\end{verbatim}
-
-\textbf{Please reference this paper if you use the PLP Package in your
-work:}
-
-\href{http://dx.doi.org/10.1093/jamia/ocy032}{Reps JM, Schuemie MJ,
-Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a
-standardized framework to generate and evaluate patient-level prediction
-models using observational healthcare data. J Am Med Inform Assoc.
-2018;25(8):969-975.}
-
-\end{document}
diff --git a/inst/doc/BuildingEnsembleModels.pdf b/inst/doc/BuildingEnsembleModels.pdf
deleted file mode 100644
index 37e714ebf..000000000
Binary files a/inst/doc/BuildingEnsembleModels.pdf and /dev/null differ
diff --git a/inst/doc/BuildingEnsembleModels.tex b/inst/doc/BuildingEnsembleModels.tex
deleted file mode 100644
index 85bc73667..000000000
--- a/inst/doc/BuildingEnsembleModels.tex
+++ /dev/null
@@ -1,369 +0,0 @@
-% Options for packages loaded elsewhere
-\PassOptionsToPackage{unicode}{hyperref}
-\PassOptionsToPackage{hyphens}{url}
-%
-\documentclass[
-]{article}
-\usepackage{lmodern}
-\usepackage{amssymb,amsmath}
-\usepackage{ifxetex,ifluatex}
-\ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex
-  \usepackage[T1]{fontenc}
-  \usepackage[utf8]{inputenc}
-  \usepackage{textcomp} % provide euro and other symbols
-\else % if luatex or xetex
-  \usepackage{unicode-math}
-  \defaultfontfeatures{Scale=MatchLowercase}
-  \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
-\fi
-% Use upquote if available, for straight quotes in verbatim environments
-\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
-\IfFileExists{microtype.sty}{% use microtype if available
-  \usepackage[]{microtype}
-  \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
-}{}
-\makeatletter
-\@ifundefined{KOMAClassName}{% if non-KOMA class
-  \IfFileExists{parskip.sty}{%
-    \usepackage{parskip}
-  }{% else
-    \setlength{\parindent}{0pt}
-    \setlength{\parskip}{6pt plus 2pt minus 1pt}}
-}{% if KOMA class
-  \KOMAoptions{parskip=half}}
-\makeatother
-\usepackage{xcolor}
-\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
-\IfFileExists{bookmark.sty}{\usepackage{bookmark}}{\usepackage{hyperref}}
-\hypersetup{
-  pdftitle={Building Ensemble Models},
-  pdfauthor={Xiaoyong Pan, Jenna Reps, Peter R. Rijnbeek},
-  hidelinks,
-  pdfcreator={LaTeX via pandoc}}
-\urlstyle{same} % disable monospaced font for URLs
-\usepackage[margin=1in]{geometry}
-\usepackage{color}
-\usepackage{fancyvrb}
-\newcommand{\VerbBar}{|}
-\newcommand{\VERB}{\Verb[commandchars=\\\{\}]}
-\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
-% Add ',fontsize=\small' for more characters per line
-\usepackage{framed}
-\definecolor{shadecolor}{RGB}{248,248,248}
-\newenvironment{Shaded}{\begin{snugshade}}{\end{snugshade}}
-\newcommand{\AlertTok}[1]{\textcolor[rgb]{0.94,0.16,0.16}{#1}}
-\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.77,0.63,0.00}{#1}}
-\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\BuiltInTok}[1]{#1}
-\newcommand{\CharTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
-\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
-\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{#1}}
-\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\ErrorTok}[1]{\textcolor[rgb]{0.64,0.00,0.00}{\textbf{#1}}}
-\newcommand{\ExtensionTok}[1]{#1}
-\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\ImportTok}[1]{#1}
-\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
-\newcommand{\NormalTok}[1]{#1}
-\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.81,0.36,0.00}{\textbf{#1}}}
-\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{#1}}
-\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
-\newcommand{\RegionMarkerTok}[1]{#1}
-\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\StringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\usepackage{graphicx,grffile}
-\makeatletter
-\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
-\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
-\makeatother
-% Scale images if necessary, so that they will not overflow the page
-% margins by default, and it is still possible to overwrite the defaults
-% using explicit options in \includegraphics[width, height, ...]{}
-\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
-% Set default figure placement to htbp
-\makeatletter
-\def\fps@figure{htbp}
-\makeatother
-\setlength{\emergencystretch}{3em} % prevent overfull lines
-\providecommand{\tightlist}{%
-  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
-\setcounter{secnumdepth}{5}
-\usepackage{fancyhdr}
-\pagestyle{fancy}
-\fancyhead{}
-\fancyhead[CO,CE]{Installation Guide}
-\fancyfoot[CO,CE]{PatientLevelPrediction Package Version 3.1.0}
-\fancyfoot[LE,RO]{\thepage}
-\renewcommand{\headrulewidth}{0.4pt}
-\renewcommand{\footrulewidth}{0.4pt}
-
-\title{Building Ensemble Models}
-\author{Xiaoyong Pan, Jenna Reps, Peter R. Rijnbeek}
-\date{2020-06-03}
-
-\begin{document}
-\maketitle
-
-{
-\setcounter{tocdepth}{2}
-\tableofcontents
-}
-\hypertarget{introduction}{%
-\section{Introduction}\label{introduction}}
-
-Ensemble models combine several models to improve the overall
-performance. Traditionally, weak learners were combined to boost
-performance but recent results show that combining several strong
-approaches can also result in a better performance. There are many
-examples in literature where ensemble models outperform individual
-models using stacking, i.e.~a final logistic regresssion layer accross
-the individual model outputs, but other approaches like weigthing has
-also shown promising results.
-
-This vignette describes how you can use the Observational Health Data
-Sciencs and Informatics (OHDSI)
-\href{http://github.com/OHDSI/PatientLevelPrediction}{\texttt{PatientLevelPrediction}}
-package to build ensemble models. This vignette assumes you have read
-and are comfortable with building single patient level prediction models
-as described in the
-\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf}{\texttt{BuildingPredictiveModels}
-vignette}.
-
-This will enable studying ensemble methods at scale in the OHDSI data
-network.
-
-\begin{figure}
-\centering
-\includegraphics{ensemble.png}
-\caption{Ensemble model}
-\end{figure}
-
-In PatientLevelPrediction package, four ensemble strategies have been
-implemented:
-
-\begin{enumerate}
-\def\labelenumi{\arabic{enumi}.}
-\tightlist
-\item
-  average ensemble: Calculate the average probability from individual
-  models
-\item
-  product ensemble: Calculate the product of probabilites from
-  individual models.
-\item
-  weighted ensemble: Calculate the weighted average probability from
-  individual models using train AUC as weights.
-\item
-  stacked ensemble: Train a logistics regression on outputs from
-  individual models
-\end{enumerate}
-
-\hypertarget{usage}{%
-\section{Usage}\label{usage}}
-
-Use the
-\href{http://github.com/OHDSI/PatientLevelPrediction}{\texttt{PatientLevelPrediction}}
-package to generate a \texttt{population} and \texttt{plpData} object.
-Alternatively, you can make use of the data simulator. The following
-code snippet creates a population of 12000 patients.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{data}\NormalTok{(plpDataSimulationProfile)}
-\KeywordTok{set.seed}\NormalTok{(}\DecValTok{1234}\NormalTok{)}
-\NormalTok{sampleSize <-}\StringTok{ }\DecValTok{2000}
-\NormalTok{plpData <-}\StringTok{ }\KeywordTok{simulatePlpData}\NormalTok{(}
-\NormalTok{  plpDataSimulationProfile,}
-  \DataTypeTok{n =}\NormalTok{ sampleSize}
-\NormalTok{)}
-
-\NormalTok{population <-}\StringTok{ }\KeywordTok{createStudyPopulation}\NormalTok{(}
-\NormalTok{  plpData,}
-  \DataTypeTok{outcomeId =} \DecValTok{2}\NormalTok{,}
-  \DataTypeTok{binary =} \OtherTok{TRUE}\NormalTok{,}
-  \DataTypeTok{firstExposureOnly =} \OtherTok{FALSE}\NormalTok{,}
-  \DataTypeTok{washoutPeriod =} \DecValTok{0}\NormalTok{,}
-  \DataTypeTok{removeSubjectsWithPriorOutcome =} \OtherTok{FALSE}\NormalTok{,}
-  \DataTypeTok{priorOutcomeLookback =} \DecValTok{99999}\NormalTok{,}
-  \DataTypeTok{requireTimeAtRisk =} \OtherTok{FALSE}\NormalTok{,}
-  \DataTypeTok{minTimeAtRisk =} \DecValTok{0}\NormalTok{,}
-  \DataTypeTok{riskWindowStart =} \DecValTok{0}\NormalTok{,}
-  \DataTypeTok{addExposureDaysToStart =} \OtherTok{FALSE}\NormalTok{,}
-  \DataTypeTok{riskWindowEnd =} \DecValTok{365}\NormalTok{,}
-  \DataTypeTok{addExposureDaysToEnd =} \OtherTok{FALSE}\NormalTok{,}
-  \DataTypeTok{verbosity =} \StringTok{"INFO"}
-\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-Specify the prediction algorithms to be combined.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\CommentTok{# Use LASSO logistic regression and Random Forest as base predictors}
-\NormalTok{model1 <-}\StringTok{ }\KeywordTok{setLassoLogisticRegression}\NormalTok{()}
-\NormalTok{model2 <-}\StringTok{ }\KeywordTok{setRandomForest}\NormalTok{()}
-\end{Highlighting}
-\end{Shaded}
-
-Specify a test fraction and a sequence of training set fractions.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{testFraction <-}\StringTok{ }\FloatTok{0.2}
-\end{Highlighting}
-\end{Shaded}
-
-Specify an ensembleStrategy to combine multiple predictors. The strategy
-used for ensembling the outputs from different models, it can be `mean',
-`product', `weighted' and `stacked': `mean' the average probability from
-differnt models `product' the product rule `weighted' the weighted
-average probability from different models using train AUC as weights.
-`stacked' the stakced ensemble trains a logistics regression on
-different models.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{ensembleStrategy <-}\StringTok{ 'stacked'}
-\end{Highlighting}
-\end{Shaded}
-
-Specify the test split to be used.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\CommentTok{# Use a split by person, alterantively a time split is possible}
-\NormalTok{testSplit <-}\StringTok{ 'person'}
-\end{Highlighting}
-\end{Shaded}
-
-Run the ensemble learning to combine model1 and model2. You can also use
-different plpData for different models.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{ensembleResults <-}\StringTok{ }\NormalTok{PatientLevelPrediction}\OperatorTok{::}\KeywordTok{runEnsembleModel}\NormalTok{(population, }
-                                   \DataTypeTok{dataList =} \KeywordTok{list}\NormalTok{(plpData, plpData), }
-                                   \DataTypeTok{modelList =} \KeywordTok{list}\NormalTok{(model1, model2),}
-                                   \DataTypeTok{testSplit=}\NormalTok{testSplit,}
-                                   \DataTypeTok{testFraction=}\NormalTok{testFraction,}
-                                   \DataTypeTok{nfold=}\DecValTok{3}\NormalTok{, }\DataTypeTok{splitSeed=}\DecValTok{1000}\NormalTok{, }
-                                   \DataTypeTok{ensembleStrategy =}\NormalTok{ ensembleStrategy) }
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{saving-and-loading-the-ensemble-model}{%
-\subsection{Saving and loading the ensemble
-model}\label{saving-and-loading-the-ensemble-model}}
-
-You can save and load the model using:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{saveEnsemblePlpModel}\NormalTok{(ensembleResults}\OperatorTok{$}\NormalTok{model, }\DataTypeTok{dirPath =} \KeywordTok{file.path}\NormalTok{(}\KeywordTok{getwd}\NormalTok{(), }\StringTok{"model"}\NormalTok{))}
-\NormalTok{ensembleModel <-}\StringTok{ }\KeywordTok{loadEnsemblePlpModel}\NormalTok{(}\KeywordTok{getwd}\NormalTok{(), }\StringTok{"model"}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{apply-ensemble-model}{%
-\section{Apply Ensemble model}\label{apply-ensemble-model}}
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{plpData <-}\StringTok{ }\KeywordTok{loadPlpData}\NormalTok{(}\StringTok{"<data file>"}\NormalTok{)}
-\NormalTok{populationSettings <-}\StringTok{ }\NormalTok{ensembleModel}\OperatorTok{$}\NormalTok{populationSettings}
-\NormalTok{populationSettings}\OperatorTok{$}\NormalTok{plpData <-}\StringTok{ }\NormalTok{plpData}
-\NormalTok{population <-}\StringTok{ }\KeywordTok{do.call}\NormalTok{(createStudyPopulation, populationSettings)}
-\end{Highlighting}
-\end{Shaded}
-
-Load the model.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{ensembleModel <-}\StringTok{ }\KeywordTok{loadEnsemblePlpModel}\NormalTok{(}\StringTok{"<model folder>"}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-Get the predictions by applying the model:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{prediction <-}\StringTok{ }\KeywordTok{applyEnsembleModel}\NormalTok{(population,}
-                                  \DataTypeTok{dataList =} \KeywordTok{list}\NormalTok{(plpData, plpData),}
-                                  \DataTypeTok{ensembleModel =}\NormalTok{ ensembleModel)}\OperatorTok{$}\NormalTok{prediction}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{demo}{%
-\section{Demo}\label{demo}}
-
-We have added a demo of the ensemble training:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\CommentTok{# Show all demos in our package: }
- \KeywordTok{demo}\NormalTok{(}\DataTypeTok{package =} \StringTok{"PatientLevelPrediction"}\NormalTok{)}
-
-\CommentTok{# Run the learning curve}
- \KeywordTok{demo}\NormalTok{(}\StringTok{"EnsembleModelDemo"}\NormalTok{, }\DataTypeTok{package =} \StringTok{"PatientLevelPrediction"}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{acknowledgments}{%
-\section{Acknowledgments}\label{acknowledgments}}
-
-Considerable work has been dedicated to provide the
-\texttt{PatientLevelPrediction} package.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{citation}\NormalTok{(}\StringTok{"PatientLevelPrediction"}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-\begin{verbatim}
-## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design and
-## implementation of a standardized framework to generate and evaluate patient-level
-## prediction models using observational healthcare data." _Journal of the American
-## Medical Informatics Association_, *25*(8), 969-975. <URL:
-## https://doi.org/10.1093/jamia/ocy032>.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }
-\end{verbatim}
-
-\textbf{Please reference this paper if you use the PLP Package in your
-work:}
-
-\href{http://dx.doi.org/10.1093/jamia/ocy032}{Reps JM, Schuemie MJ,
-Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a
-standardized framework to generate and evaluate patient-level prediction
-models using observational healthcare data. J Am Med Inform Assoc.
-2018;25(8):969-975.}
-
-\end{document}
diff --git a/inst/doc/BuildingMultiplePredictiveModels.tex b/inst/doc/BuildingMultiplePredictiveModels.tex
deleted file mode 100644
index a4e00e6a2..000000000
--- a/inst/doc/BuildingMultiplePredictiveModels.tex
+++ /dev/null
@@ -1,449 +0,0 @@
-% Options for packages loaded elsewhere
-\PassOptionsToPackage{unicode}{hyperref}
-\PassOptionsToPackage{hyphens}{url}
-%
-\documentclass[
-]{article}
-\usepackage{lmodern}
-\usepackage{amssymb,amsmath}
-\usepackage{ifxetex,ifluatex}
-\ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex
-  \usepackage[T1]{fontenc}
-  \usepackage[utf8]{inputenc}
-  \usepackage{textcomp} % provide euro and other symbols
-\else % if luatex or xetex
-  \usepackage{unicode-math}
-  \defaultfontfeatures{Scale=MatchLowercase}
-  \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
-\fi
-% Use upquote if available, for straight quotes in verbatim environments
-\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
-\IfFileExists{microtype.sty}{% use microtype if available
-  \usepackage[]{microtype}
-  \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
-}{}
-\makeatletter
-\@ifundefined{KOMAClassName}{% if non-KOMA class
-  \IfFileExists{parskip.sty}{%
-    \usepackage{parskip}
-  }{% else
-    \setlength{\parindent}{0pt}
-    \setlength{\parskip}{6pt plus 2pt minus 1pt}}
-}{% if KOMA class
-  \KOMAoptions{parskip=half}}
-\makeatother
-\usepackage{xcolor}
-\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
-\IfFileExists{bookmark.sty}{\usepackage{bookmark}}{\usepackage{hyperref}}
-\hypersetup{
-  pdftitle={Automatically Build Multiple Patient-Level Predictive Models},
-  pdfauthor={Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek},
-  hidelinks,
-  pdfcreator={LaTeX via pandoc}}
-\urlstyle{same} % disable monospaced font for URLs
-\usepackage[margin=1in]{geometry}
-\usepackage{color}
-\usepackage{fancyvrb}
-\newcommand{\VerbBar}{|}
-\newcommand{\VERB}{\Verb[commandchars=\\\{\}]}
-\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
-% Add ',fontsize=\small' for more characters per line
-\usepackage{framed}
-\definecolor{shadecolor}{RGB}{248,248,248}
-\newenvironment{Shaded}{\begin{snugshade}}{\end{snugshade}}
-\newcommand{\AlertTok}[1]{\textcolor[rgb]{0.94,0.16,0.16}{#1}}
-\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.77,0.63,0.00}{#1}}
-\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\BuiltInTok}[1]{#1}
-\newcommand{\CharTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
-\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
-\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{#1}}
-\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\ErrorTok}[1]{\textcolor[rgb]{0.64,0.00,0.00}{\textbf{#1}}}
-\newcommand{\ExtensionTok}[1]{#1}
-\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\ImportTok}[1]{#1}
-\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
-\newcommand{\NormalTok}[1]{#1}
-\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.81,0.36,0.00}{\textbf{#1}}}
-\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{#1}}
-\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
-\newcommand{\RegionMarkerTok}[1]{#1}
-\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\StringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\usepackage{graphicx,grffile}
-\makeatletter
-\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
-\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
-\makeatother
-% Scale images if necessary, so that they will not overflow the page
-% margins by default, and it is still possible to overwrite the defaults
-% using explicit options in \includegraphics[width, height, ...]{}
-\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
-% Set default figure placement to htbp
-\makeatletter
-\def\fps@figure{htbp}
-\makeatother
-\setlength{\emergencystretch}{3em} % prevent overfull lines
-\providecommand{\tightlist}{%
-  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
-\setcounter{secnumdepth}{5}
-\usepackage{fancyhdr}
-\pagestyle{fancy}
-\fancyhead{}
-\fancyhead[CO,CE]{Automatically Build Multiple Patient-Level Predictive Models}
-\fancyfoot[CO,CE]{PatientLevelPrediction Package Version 3.1.0}
-\fancyfoot[LE,RO]{\thepage}
-\renewcommand{\headrulewidth}{0.4pt}
-\renewcommand{\footrulewidth}{0.4pt}
-
-\title{Automatically Build Multiple Patient-Level Predictive Models}
-\author{Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek}
-\date{2020-06-03}
-
-\begin{document}
-\maketitle
-
-{
-\setcounter{tocdepth}{2}
-\tableofcontents
-}
-\hypertarget{introduction}{%
-\section{Introduction}\label{introduction}}
-
-In our
-\href{https://academic.oup.com/jamia/article/25/8/969/4989437}{\texttt{paper}},
-we propose a standardised framework for patient-level prediction that
-utilizes the OMOP CDM and standardized vocabularies, and describe the
-open-source software that we developed implementing the framework's
-pipeline. The framework is the first to enforce existing best practice
-guidelines and will enable open dissemination of models that can be
-extensively validated across the network of OHDSI collaborators.
-
-One our best practices is that we see the selection of models and all
-study setting as an emperical question, i.e.~we should use a data-driven
-approach in which we try many settings. This vignette describes how you
-can use the Observational Health Data Sciencs and Informatics (OHDSI)
-\href{http://github.com/OHDSI/PatientLevelPrediction}{\texttt{PatientLevelPrediction}}
-package to automatically build multiple patient-level predictive models,
-e.g.~different population settings, covariate settings, and
-modelsetting. This vignette assumes you have read and are comfortable
-with building single patient level prediction models as described in the
-\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf}{\texttt{BuildingPredictiveModels}
-vignette}.
-
-Note that it is also possible to generate a Study Package directly in
-Atlas that allows for multiple patient-level prediction analyses this is
-out-of-scope for this vignette.
-
-\hypertarget{creating-the-setting-lists}{%
-\section{Creating the setting lists}\label{creating-the-setting-lists}}
-
-To develop multiple models the user has to create a list of Study
-Populations Settings, Covariate Settings, and Model Settings. These
-lists will then be combined in a Model Analysis List and all
-combinations of the elements in this list will be automatically run by
-the package.
-
-\hypertarget{study-population-settings}{%
-\subsection{Study population settings}\label{study-population-settings}}
-
-Suppose we like to make the following three population settings:
-
-\begin{itemize}
-\tightlist
-\item
-  study population 1: allows persons who have the outcome to leave the
-  database before the end of time-at-risk and only those without the
-  outcome who are observed for the whole time-at-risk period
-  (requireTimeAtRisk = T).
-\item
-  study population 2: does not impose the restriction that persons who
-  do not experience the outcome need to be observed for the full
-  time-at-risk period (requireTimeAtRisk = F).
-\item
-  study population 3: does impose the restriction that persons who do
-  not experience the outcome need to be observed for the full
-  time-at-risk period (requireTimeAtRisk = T) and allows persons that
-  had the outcome before (removeSubjectsWithPriorOutcome = F)
-\end{itemize}
-
-The create a study population setting list we use the function
-\texttt{createStudyPopulationSettings} as described below:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\CommentTok{# define all study population settings}
-\NormalTok{studyPop1 <-}\StringTok{ }\KeywordTok{createStudyPopulationSettings}\NormalTok{(}\DataTypeTok{binary =}\NormalTok{ T,}
-                                          \DataTypeTok{includeAllOutcomes =}\NormalTok{ F,}
-                                          \DataTypeTok{removeSubjectsWithPriorOutcome =}\NormalTok{ T,}
-                                          \DataTypeTok{priorOutcomeLookback =} \DecValTok{99999}\NormalTok{,}
-                                          \DataTypeTok{requireTimeAtRisk =}\NormalTok{ T,}
-                                          \DataTypeTok{minTimeAtRisk=}\DecValTok{364}\NormalTok{,}
-                                          \DataTypeTok{riskWindowStart =} \DecValTok{1}\NormalTok{,}
-                                          \DataTypeTok{riskWindowEnd =} \DecValTok{365}\NormalTok{,}
-                                          \DataTypeTok{verbosity =} \StringTok{"INFO"}\NormalTok{)}
-
-\NormalTok{studyPop2 <-}\StringTok{ }\KeywordTok{createStudyPopulationSettings}\NormalTok{(}\DataTypeTok{binary =}\NormalTok{ T,}
-                                           \DataTypeTok{includeAllOutcomes =}\NormalTok{ F,}
-                                           \DataTypeTok{removeSubjectsWithPriorOutcome =}\NormalTok{ T,}
-                                           \DataTypeTok{priorOutcomeLookback =} \DecValTok{99999}\NormalTok{,}
-                                           \DataTypeTok{requireTimeAtRisk =}\NormalTok{ F,}
-                                           \DataTypeTok{minTimeAtRisk=}\DecValTok{364}\NormalTok{,}
-                                           \DataTypeTok{riskWindowStart =} \DecValTok{1}\NormalTok{,}
-                                           \DataTypeTok{riskWindowEnd =} \DecValTok{365}\NormalTok{,}
-                                           \DataTypeTok{verbosity =} \StringTok{"INFO"}\NormalTok{)}
-
-\NormalTok{studyPop3 <-}\StringTok{ }\KeywordTok{createStudyPopulationSettings}\NormalTok{(}\DataTypeTok{binary =}\NormalTok{ T,}
-                                           \DataTypeTok{includeAllOutcomes =}\NormalTok{ F,}
-                                           \DataTypeTok{removeSubjectsWithPriorOutcome =}\NormalTok{ F,}
-                                           \DataTypeTok{priorOutcomeLookback =} \DecValTok{99999}\NormalTok{,}
-                                           \DataTypeTok{requireTimeAtRisk =}\NormalTok{ T,}
-                                           \DataTypeTok{minTimeAtRisk=}\DecValTok{364}\NormalTok{,}
-                                           \DataTypeTok{riskWindowStart =} \DecValTok{1}\NormalTok{,}
-                                           \DataTypeTok{riskWindowEnd =} \DecValTok{365}\NormalTok{,}
-                                           \DataTypeTok{verbosity =} \StringTok{"INFO"}\NormalTok{)}
-                                           
-\CommentTok{# combine these in a population setting list}
-\NormalTok{populationSettingList <-}\StringTok{ }\KeywordTok{list}\NormalTok{(studyPop1,studyPop2,studyPop3)}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{covariate-settings}{%
-\subsection{Covariate settings}\label{covariate-settings}}
-
-The covariate settings are created using
-\texttt{createCovariateSettings}. We can create multiple covariate
-settings and then combine them in a list:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{covSet1 <-}\StringTok{ }\KeywordTok{createCovariateSettings}\NormalTok{(}\DataTypeTok{useDemographicsGender =}\NormalTok{ T, }
-                                   \DataTypeTok{useDemographicsAgeGroup =}\NormalTok{ T, }
-                                   \DataTypeTok{useConditionGroupEraAnyTimePrior =}\NormalTok{ T,}
-                                   \DataTypeTok{useDrugGroupEraAnyTimePrior =}\NormalTok{ T)}
-
-\NormalTok{covSet2 <-}\StringTok{ }\KeywordTok{createCovariateSettings}\NormalTok{(}\DataTypeTok{useDemographicsGender =}\NormalTok{ T, }
-                                   \DataTypeTok{useDemographicsAgeGroup =}\NormalTok{ T, }
-                                   \DataTypeTok{useConditionGroupEraAnyTimePrior =}\NormalTok{ T,}
-                                   \DataTypeTok{useDrugGroupEraAnyTimePrior =}\NormalTok{ F)}
-
-\NormalTok{covariateSettingList <-}\StringTok{ }\KeywordTok{list}\NormalTok{(covSet1, covSet2)}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{algorithm-settings}{%
-\subsection{Algorithm settings}\label{algorithm-settings}}
-
-The model settings requires running the setModel functions for the
-machine learning algorithms of interest and specifying the
-hyper-parameter search and then combining these into a list. For
-example, if we wanted to try a logistic regression, gradient boosting
-machine and ada boost model then:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{gbm <-}\StringTok{ }\KeywordTok{setGradientBoostingMachine}\NormalTok{()}
-\NormalTok{lr <-}\StringTok{ }\KeywordTok{setLassoLogisticRegression}\NormalTok{()}
-\NormalTok{ada <-}\StringTok{ }\KeywordTok{setAdaBoost}\NormalTok{()}
-
-\NormalTok{modelList <-}\StringTok{ }\KeywordTok{list}\NormalTok{(gbm, lr, ada)}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{model-analysis-list}{%
-\subsection{Model analysis list}\label{model-analysis-list}}
-
-To create the complete plp model settings use
-\texttt{createPlpModelSettings} to combine the population, covariate and
-model settings.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{modelAnalysisList <-}\StringTok{ }\KeywordTok{createPlpModelSettings}\NormalTok{(}\DataTypeTok{modelList =}\NormalTok{ modelList, }
-                                   \DataTypeTok{covariateSettingList =}\NormalTok{ covariateSettingList,}
-                                   \DataTypeTok{populationSettingList =}\NormalTok{ populationSettingList)}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{running-multiple-models}{%
-\section{Running multiple models}\label{running-multiple-models}}
-
-As we will be downloading loads of data in the multiple plp analysis it
-is useful to set the Andromeda temp folder to a directory with write
-access and plenty of space.
-\texttt{options(andromedaTempFolder\ =\ "c:/andromedaTemp")}
-
-To run the study requires setting up a connectionDetails object
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{dbms <-}\StringTok{ "your dbms"}
-\NormalTok{user <-}\StringTok{ "your username"}
-\NormalTok{pw <-}\StringTok{ "your password"}
-\NormalTok{server <-}\StringTok{ "your server"}
-\NormalTok{port <-}\StringTok{ "your port"}
-
-\NormalTok{connectionDetails <-}\StringTok{ }\NormalTok{DatabaseConnector}\OperatorTok{::}\KeywordTok{createConnectionDetails}\NormalTok{(}\DataTypeTok{dbms =}\NormalTok{ dbms,}
-                                                                \DataTypeTok{server =}\NormalTok{ server,}
-                                                                \DataTypeTok{user =}\NormalTok{ user,}
-                                                                \DataTypeTok{password =}\NormalTok{ pw,}
-                                                                \DataTypeTok{port =}\NormalTok{ port)}
-\end{Highlighting}
-\end{Shaded}
-
-Next you need to specify the cdmDatabaseSchema where your cdm database
-is found and workDatabaseSchema where your target population and outcome
-cohorts are and you need to specify a label for the database name: a
-string with a shareable name of the database (this will be shown to
-OHDSI researchers if the results get transported).
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{cdmDatabaseSchema <-}\StringTok{ "your cdmDatabaseSchema"}
-\NormalTok{workDatabaseSchema <-}\StringTok{ "your workDatabaseSchema"}
-\NormalTok{cdmDatabaseName <-}\StringTok{ "your cdmDatabaseName"}
-\end{Highlighting}
-\end{Shaded}
-
-Now you can run the multiple patient-level prediction analysis by
-specifying the target cohort ids and outcome ids
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{allresults <-}\StringTok{ }\KeywordTok{runPlpAnalyses}\NormalTok{(}\DataTypeTok{connectionDetails =}\NormalTok{ connectionDetails,}
-                           \DataTypeTok{cdmDatabaseSchema =}\NormalTok{ cdmDatabaseSchema,}
-                           \DataTypeTok{cdmDatabaseName =}\NormalTok{ cdmDatabaseName,}
-                           \DataTypeTok{oracleTempSchema =}\NormalTok{ cdmDatabaseSchema,}
-                           \DataTypeTok{cohortDatabaseSchema =}\NormalTok{ workDatabaseSchema,}
-                           \DataTypeTok{cohortTable =} \StringTok{"your cohort table"}\NormalTok{,}
-                           \DataTypeTok{outcomeDatabaseSchema =}\NormalTok{ workDatabaseSchema,}
-                           \DataTypeTok{outcomeTable =} \StringTok{"your cohort table"}\NormalTok{,}
-                           \DataTypeTok{cdmVersion =} \DecValTok{5}\NormalTok{,}
-                           \DataTypeTok{outputFolder =} \StringTok{"./PlpMultiOutput"}\NormalTok{,}
-                           \DataTypeTok{modelAnalysisList =}\NormalTok{ modelAnalysisList,}
-                           \DataTypeTok{cohortIds =} \KeywordTok{c}\NormalTok{(}\DecValTok{2484}\NormalTok{,}\DecValTok{6970}\NormalTok{),}
-                           \DataTypeTok{cohortNames =} \KeywordTok{c}\NormalTok{(}\StringTok{'visit 2010'}\NormalTok{,}\StringTok{'test cohort'}\NormalTok{),}
-                           \DataTypeTok{outcomeIds =} \KeywordTok{c}\NormalTok{(}\DecValTok{7331}\NormalTok{,}\DecValTok{5287}\NormalTok{),}
-                           \DataTypeTok{outcomeNames =}  \KeywordTok{c}\NormalTok{(}\StringTok{'outcome 1'}\NormalTok{,}\StringTok{'outcome 2'}\NormalTok{),}
-                           \DataTypeTok{maxSampleSize =} \OtherTok{NULL}\NormalTok{,}
-                           \DataTypeTok{minCovariateFraction =} \DecValTok{0}\NormalTok{,}
-                           \DataTypeTok{normalizeData =}\NormalTok{ T,}
-                           \DataTypeTok{testSplit =} \StringTok{"stratified"}\NormalTok{,}
-                           \DataTypeTok{testFraction =} \FloatTok{0.25}\NormalTok{,}
-                           \DataTypeTok{splitSeed =} \OtherTok{NULL}\NormalTok{,}
-                           \DataTypeTok{nfold =} \DecValTok{3}\NormalTok{,}
-                           \DataTypeTok{verbosity =} \StringTok{"INFO"}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-This will then save all the plpData objects from the study into
-``./PlpMultiOutput/plpData'', the populations for the analysis into
-``./PlpMultiOutput/population'' and the results into
-``./PlpMultiOutput/Result''. The csv named settings.csv found in
-``./PlpMultiOutput'' has a row for each prediction model developed and
-points to the plpData and population used for the model development, it
-also has descriptions of the cohorts and settings if these are input by
-the user.
-
-Note that if for some reason the run is interrupted, e.g.~because of an
-error, a new call to \texttt{RunPlpAnalyses} will continue and not
-restart until you remove the output folder.
-
-\hypertarget{validating-multiple-models}{%
-\section{Validating multiple models}\label{validating-multiple-models}}
-
-If you have access to multiple databases on the same server in different
-schemas you could evaluate accross these using this call:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{val <-}\StringTok{ }\KeywordTok{evaluateMultiplePlp}\NormalTok{(}\DataTypeTok{analysesLocation =} \StringTok{"./PlpMultiOutput"}\NormalTok{,}
-                           \DataTypeTok{outputLocation =} \StringTok{"./PlpMultiOutput/validation"}\NormalTok{,}
-                           \DataTypeTok{connectionDetails =}\NormalTok{ connectionDetails, }
-                           \DataTypeTok{validationSchemaTarget =} \KeywordTok{list}\NormalTok{(}\StringTok{'new_database_1.dbo'}\NormalTok{,}
-                                                              \StringTok{'new_database_2.dbo'}\NormalTok{),}
-                           \DataTypeTok{validationSchemaOutcome =} \KeywordTok{list}\NormalTok{(}\StringTok{'new_database_1.dbo'}\NormalTok{,}
-                                                              \StringTok{'new_database_2.dbo'}\NormalTok{),}
-                           \DataTypeTok{validationSchemaCdm =} \KeywordTok{list}\NormalTok{(}\StringTok{'new_database_1.dbo'}\NormalTok{,}
-                                                              \StringTok{'new_database_2.dbo'}\NormalTok{), }
-                           \DataTypeTok{databaseNames =} \KeywordTok{c}\NormalTok{(}\StringTok{'database1'}\NormalTok{,}\StringTok{'database2'}\NormalTok{),}
-                           \DataTypeTok{validationTableTarget =} \StringTok{'your new cohort table'}\NormalTok{,}
-                           \DataTypeTok{validationTableOutcome =} \StringTok{'your new cohort table'}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-This then saves the external validation results in the validation folder
-of the main study (the outputLocation you used in runPlpAnalyses).
-
-\hypertarget{viewing-the-results}{%
-\section{Viewing the results}\label{viewing-the-results}}
-
-To view the results for the multiple prediction analysis:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{viewMultiplePlp}\NormalTok{(}\DataTypeTok{analysesLocation=}\StringTok{"./PlpMultiOutput"}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-If the validation directory in ``./PlpMultiOutput'' has results, the
-external validation will also be displayed.
-
-\hypertarget{acknowledgments}{%
-\section{Acknowledgments}\label{acknowledgments}}
-
-Considerable work has been dedicated to provide the
-\texttt{PatientLevelPrediction} package.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{citation}\NormalTok{(}\StringTok{"PatientLevelPrediction"}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-\begin{verbatim}
-## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design and
-## implementation of a standardized framework to generate and evaluate patient-level
-## prediction models using observational healthcare data." _Journal of the American
-## Medical Informatics Association_, *25*(8), 969-975. <URL:
-## https://doi.org/10.1093/jamia/ocy032>.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }
-\end{verbatim}
-
-\textbf{Please reference this paper if you use the PLP Package in your
-work:}
-
-\href{http://dx.doi.org/10.1093/jamia/ocy032}{Reps JM, Schuemie MJ,
-Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a
-standardized framework to generate and evaluate patient-level prediction
-models using observational healthcare data. J Am Med Inform Assoc.
-2018;25(8):969-975.}
-
-\end{document}
diff --git a/inst/doc/BuildingPredictiveModels.tex b/inst/doc/BuildingPredictiveModels.tex
index 8a29c5af4..ae3ce9ba2 100644
--- a/inst/doc/BuildingPredictiveModels.tex
+++ b/inst/doc/BuildingPredictiveModels.tex
@@ -4,18 +4,21 @@
 %
 \documentclass[
 ]{article}
-\usepackage{lmodern}
-\usepackage{amssymb,amsmath}
-\usepackage{ifxetex,ifluatex}
-\ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex
+\usepackage{amsmath,amssymb}
+\usepackage{iftex}
+\ifPDFTeX
   \usepackage[T1]{fontenc}
   \usepackage[utf8]{inputenc}
   \usepackage{textcomp} % provide euro and other symbols
 \else % if luatex or xetex
-  \usepackage{unicode-math}
+  \usepackage{unicode-math} % this also loads fontspec
   \defaultfontfeatures{Scale=MatchLowercase}
   \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
 \fi
+\usepackage{lmodern}
+\ifPDFTeX\else
+  % xetex/luatex font selection
+\fi
 % Use upquote if available, for straight quotes in verbatim environments
 \IfFileExists{upquote.sty}{\usepackage{upquote}}{}
 \IfFileExists{microtype.sty}{% use microtype if available
@@ -33,14 +36,6 @@
   \KOMAoptions{parskip=half}}
 \makeatother
 \usepackage{xcolor}
-\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
-\IfFileExists{bookmark.sty}{\usepackage{bookmark}}{\usepackage{hyperref}}
-\hypersetup{
-  pdftitle={Building patient-level predictive models},
-  pdfauthor={Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek},
-  hidelinks,
-  pdfcreator={LaTeX via pandoc}}
-\urlstyle{same} % disable monospaced font for URLs
 \usepackage[margin=1in]{geometry}
 \usepackage{color}
 \usepackage{fancyvrb}
@@ -53,13 +48,13 @@
 \newenvironment{Shaded}{\begin{snugshade}}{\end{snugshade}}
 \newcommand{\AlertTok}[1]{\textcolor[rgb]{0.94,0.16,0.16}{#1}}
 \newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.77,0.63,0.00}{#1}}
+\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{#1}}
 \newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
 \newcommand{\BuiltInTok}[1]{#1}
 \newcommand{\CharTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
 \newcommand{\CommentTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
 \newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
+\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{#1}}
 \newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
 \newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{#1}}
 \newcommand{\DecValTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
@@ -67,7 +62,7 @@
 \newcommand{\ErrorTok}[1]{\textcolor[rgb]{0.64,0.00,0.00}{\textbf{#1}}}
 \newcommand{\ExtensionTok}[1]{#1}
 \newcommand{\FloatTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
+\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
 \newcommand{\ImportTok}[1]{#1}
 \newcommand{\InformationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
 \newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
@@ -76,13 +71,14 @@
 \newcommand{\OtherTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{#1}}
 \newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
 \newcommand{\RegionMarkerTok}[1]{#1}
-\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
+\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.81,0.36,0.00}{\textbf{#1}}}
 \newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
 \newcommand{\StringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
 \newcommand{\VariableTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
 \newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
 \newcommand{\WarningTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\usepackage{longtable,booktabs}
+\usepackage{longtable,booktabs,array}
+\usepackage{calc} % for calculating minipage widths
 % Correct order of tables after \paragraph or \subparagraph
 \usepackage{etoolbox}
 \makeatletter
@@ -91,7 +87,7 @@
 % Allow footnotes in longtable head/foot
 \IfFileExists{footnotehyper.sty}{\usepackage{footnotehyper}}{\usepackage{footnote}}
 \makesavenoteenv{longtable}
-\usepackage{graphicx,grffile}
+\usepackage{graphicx}
 \makeatletter
 \def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
 \def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
@@ -112,14 +108,26 @@
 \pagestyle{fancy}
 \fancyhead{}
 \fancyhead[CO,CE]{Installation Guide}
-\fancyfoot[CO,CE]{PatientLevelPrediction Package Version 3.1.0}
+\fancyfoot[CO,CE]{PatientLevelPrediction Package Version 6.3.7.9999}
 \fancyfoot[LE,RO]{\thepage}
 \renewcommand{\headrulewidth}{0.4pt}
 \renewcommand{\footrulewidth}{0.4pt}
+\ifLuaTeX
+  \usepackage{selnolig}  % disable illegal ligatures
+\fi
+\IfFileExists{bookmark.sty}{\usepackage{bookmark}}{\usepackage{hyperref}}
+\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
+\urlstyle{same}
+\hypersetup{
+  pdftitle={Building patient-level predictive models},
+  pdfauthor={Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek},
+  hidelinks,
+  pdfcreator={LaTeX via pandoc}}
 
 \title{Building patient-level predictive models}
-\author{Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek}
-\date{2020-06-03}
+\author{Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R.
+Rijnbeek}
+\date{2024-04-26}
 
 \begin{document}
 \maketitle
@@ -178,7 +186,7 @@ \section{Introduction}\label{introduction}}
 
 \begin{figure}
 \centering
-\includegraphics{Figure1.png}
+\includegraphics{Figure1.webp}
 \caption{The prediction problem}
 \end{figure}
 
@@ -192,13 +200,13 @@ \section{Introduction}\label{introduction}}
 
 \begin{figure}
 \centering
-\includegraphics{studydesign.png}
+\includegraphics{studydesign.webp}
 \caption{Design choices}
 \end{figure}
 
 \begin{figure}
 \centering
-\includegraphics{problems.png}
+\includegraphics{problems.webp}
 \caption{Examples of prediction problems}
 \end{figure}
 
@@ -208,7 +216,7 @@ \section{Introduction}\label{introduction}}
 and model evaluation using data from databases that are translated into
 the OMOP CDM. In this vignette we assume you have installed the package
 correctly using the
-\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/InstallationGuide.pdf}{\texttt{InstallationGuide}}.
+\href{https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/InstallationGuide.pdf}{\texttt{InstallationGuide}}.
 
 \hypertarget{study-specification}{%
 \section{Study specification}\label{study-specification}}
@@ -374,157 +382,112 @@ \subsection{Model development
 lunch). In our package we therefore aim to implement many algorithms.
 Furthermore, we made the system modular so you can add your own custom
 algorithms as described in more detail in the
-\href{Link\%20to\%20be\%20added}{\texttt{AddingCustomAlgorithms}}
+\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomModels.pdf}{\texttt{AddingCustomModels}}
 vignette.
 
 Our package currently contains the following algorithms to choose from:
 
-\begin{longtable}[]{@{}lll@{}}
-\toprule
-\begin{minipage}[b]{0.11\columnwidth}\raggedright
-Algorihm\strut
-\end{minipage} & \begin{minipage}[b]{0.55\columnwidth}\raggedright
-Description\strut
-\end{minipage} & \begin{minipage}[b]{0.25\columnwidth}\raggedright
-Hyper-parameters\strut
-\end{minipage}\tabularnewline
-\midrule
+\begin{longtable}[]{@{}
+  >{\raggedright\arraybackslash}p{(\columnwidth - 4\tabcolsep) * \real{0.1190}}
+  >{\raggedright\arraybackslash}p{(\columnwidth - 4\tabcolsep) * \real{0.6071}}
+  >{\raggedright\arraybackslash}p{(\columnwidth - 4\tabcolsep) * \real{0.2738}}@{}}
+\toprule\noalign{}
+\begin{minipage}[b]{\linewidth}\raggedright
+Algorihm
+\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright
+Description
+\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright
+Hyper-parameters
+\end{minipage} \\
+\midrule\noalign{}
 \endhead
-\begin{minipage}[t]{0.11\columnwidth}\raggedright
-Regularized Logistic Regression\strut
-\end{minipage} & \begin{minipage}[t]{0.55\columnwidth}\raggedright
-Lasso logistic regression belongs to the family of generalized linear
-models, where a linear combination of the variables is learned and
-finally a logistic function maps the linear combination to a value
-between 0 and 1. The lasso regularization adds a cost based on model
-complexity to the objective function when training the model. This cost
-is the sum of the absolute values of the linear combination of the
-coefficients. The model automatically performs feature selection by
-minimizing this cost. We use the Cyclic coordinate descent for logistic,
-Poisson and survival analysis (Cyclops) package to perform large-scale
-regularized logistic regression:
-\url{https://github.com/OHDSI/Cyclops}\strut
-\end{minipage} & \begin{minipage}[t]{0.25\columnwidth}\raggedright
-var (starting variance), seed\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.11\columnwidth}\raggedright
-Gradient boosting machines\strut
-\end{minipage} & \begin{minipage}[t]{0.55\columnwidth}\raggedright
-Gradient boosting machines is a boosting ensemble technique and in our
-framework it combines multiple decision trees. Boosting works by
-iteratively adding decision trees but adds more weight to the
-data-points that are misclassified by prior decision trees in the cost
-function when training the next tree. We use Extreme Gradient Boosting,
-which is an efficient implementation of the gradient boosting framework
-implemented in the xgboost R package available from CRAN.\strut
-\end{minipage} & \begin{minipage}[t]{0.25\columnwidth}\raggedright
-ntree (number of trees), max depth (max levels in tree), min rows
-(minimum data points in in node), learning rate, balance (balance class
-labels), seed\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.11\columnwidth}\raggedright
-Random forest\strut
-\end{minipage} & \begin{minipage}[t]{0.55\columnwidth}\raggedright
-Random forest is a bagging ensemble technique that combines multiple
-decision trees. The idea behind bagging is to reduce the likelihood of
-overfitting, by using weak classifiers, but combining multiple diverse
-weak classifiers into a strong classifier. Random forest accomplishes
-this by training multiple decision trees but only using a subset of the
-variables in each tree and the subset of variables differ between trees.
-Our packages uses the sklearn learn implementation of Random Forest in
-python.\strut
-\end{minipage} & \begin{minipage}[t]{0.25\columnwidth}\raggedright
-mtry (number of features in each tree),ntree (number of trees), maxDepth
-(max levels in tree), minRows (minimum data points in in node),balance
-(balance class labels), seed\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.11\columnwidth}\raggedright
-K-nearest neighbors\strut
-\end{minipage} & \begin{minipage}[t]{0.55\columnwidth}\raggedright
-K-nearest neighbors (KNN) is an algorithm that uses some metric to find
-the K closest labelled data-points, given the specified metric, to a new
-unlabelled data-point. The prediction of the new data-points is then the
-most prevalent class of the K-nearest labelled data-points. There is a
-sharing limitation of KNN, as the model requires labelled data to
-perform the prediction on new data, and it is often not possible to
-share this data across data sites.We included the BigKnn classifier
-developed in OHDSI which is a large scale k-nearest neighbor classifier
-using the Lucene search engine:
-\url{https://github.com/OHDSI/BigKnn}\strut
-\end{minipage} & \begin{minipage}[t]{0.25\columnwidth}\raggedright
-k (number of neighbours),weighted (weight by inverse frequency)\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.11\columnwidth}\raggedright
-Naive Bayes\strut
-\end{minipage} & \begin{minipage}[t]{0.55\columnwidth}\raggedright
-The Naive Bayes algorithm applies the Bayes theorem with the `naive'
-assumption of conditional independence between every pair of features
-given the value of the class variable. Based on the likelihood the data
-belongs to a class and the prior distribution of the class, a posterior
-distribution is obtained.\strut
-\end{minipage} & \begin{minipage}[t]{0.25\columnwidth}\raggedright
-none\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.11\columnwidth}\raggedright
-AdaBoost\strut
-\end{minipage} & \begin{minipage}[t]{0.55\columnwidth}\raggedright
-AdaBoost is a boosting ensemble technique. Boosting works by iteratively
-adding classifiers but adds more weight to the data-points that are
-misclassified by prior classifiers in the cost function when training
-the next classifier. We use the sklearn `AdaboostClassifier'
-implementation in Python.\strut
-\end{minipage} & \begin{minipage}[t]{0.25\columnwidth}\raggedright
-nEstimators (the maximum number of estimators at which boosting is
-terminated), learningRate (learning rate shrinks the contribution of
-each classifier by learning\_rate. There is a trade-off between
-learningRate and nEstimators)\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.11\columnwidth}\raggedright
-Decision Tree\strut
-\end{minipage} & \begin{minipage}[t]{0.55\columnwidth}\raggedright
-A decision tree is a classifier that partitions the variable space using
-individual tests selected using a greedy approach. It aims to find
-partitions that have the highest information gain to separate the
-classes. The decision tree can easily overfit by enabling a large number
-of partitions (tree depth) and often needs some regularization (e.g.,
-pruning or specifying hyper-parameters that limit the complexity of the
-model). We use the sklearn `DecisionTreeClassifier' implementation in
-Python.\strut
-\end{minipage} & \begin{minipage}[t]{0.25\columnwidth}\raggedright
-maxDepth (the maximum depth of the tree),
-minSamplesSplit,minSamplesLeaf, minImpuritySplit (threshold for early
-stopping in tree growth. A node will split if its impurity is above the
-threshold, otherwise it is a leaf.), seed,classWeight (`Balance' or
-`None')\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.11\columnwidth}\raggedright
-Multilayer Perception\strut
-\end{minipage} & \begin{minipage}[t]{0.55\columnwidth}\raggedright
-Neural networks contain multiple layers that weight their inputs using a
-non-linear function. The first layer is the input layer, the last layer
-is the output layer the between are the hidden layers. Neural networks
-are generally trained using feed forward back-propagation. This is when
-you go through the network with a data-point and calculate the error
-between the true label and predicted label, then go backwards through
-the network and update the linear function weights based on the error.
-This can also be performed as a batch, where multiple data-points are
-fee\strut
-\end{minipage} & \begin{minipage}[t]{0.25\columnwidth}\raggedright
-size (the number of hidden nodes), alpha (the l2 regularisation),
-seed\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.11\columnwidth}\raggedright
-Deep Learning\strut
-\end{minipage} & \begin{minipage}[t]{0.55\columnwidth}\raggedright
+\bottomrule\noalign{}
+\endlastfoot
+Regularized Logistic Regression & Lasso logistic regression belongs to
+the family of generalized linear models, where a linear combination of
+the variables is learned and finally a logistic function maps the linear
+combination to a value between 0 and 1. The lasso regularization adds a
+cost based on model complexity to the objective function when training
+the model. This cost is the sum of the absolute values of the linear
+combination of the coefficients. The model automatically performs
+feature selection by minimizing this cost. We use the Cyclic coordinate
+descent for logistic, Poisson and survival analysis (Cyclops) package to
+perform large-scale regularized logistic regression:
+\url{https://github.com/OHDSI/Cyclops} & var (starting variance),
+seed \\
+Gradient boosting machines & Gradient boosting machines is a boosting
+ensemble technique and in our framework it combines multiple decision
+trees. Boosting works by iteratively adding decision trees but adds more
+weight to the data-points that are misclassified by prior decision trees
+in the cost function when training the next tree. We use Extreme
+Gradient Boosting, which is an efficient implementation of the gradient
+boosting framework implemented in the xgboost R package available from
+CRAN. & ntree (number of trees), max depth (max levels in tree), min
+rows (minimum data points in in node), learning rate, balance (balance
+class labels), seed \\
+Random forest & Random forest is a bagging ensemble technique that
+combines multiple decision trees. The idea behind bagging is to reduce
+the likelihood of overfitting, by using weak classifiers, but combining
+multiple diverse weak classifiers into a strong classifier. Random
+forest accomplishes this by training multiple decision trees but only
+using a subset of the variables in each tree and the subset of variables
+differ between trees. Our packages uses the sklearn learn implementation
+of Random Forest in python. & mtry (number of features in each
+tree),ntree (number of trees), maxDepth (max levels in tree), minRows
+(minimum data points in in node),balance (balance class labels), seed \\
+K-nearest neighbors & K-nearest neighbors (KNN) is an algorithm that
+uses some metric to find the K closest labelled data-points, given the
+specified metric, to a new unlabelled data-point. The prediction of the
+new data-points is then the most prevalent class of the K-nearest
+labelled data-points. There is a sharing limitation of KNN, as the model
+requires labelled data to perform the prediction on new data, and it is
+often not possible to share this data across data sites.We included the
+BigKnn classifier developed in OHDSI which is a large scale k-nearest
+neighbor classifier using the Lucene search engine:
+\url{https://github.com/OHDSI/BigKnn} & k (number of
+neighbours),weighted (weight by inverse frequency) \\
+Naive Bayes & The Naive Bayes algorithm applies the Bayes theorem with
+the `naive' assumption of conditional independence between every pair of
+features given the value of the class variable. Based on the likelihood
+the data belongs to a class and the prior distribution of the class, a
+posterior distribution is obtained. & none \\
+AdaBoost & AdaBoost is a boosting ensemble technique. Boosting works by
+iteratively adding classifiers but adds more weight to the data-points
+that are misclassified by prior classifiers in the cost function when
+training the next classifier. We use the sklearn `AdaboostClassifier'
+implementation in Python. & nEstimators (the maximum number of
+estimators at which boosting is terminated), learningRate (learning rate
+shrinks the contribution of each classifier by learning\_rate. There is
+a trade-off between learningRate and nEstimators) \\
+Decision Tree & A decision tree is a classifier that partitions the
+variable space using individual tests selected using a greedy approach.
+It aims to find partitions that have the highest information gain to
+separate the classes. The decision tree can easily overfit by enabling a
+large number of partitions (tree depth) and often needs some
+regularization (e.g., pruning or specifying hyper-parameters that limit
+the complexity of the model). We use the sklearn
+`DecisionTreeClassifier' implementation in Python. & maxDepth (the
+maximum depth of the tree), minSamplesSplit,minSamplesLeaf,
+minImpuritySplit (threshold for early stopping in tree growth. A node
+will split if its impurity is above the threshold, otherwise it is a
+leaf.), seed,classWeight (`Balance' or `None') \\
+Multilayer Perception & Neural networks contain multiple layers that
+weight their inputs using a non-linear function. The first layer is the
+input layer, the last layer is the output layer the between are the
+hidden layers. Neural networks are generally trained using feed forward
+back-propagation. This is when you go through the network with a
+data-point and calculate the error between the true label and predicted
+label, then go backwards through the network and update the linear
+function weights based on the error. This can also be performed as a
+batch, where multiple data-points are fee & size (the number of hidden
+nodes), alpha (the l2 regularisation), seed \\
+Deep Learning (now in seperate DeepPatientLevelPrediction R package) &
 Deep learning such as deep nets, convolutional neural networks or
 recurrent neural networks are similar to a neural network but have
 multiple hidden layers that aim to learn latent representations useful
 for prediction. In the seperate BuildingDeepLearningModels vignette we
-describe these models and hyper-parameters in more detail\strut
-\end{minipage} & \begin{minipage}[t]{0.25\columnwidth}\raggedright
-see BuildingDeepLearningModels vignette\strut
-\end{minipage}\tabularnewline
-\bottomrule
+describe these models and hyper-parameters in more detail & see
+OHDSI/DeepPatientLevelPrediction \\
 \end{longtable}
 
 Furthermore, we have to decide on the \textbf{covariates} that we will
@@ -578,113 +541,44 @@ \subsection{Study Specification}\label{study-specification-1}}
 Logistic Regression and will use the default parameters. We will do a
 75\%-25\% split by person.
 
-\begin{longtable}[]{@{}ll@{}}
-\toprule
-\begin{minipage}[b]{0.42\columnwidth}\raggedright
-Definition\strut
-\end{minipage} & \begin{minipage}[b]{0.52\columnwidth}\raggedright
-Value\strut
-\end{minipage}\tabularnewline
-\midrule
+\begin{longtable}[]{@{}
+  >{\raggedright\arraybackslash}p{(\columnwidth - 2\tabcolsep) * \real{0.2361}}
+  >{\raggedright\arraybackslash}p{(\columnwidth - 2\tabcolsep) * \real{0.7639}}@{}}
+\toprule\noalign{}
+\begin{minipage}[b]{\linewidth}\raggedright
+Definition
+\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright
+Value
+\end{minipage} \\
+\midrule\noalign{}
 \endhead
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-\textbf{Problem Definition}\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Target Cohort (T)\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-`Patients who are newly diagnosed with Atrial Fibrillation' defined as
-the first condition record of cardiac arrhythmia, which is followed by
-another cardiac arrhythmia condition record, at least two drug records
-for a drug used to treat arrhythmias, or a procedure to treat
-arrhythmias.\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Outcome Cohort (O)\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-`Ischemic stroke events' defined as ischemic stroke condition records
-during an inpatient or ER visit; successive records with \textgreater{}
-180 day gap are considered independent episodes.\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Time-at-risk (TAR)\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-1 day till 365 days from cohort start\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-\textbf{Population Definition}\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Washout Period\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-1095\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Enter the target cohort multiple times?\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-No\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Allow prior outcomes?\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-Yes\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Start of time-at-risk\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-1 day\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-End of time-at-risk\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-365 days\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Require a minimum amount of time-at-risk?\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-Yes (364 days)\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-\textbf{Model Development}\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Algorithm\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-Regularized Logistic Regression\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Hyper-parameters\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-variance = 0.01 (Default)\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Covariates\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-Gender, Age, Conditions (ever before, \textless365), Drugs Groups (ever
-before, \textless365), and Visit Count\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Data split\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-75\% train, 25\% test. Randomly assigned by person\strut
-\end{minipage}\tabularnewline
-\bottomrule
+\bottomrule\noalign{}
+\endlastfoot
+\textbf{Problem Definition} & \\
+Target Cohort (T) & `Patients who are newly diagnosed with Atrial
+Fibrillation' defined as the first condition record of cardiac
+arrhythmia, which is followed by another cardiac arrhythmia condition
+record, at least two drug records for a drug used to treat arrhythmias,
+or a procedure to treat arrhythmias. \\
+Outcome Cohort (O) & `Ischemic stroke events' defined as ischemic stroke
+condition records during an inpatient or ER visit; successive records
+with \textgreater{} 180 day gap are considered independent episodes. \\
+Time-at-risk (TAR) & 1 day till 365 days from cohort start \\
+& \\
+\textbf{Population Definition} & \\
+Washout Period & 1095 \\
+Enter the target cohort multiple times? & No \\
+Allow prior outcomes? & Yes \\
+Start of time-at-risk & 1 day \\
+End of time-at-risk & 365 days \\
+Require a minimum amount of time-at-risk? & Yes (364 days) \\
+& \\
+\textbf{Model Development} & \\
+Algorithm & Regularized Logistic Regression \\
+Hyper-parameters & variance = 0.01 (Default) \\
+Covariates & Gender, Age, Conditions (ever before, \textless365), Drugs
+Groups (ever before, \textless365), and Visit Count \\
+Data split & 75\% train, 25\% test. Randomly assigned by person \\
 \end{longtable}
 
 According to the best practices we need to make a protocol that
@@ -748,7 +642,7 @@ \subsubsection{ATLAS cohort builder}\label{atlas-cohort-builder}}
 
 \begin{figure}
 \centering
-\includegraphics{example1/ATLAS_T.png}
+\includegraphics{example1/ATLAS_T.webp}
 \caption{Target Cohort Atrial Fibrillation}
 \end{figure}
 
@@ -765,7 +659,7 @@ \subsubsection{ATLAS cohort builder}\label{atlas-cohort-builder}}
 
 \begin{figure}
 \centering
-\includegraphics{example1/ATLAS_O.png}
+\includegraphics{example1/ATLAS_O.webp}
 \caption{Outcome Cohort Stroke}
 \end{figure}
 
@@ -809,94 +703,94 @@ \subsubsection{Custom cohorts}\label{custom-cohorts}}
 \CommentTok{Create a table to store the persons in the T and C cohort}
 \CommentTok{*/}
 
-\ControlFlowTok{IF}\NormalTok{ OBJECT_ID(}\StringTok{'@resultsDatabaseSchema.PLPAFibStrokeCohort'}\NormalTok{, }\StringTok{'U'}\NormalTok{) }\KeywordTok{IS} \KeywordTok{NOT} \KeywordTok{NULL} 
+\ControlFlowTok{IF}\NormalTok{ OBJECT\_ID(}\StringTok{\textquotesingle{}@resultsDatabaseSchema.PLPAFibStrokeCohort\textquotesingle{}}\NormalTok{, }\StringTok{\textquotesingle{}U\textquotesingle{}}\NormalTok{) }\KeywordTok{IS} \KeywordTok{NOT} \KeywordTok{NULL} 
 \KeywordTok{DROP} \KeywordTok{TABLE}\NormalTok{ @resultsDatabaseSchema.PLPAFibStrokeCohort;}
 
 \KeywordTok{CREATE} \KeywordTok{TABLE}\NormalTok{ @resultsDatabaseSchema.PLPAFibStrokeCohort }
 \NormalTok{( }
-\NormalTok{cohort_definition_id }\DataTypeTok{INT}\NormalTok{, }
-\NormalTok{subject_id BIGINT,}
-\NormalTok{cohort_start_date }\DataTypeTok{DATE}\NormalTok{, }
-\NormalTok{cohort_end_date }\DataTypeTok{DATE}
+\NormalTok{cohort\_definition\_id }\DataTypeTok{INT}\NormalTok{, }
+\NormalTok{subject\_id BIGINT,}
+\NormalTok{cohort\_start\_date }\DataTypeTok{DATE}\NormalTok{, }
+\NormalTok{cohort\_end\_date }\DataTypeTok{DATE}
 \NormalTok{);}
 
 
 \CommentTok{/*}
 \CommentTok{T cohort:  [PatientLevelPrediction vignette]:  T : patients who are newly }
 \CommentTok{diagnosed with Atrial fibrillation}
-\CommentTok{- persons with a condition occurrence record of 'Atrial fibrillation' or }
+\CommentTok{{-} persons with a condition occurrence record of \textquotesingle{}Atrial fibrillation\textquotesingle{} or }
 \CommentTok{any descendants, indexed at the first diagnosis}
-\CommentTok{- who have >1095 days of prior observation before their first diagnosis}
-\CommentTok{- and have no warfarin exposure any time prior to first AFib diagnosis}
+\CommentTok{{-} who have \textgreater{}1095 days of prior observation before their first diagnosis}
+\CommentTok{{-} and have no warfarin exposure any time prior to first AFib diagnosis}
 \CommentTok{*/}
-\KeywordTok{INSERT} \KeywordTok{INTO}\NormalTok{ @resultsDatabaseSchema.AFibStrokeCohort (cohort_definition_id, }
-\NormalTok{subject_id, }
-\NormalTok{cohort_start_date, }
-\NormalTok{cohort_end_date)}
-\KeywordTok{SELECT} \DecValTok{1} \KeywordTok{AS}\NormalTok{ cohort_definition_id,}
-\NormalTok{AFib.person_id }\KeywordTok{AS}\NormalTok{ subject_id,}
-\NormalTok{AFib.condition_start_date }\KeywordTok{AS}\NormalTok{ cohort_start_date,}
-\NormalTok{observation_period.observation_period_end_date }\KeywordTok{AS}\NormalTok{ cohort_end_date}
+\KeywordTok{INSERT} \KeywordTok{INTO}\NormalTok{ @resultsDatabaseSchema.AFibStrokeCohort (cohort\_definition\_id, }
+\NormalTok{subject\_id, }
+\NormalTok{cohort\_start\_date, }
+\NormalTok{cohort\_end\_date)}
+\KeywordTok{SELECT} \DecValTok{1} \KeywordTok{AS}\NormalTok{ cohort\_definition\_id,}
+\NormalTok{AFib.person\_id }\KeywordTok{AS}\NormalTok{ subject\_id,}
+\NormalTok{AFib.condition\_start\_date }\KeywordTok{AS}\NormalTok{ cohort\_start\_date,}
+\NormalTok{observation\_period.observation\_period\_end\_date }\KeywordTok{AS}\NormalTok{ cohort\_end\_date}
 \KeywordTok{FROM}
 \NormalTok{(}
-  \KeywordTok{SELECT}\NormalTok{ person_id, }\FunctionTok{min}\NormalTok{(condition_start_date) }\KeywordTok{as}\NormalTok{ condition_start_date}
-  \KeywordTok{FROM}\NormalTok{ @cdmDatabaseSchema.condition_occurrence}
-  \KeywordTok{WHERE}\NormalTok{ condition_concept_id }\KeywordTok{IN}\NormalTok{ (}\KeywordTok{SELECT}\NormalTok{ descendant_concept_id }\KeywordTok{FROM} 
-\NormalTok{  @cdmDatabaseSchema.concept_ancestor }\KeywordTok{WHERE}\NormalTok{ ancestor_concept_id }\KeywordTok{IN} 
+  \KeywordTok{SELECT}\NormalTok{ person\_id, }\FunctionTok{min}\NormalTok{(condition\_start\_date) }\KeywordTok{as}\NormalTok{ condition\_start\_date}
+  \KeywordTok{FROM}\NormalTok{ @cdmDatabaseSchema.condition\_occurrence}
+  \KeywordTok{WHERE}\NormalTok{ condition\_concept\_id }\KeywordTok{IN}\NormalTok{ (}\KeywordTok{SELECT}\NormalTok{ descendant\_concept\_id }\KeywordTok{FROM} 
+\NormalTok{  @cdmDatabaseSchema.concept\_ancestor }\KeywordTok{WHERE}\NormalTok{ ancestor\_concept\_id }\KeywordTok{IN} 
 \NormalTok{  (}\DecValTok{313217} \CommentTok{/*atrial fibrillation*/}\NormalTok{))}
-  \KeywordTok{GROUP} \KeywordTok{BY}\NormalTok{ person_id}
+  \KeywordTok{GROUP} \KeywordTok{BY}\NormalTok{ person\_id}
 \NormalTok{) AFib}
-  \KeywordTok{INNER} \KeywordTok{JOIN}\NormalTok{ @cdmDatabaseSchema.observation_period}
-  \KeywordTok{ON}\NormalTok{ AFib.person_id }\OperatorTok{=}\NormalTok{ observation_period.person_id}
-  \KeywordTok{AND}\NormalTok{ AFib.condition_start_date }\OperatorTok{>=}\NormalTok{ dateadd(dd,}\DecValTok{1095}\NormalTok{, }
-\NormalTok{  observation_period.observation_period_start_date)}
-  \KeywordTok{AND}\NormalTok{ AFib.condition_start_date }\OperatorTok{<=}\NormalTok{ observation_period.observation_period_end_date}
+  \KeywordTok{INNER} \KeywordTok{JOIN}\NormalTok{ @cdmDatabaseSchema.observation\_period}
+  \KeywordTok{ON}\NormalTok{ AFib.person\_id }\OperatorTok{=}\NormalTok{ observation\_period.person\_id}
+  \KeywordTok{AND}\NormalTok{ AFib.condition\_start\_date }\OperatorTok{\textgreater{}=}\NormalTok{ dateadd(dd,}\DecValTok{1095}\NormalTok{, }
+\NormalTok{  observation\_period.observation\_period\_start\_date)}
+  \KeywordTok{AND}\NormalTok{ AFib.condition\_start\_date }\OperatorTok{\textless{}=}\NormalTok{ observation\_period.observation\_period\_end\_date}
   \KeywordTok{LEFT} \KeywordTok{JOIN}
 \NormalTok{  (}
-  \KeywordTok{SELECT}\NormalTok{ person_id, }\FunctionTok{min}\NormalTok{(drug_exposure_start_date) }\KeywordTok{as}\NormalTok{ drug_exposure_start_date}
-  \KeywordTok{FROM}\NormalTok{ @cdmDatabaseSchema.drug_exposure}
-  \KeywordTok{WHERE}\NormalTok{ drug_concept_id }\KeywordTok{IN}\NormalTok{ (}\KeywordTok{SELECT}\NormalTok{ descendant_concept_id }\KeywordTok{FROM} 
-\NormalTok{  @cdmDatabaseSchema.concept_ancestor }\KeywordTok{WHERE}\NormalTok{ ancestor_concept_id }\KeywordTok{IN} 
+  \KeywordTok{SELECT}\NormalTok{ person\_id, }\FunctionTok{min}\NormalTok{(drug\_exposure\_start\_date) }\KeywordTok{as}\NormalTok{ drug\_exposure\_start\_date}
+  \KeywordTok{FROM}\NormalTok{ @cdmDatabaseSchema.drug\_exposure}
+  \KeywordTok{WHERE}\NormalTok{ drug\_concept\_id }\KeywordTok{IN}\NormalTok{ (}\KeywordTok{SELECT}\NormalTok{ descendant\_concept\_id }\KeywordTok{FROM} 
+\NormalTok{  @cdmDatabaseSchema.concept\_ancestor }\KeywordTok{WHERE}\NormalTok{ ancestor\_concept\_id }\KeywordTok{IN} 
 \NormalTok{  (}\DecValTok{1310149} \CommentTok{/*warfarin*/}\NormalTok{))}
-  \KeywordTok{GROUP} \KeywordTok{BY}\NormalTok{ person_id}
+  \KeywordTok{GROUP} \KeywordTok{BY}\NormalTok{ person\_id}
 \NormalTok{  ) warfarin}
-  \KeywordTok{ON}\NormalTok{ Afib.person_id }\OperatorTok{=}\NormalTok{ warfarin.person_id}
-  \KeywordTok{AND}\NormalTok{ Afib.condition_start_date }\OperatorTok{>}\NormalTok{ warfarin.drug_exposure_start_date}
-  \KeywordTok{WHERE}\NormalTok{ warfarin.person_id }\KeywordTok{IS} \KeywordTok{NULL}
+  \KeywordTok{ON}\NormalTok{ Afib.person\_id }\OperatorTok{=}\NormalTok{ warfarin.person\_id}
+  \KeywordTok{AND}\NormalTok{ Afib.condition\_start\_date }\OperatorTok{\textgreater{}}\NormalTok{ warfarin.drug\_exposure\_start\_date}
+  \KeywordTok{WHERE}\NormalTok{ warfarin.person\_id }\KeywordTok{IS} \KeywordTok{NULL}
 \NormalTok{  ;}
   
   \CommentTok{/*}
 \CommentTok{  C cohort:  [PatientLevelPrediction vignette]:  O: Ischemic stroke events}
-\CommentTok{  - inpatient visits that include a condition occurrence record for }
-\CommentTok{  'cerebral infarction' and descendants, 'cerebral thrombosis', }
-\CommentTok{  'cerebral embolism', 'cerebral artery occlusion' }
+\CommentTok{  {-} inpatient visits that include a condition occurrence record for }
+\CommentTok{  \textquotesingle{}cerebral infarction\textquotesingle{} and descendants, \textquotesingle{}cerebral thrombosis\textquotesingle{}, }
+\CommentTok{  \textquotesingle{}cerebral embolism\textquotesingle{}, \textquotesingle{}cerebral artery occlusion\textquotesingle{} }
 \CommentTok{  */}
-  \KeywordTok{INSERT} \KeywordTok{INTO}\NormalTok{ @resultsDatabaseSchema.AFibStrokeCohort (cohort_definition_id, }
-\NormalTok{  subject_id, }
-\NormalTok{  cohort_start_date, }
-\NormalTok{  cohort_end_date)}
-  \KeywordTok{SELECT} \DecValTok{2} \KeywordTok{AS}\NormalTok{ cohort_definition_id,}
-\NormalTok{  visit_occurrence.person_id }\KeywordTok{AS}\NormalTok{ subject_id,}
-\NormalTok{  visit_occurrence.visit_start_date }\KeywordTok{AS}\NormalTok{ cohort_start_date,}
-\NormalTok{  visit_occurrence.visit_end_date }\KeywordTok{AS}\NormalTok{ cohort_end_date}
+  \KeywordTok{INSERT} \KeywordTok{INTO}\NormalTok{ @resultsDatabaseSchema.AFibStrokeCohort (cohort\_definition\_id, }
+\NormalTok{  subject\_id, }
+\NormalTok{  cohort\_start\_date, }
+\NormalTok{  cohort\_end\_date)}
+  \KeywordTok{SELECT} \DecValTok{2} \KeywordTok{AS}\NormalTok{ cohort\_definition\_id,}
+\NormalTok{  visit\_occurrence.person\_id }\KeywordTok{AS}\NormalTok{ subject\_id,}
+\NormalTok{  visit\_occurrence.visit\_start\_date }\KeywordTok{AS}\NormalTok{ cohort\_start\_date,}
+\NormalTok{  visit\_occurrence.visit\_end\_date }\KeywordTok{AS}\NormalTok{ cohort\_end\_date}
   \KeywordTok{FROM}  
 \NormalTok{  (}
-  \KeywordTok{SELECT}\NormalTok{ person_id, condition_start_date}
-  \KeywordTok{FROM}\NormalTok{ @cdmDatabaseSchema.condition_occurrence}
-  \KeywordTok{WHERE}\NormalTok{ condition_concept_id }\KeywordTok{IN}\NormalTok{ (}\KeywordTok{SELECT} \KeywordTok{DISTINCT}\NormalTok{ descendant_concept_id }\KeywordTok{FROM} 
-\NormalTok{  @cdmDatabaseSchema.concept_ancestor }\KeywordTok{WHERE}\NormalTok{ ancestor_concept_id }\KeywordTok{IN} 
-\NormalTok{  (}\DecValTok{443454} \CommentTok{/*cerebral infarction*/}\NormalTok{) }\KeywordTok{OR}\NormalTok{ descendant_concept_id }\KeywordTok{IN} 
+  \KeywordTok{SELECT}\NormalTok{ person\_id, condition\_start\_date}
+  \KeywordTok{FROM}\NormalTok{ @cdmDatabaseSchema.condition\_occurrence}
+  \KeywordTok{WHERE}\NormalTok{ condition\_concept\_id }\KeywordTok{IN}\NormalTok{ (}\KeywordTok{SELECT} \KeywordTok{DISTINCT}\NormalTok{ descendant\_concept\_id }\KeywordTok{FROM} 
+\NormalTok{  @cdmDatabaseSchema.concept\_ancestor }\KeywordTok{WHERE}\NormalTok{ ancestor\_concept\_id }\KeywordTok{IN} 
+\NormalTok{  (}\DecValTok{443454} \CommentTok{/*cerebral infarction*/}\NormalTok{) }\KeywordTok{OR}\NormalTok{ descendant\_concept\_id }\KeywordTok{IN} 
 \NormalTok{  (}\DecValTok{441874} \CommentTok{/*cerebral thrombosis*/}\NormalTok{, }\DecValTok{375557} \CommentTok{/*cerebral embolism*/}\NormalTok{, }
   \DecValTok{372924} \CommentTok{/*cerebral artery occlusion*/}\NormalTok{))}
 \NormalTok{  ) stroke}
-  \KeywordTok{INNER} \KeywordTok{JOIN}\NormalTok{ @cdmDatabaseSchema.visit_occurrence}
-  \KeywordTok{ON}\NormalTok{ stroke.person_id }\OperatorTok{=}\NormalTok{ visit_occurrence.person_id}
-  \KeywordTok{AND}\NormalTok{ stroke.condition_start_date }\OperatorTok{>=}\NormalTok{ visit_occurrence.visit_start_date}
-  \KeywordTok{AND}\NormalTok{ stroke.condition_start_date }\OperatorTok{<=}\NormalTok{ visit_occurrence.visit_end_date}
-  \KeywordTok{AND}\NormalTok{ visit_occurrence.visit_concept_id }\KeywordTok{IN}\NormalTok{ (}\DecValTok{9201}\NormalTok{, }\DecValTok{262} \CommentTok{/*'Inpatient Visit'  or }
-\CommentTok{  'Emergency Room and Inpatient Visit'*/}\NormalTok{)}
-  \KeywordTok{GROUP} \KeywordTok{BY}\NormalTok{ visit_occurrence.person_id, visit_occurrence.visit_start_date, }
-\NormalTok{  visit_occurrence.visit_end_date}
+  \KeywordTok{INNER} \KeywordTok{JOIN}\NormalTok{ @cdmDatabaseSchema.visit\_occurrence}
+  \KeywordTok{ON}\NormalTok{ stroke.person\_id }\OperatorTok{=}\NormalTok{ visit\_occurrence.person\_id}
+  \KeywordTok{AND}\NormalTok{ stroke.condition\_start\_date }\OperatorTok{\textgreater{}=}\NormalTok{ visit\_occurrence.visit\_start\_date}
+  \KeywordTok{AND}\NormalTok{ stroke.condition\_start\_date }\OperatorTok{\textless{}=}\NormalTok{ visit\_occurrence.visit\_end\_date}
+  \KeywordTok{AND}\NormalTok{ visit\_occurrence.visit\_concept\_id }\KeywordTok{IN}\NormalTok{ (}\DecValTok{9201}\NormalTok{, }\DecValTok{262} \CommentTok{/*\textquotesingle{}Inpatient Visit\textquotesingle{}  or }
+\CommentTok{  \textquotesingle{}Emergency Room and Inpatient Visit\textquotesingle{}*/}\NormalTok{)}
+  \KeywordTok{GROUP} \KeywordTok{BY}\NormalTok{ visit\_occurrence.person\_id, visit\_occurrence.visit\_start\_date, }
+\NormalTok{  visit\_occurrence.visit\_end\_date}
 \NormalTok{  ;}
   
 \end{Highlighting}
@@ -922,14 +816,14 @@ \subsubsection{Custom cohorts}\label{custom-cohorts}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{  connectionDetails <-}\StringTok{ }\KeywordTok{createConnectionDetails}\NormalTok{(}\DataTypeTok{dbms =} \StringTok{"postgresql"}\NormalTok{, }
-  \DataTypeTok{server =} \StringTok{"localhost/ohdsi"}\NormalTok{, }
-  \DataTypeTok{user =} \StringTok{"joe"}\NormalTok{, }
-  \DataTypeTok{password =} \StringTok{"supersecret"}\NormalTok{)}
+\NormalTok{  connectionDetails }\OtherTok{\textless{}{-}} \FunctionTok{createConnectionDetails}\NormalTok{(}\AttributeTok{dbms =} \StringTok{"postgresql"}\NormalTok{, }
+  \AttributeTok{server =} \StringTok{"localhost/ohdsi"}\NormalTok{, }
+  \AttributeTok{user =} \StringTok{"joe"}\NormalTok{, }
+  \AttributeTok{password =} \StringTok{"supersecret"}\NormalTok{)}
   
-\NormalTok{  cdmDatabaseSchema <-}\StringTok{ "my_cdm_data"}
-\NormalTok{  cohortsDatabaseSchema <-}\StringTok{ "my_results"}
-\NormalTok{  cdmVersion <-}\StringTok{ "5"}
+\NormalTok{  cdmDatabaseSchema }\OtherTok{\textless{}{-}} \StringTok{"my\_cdm\_data"}
+\NormalTok{  cohortsDatabaseSchema }\OtherTok{\textless{}{-}} \StringTok{"my\_results"}
+\NormalTok{  cdmVersion }\OtherTok{\textless{}{-}} \StringTok{"5"}
 \end{Highlighting}
 \end{Shaded}
 
@@ -943,17 +837,17 @@ \subsubsection{Custom cohorts}\label{custom-cohorts}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-  \KeywordTok{library}\NormalTok{(SqlRender)}
-\NormalTok{  sql <-}\StringTok{ }\KeywordTok{readSql}\NormalTok{(}\StringTok{"AfStrokeCohorts.sql"}\NormalTok{)}
-\NormalTok{  sql <-}\StringTok{ }\KeywordTok{renderSql}\NormalTok{(sql,}
-  \DataTypeTok{cdmDatabaseSchema =}\NormalTok{ cdmDatabaseSchema,}
-  \DataTypeTok{cohortsDatabaseSchema =}\NormalTok{ cohortsDatabaseSchema,}
-  \DataTypeTok{post_time =} \DecValTok{30}\NormalTok{,}
-  \DataTypeTok{pre_time =} \DecValTok{365}\NormalTok{)}\OperatorTok{$}\NormalTok{sql}
-\NormalTok{  sql <-}\StringTok{ }\KeywordTok{translateSql}\NormalTok{(sql, }\DataTypeTok{targetDialect =}\NormalTok{ connectionDetails}\OperatorTok{$}\NormalTok{dbms)}\OperatorTok{$}\NormalTok{sql}
+  \FunctionTok{library}\NormalTok{(SqlRender)}
+\NormalTok{  sql }\OtherTok{\textless{}{-}} \FunctionTok{readSql}\NormalTok{(}\StringTok{"AfStrokeCohorts.sql"}\NormalTok{)}
+\NormalTok{  sql }\OtherTok{\textless{}{-}} \FunctionTok{renderSql}\NormalTok{(sql,}
+  \AttributeTok{cdmDatabaseSchema =}\NormalTok{ cdmDatabaseSchema,}
+  \AttributeTok{cohortsDatabaseSchema =}\NormalTok{ cohortsDatabaseSchema,}
+  \AttributeTok{post\_time =} \DecValTok{30}\NormalTok{,}
+  \AttributeTok{pre\_time =} \DecValTok{365}\NormalTok{)}\SpecialCharTok{$}\NormalTok{sql}
+\NormalTok{  sql }\OtherTok{\textless{}{-}} \FunctionTok{translateSql}\NormalTok{(sql, }\AttributeTok{targetDialect =}\NormalTok{ connectionDetails}\SpecialCharTok{$}\NormalTok{dbms)}\SpecialCharTok{$}\NormalTok{sql}
   
-\NormalTok{  connection <-}\StringTok{ }\KeywordTok{connect}\NormalTok{(connectionDetails)}
-  \KeywordTok{executeSql}\NormalTok{(connection, sql)}
+\NormalTok{  connection }\OtherTok{\textless{}{-}} \FunctionTok{connect}\NormalTok{(connectionDetails)}
+  \FunctionTok{executeSql}\NormalTok{(connection, sql)}
 \end{Highlighting}
 \end{Shaded}
 
@@ -968,13 +862,13 @@ \subsubsection{Custom cohorts}\label{custom-cohorts}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{  sql <-}\StringTok{ }\KeywordTok{paste}\NormalTok{(}\StringTok{"SELECT cohort_definition_id, COUNT(*) AS count"}\NormalTok{,}
+\NormalTok{  sql }\OtherTok{\textless{}{-}} \FunctionTok{paste}\NormalTok{(}\StringTok{"SELECT cohort\_definition\_id, COUNT(*) AS count"}\NormalTok{,}
   \StringTok{"FROM @cohortsDatabaseSchema.AFibStrokeCohort"}\NormalTok{,}
-  \StringTok{"GROUP BY cohort_definition_id"}\NormalTok{)}
-\NormalTok{  sql <-}\StringTok{ }\KeywordTok{renderSql}\NormalTok{(sql, }\DataTypeTok{cohortsDatabaseSchema =}\NormalTok{ cohortsDatabaseSchema)}\OperatorTok{$}\NormalTok{sql}
-\NormalTok{  sql <-}\StringTok{ }\KeywordTok{translateSql}\NormalTok{(sql, }\DataTypeTok{targetDialect =}\NormalTok{ connectionDetails}\OperatorTok{$}\NormalTok{dbms)}\OperatorTok{$}\NormalTok{sql}
+  \StringTok{"GROUP BY cohort\_definition\_id"}\NormalTok{)}
+\NormalTok{  sql }\OtherTok{\textless{}{-}} \FunctionTok{renderSql}\NormalTok{(sql, }\AttributeTok{cohortsDatabaseSchema =}\NormalTok{ cohortsDatabaseSchema)}\SpecialCharTok{$}\NormalTok{sql}
+\NormalTok{  sql }\OtherTok{\textless{}{-}} \FunctionTok{translateSql}\NormalTok{(sql, }\AttributeTok{targetDialect =}\NormalTok{ connectionDetails}\SpecialCharTok{$}\NormalTok{dbms)}\SpecialCharTok{$}\NormalTok{sql}
   
-  \KeywordTok{querySql}\NormalTok{(connection, sql)}
+  \FunctionTok{querySql}\NormalTok{(connection, sql)}
 \end{Highlighting}
 \end{Shaded}
 
@@ -997,25 +891,25 @@ \subsubsection{Data extraction}\label{data-extraction}}
 
 Now we can tell \texttt{PatientLevelPrediction} to extract all necessary
 data for our analysis. This is done using the
-\href{https://github.com/OHDSI/FeatureExtration}{\texttt{FeatureExtractionPackage}}.
+\href{https://github.com/OHDSI/FeatureExtraction}{\texttt{FeatureExtractionPackage}}.
 In short the FeatureExtractionPackage allows you to specify which
 features (covariates) need to be extracted, e.g.~all conditions and drug
 exposures. It also supports the creation of custom covariates. For more
 detailed information on the FeatureExtraction package see its
-\href{https://github.com/OHDSI/FeatureExtration}{vignettes}. For our
+\href{https://github.com/OHDSI/FeatureExtraction}{vignettes}. For our
 example study we decided to use these settings:
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{  covariateSettings <-}\StringTok{ }\KeywordTok{createCovariateSettings}\NormalTok{(}\DataTypeTok{useDemographicsGender =} \OtherTok{TRUE}\NormalTok{,}
-  \DataTypeTok{useDemographicsAge =} \OtherTok{TRUE}\NormalTok{,}
-  \DataTypeTok{useConditionGroupEraLongTerm =} \OtherTok{TRUE}\NormalTok{,}
-  \DataTypeTok{useConditionGroupEraAnyTimePrior =} \OtherTok{TRUE}\NormalTok{,}
-  \DataTypeTok{useDrugGroupEraLongTerm =} \OtherTok{TRUE}\NormalTok{,}
-  \DataTypeTok{useDrugGroupEraAnyTimePrior =} \OtherTok{TRUE}\NormalTok{,}
-  \DataTypeTok{useVisitConceptCountLongTerm =} \OtherTok{TRUE}\NormalTok{,}
-  \DataTypeTok{longTermStartDays =} \DecValTok{-365}\NormalTok{,}
-  \DataTypeTok{endDays =} \DecValTok{-1}\NormalTok{)}
+\NormalTok{  covariateSettings }\OtherTok{\textless{}{-}} \FunctionTok{createCovariateSettings}\NormalTok{(}\AttributeTok{useDemographicsGender =} \ConstantTok{TRUE}\NormalTok{,}
+  \AttributeTok{useDemographicsAge =} \ConstantTok{TRUE}\NormalTok{,}
+  \AttributeTok{useConditionGroupEraLongTerm =} \ConstantTok{TRUE}\NormalTok{,}
+  \AttributeTok{useConditionGroupEraAnyTimePrior =} \ConstantTok{TRUE}\NormalTok{,}
+  \AttributeTok{useDrugGroupEraLongTerm =} \ConstantTok{TRUE}\NormalTok{,}
+  \AttributeTok{useDrugGroupEraAnyTimePrior =} \ConstantTok{TRUE}\NormalTok{,}
+  \AttributeTok{useVisitConceptCountLongTerm =} \ConstantTok{TRUE}\NormalTok{,}
+  \AttributeTok{longTermStartDays =} \SpecialCharTok{{-}}\DecValTok{365}\NormalTok{,}
+  \AttributeTok{endDays =} \SpecialCharTok{{-}}\DecValTok{1}\NormalTok{)}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1029,37 +923,50 @@ \subsubsection{Data extraction}\label{data-extraction}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{  plpData <-}\StringTok{ }\KeywordTok{getPlpData}\NormalTok{(}\DataTypeTok{connectionDetails =}\NormalTok{ connectionDetails,}
-  \DataTypeTok{cdmDatabaseSchema =}\NormalTok{ cdmDatabaseSchema,}
-  \DataTypeTok{cohortDatabaseSchema =}\NormalTok{ resultsDatabaseSchema,}
-  \DataTypeTok{cohortTable =} \StringTok{'AFibStrokeCohort'}\NormalTok{,}
-  \DataTypeTok{cohortId =} \DecValTok{1}\NormalTok{,}
-  \DataTypeTok{covariateSettings =}\NormalTok{ covariateSettings,}
-  \DataTypeTok{outcomeDatabaseSchema =}\NormalTok{ resultsDatabaseSchema,}
-  \DataTypeTok{outcomeTable =} \StringTok{'AFibStrokeCohort'}\NormalTok{,}
-  \DataTypeTok{outcomeIds =} \DecValTok{2}\NormalTok{,}
-  \DataTypeTok{sampleSize =} \DecValTok{10000}
+\NormalTok{databaseDetails }\OtherTok{\textless{}{-}} \FunctionTok{createDatabaseDetails}\NormalTok{(}
+  \AttributeTok{connectionDetails =}\NormalTok{ connectionDetails,}
+  \AttributeTok{cdmDatabaseSchema =}\NormalTok{ cdmDatabaseSchema,}
+  \AttributeTok{cdmDatabaseName =} \StringTok{\textquotesingle{}\textquotesingle{}}\NormalTok{,}
+  \AttributeTok{cohortDatabaseSchema =}\NormalTok{ resultsDatabaseSchema,}
+  \AttributeTok{cohortTable =} \StringTok{\textquotesingle{}AFibStrokeCohort\textquotesingle{}}\NormalTok{,}
+  \AttributeTok{cohortId =} \DecValTok{1}\NormalTok{,}
+  \AttributeTok{outcomeDatabaseSchema =}\NormalTok{ resultsDatabaseSchema,}
+  \AttributeTok{outcomeTable =} \StringTok{\textquotesingle{}AFibStrokeCohort\textquotesingle{}}\NormalTok{,}
+  \AttributeTok{outcomeIds =} \DecValTok{2}\NormalTok{,}
+  \AttributeTok{cdmVersion =} \DecValTok{5}
+\NormalTok{  )}
+
+\CommentTok{\# here you can define whether you want to sample the target cohort and add any}
+\CommentTok{\# restrictions based on minimum prior observation, index date restrictions}
+\CommentTok{\# or restricting to first index date (if people can be in target cohort multiple times)}
+\NormalTok{restrictPlpDataSettings }\OtherTok{\textless{}{-}} \FunctionTok{createRestrictPlpDataSettings}\NormalTok{(}\AttributeTok{sampleSize =} \DecValTok{10000}\NormalTok{)}
+
+\NormalTok{  plpData }\OtherTok{\textless{}{-}} \FunctionTok{getPlpData}\NormalTok{(}
+    \AttributeTok{databaseDetails =}\NormalTok{ databaseDetails, }
+    \AttributeTok{covariateSettings =}\NormalTok{ covariateSettings,}
+    \AttributeTok{restrictPlpDataSettings =}\NormalTok{ restrictPlpDataSettings}
 \NormalTok{  )}
 \end{Highlighting}
 \end{Shaded}
 
 Note that if the cohorts are created in ATLAS its corresponding cohort
 database schema needs to be selected. There are many additional
-parameters for the \texttt{getPlpData} function which are all documented
-in the \texttt{PatientLevelPrediction} manual. The resulting
-\texttt{plpData} object uses the package \texttt{ff} to store
+parameters for the \texttt{createRestrictPlpDataSettings} function which
+are all documented in the \texttt{PatientLevelPrediction} manual. The
+resulting \texttt{plpData} object uses the package \texttt{Andromeda}
+(which uses \href{https://www.sqlite.org/index.html}{SQLite}) to store
 information in a way that ensures R does not run out of memory, even
 when the data are large.
 
 Creating the \texttt{plpData} object can take considerable computing
 time, and it is probably a good idea to save it for future sessions.
-Because \texttt{plpData} uses \texttt{ff}, we cannot use R's regular
-save function. Instead, we'll have to use the \texttt{savePlpData()}
-function:
+Because \texttt{plpData} uses \texttt{Andromeda}, we cannot use R's
+regular save function. Instead, we'll have to use the
+\texttt{savePlpData()} function:
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\KeywordTok{savePlpData}\NormalTok{(plpData, }\StringTok{"stroke_in_af_data"}\NormalTok{)}
+  \FunctionTok{savePlpData}\NormalTok{(plpData, }\StringTok{"stroke\_in\_af\_data"}\NormalTok{)}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1092,24 +999,138 @@ \subsubsection{Additional inclusion
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{  population <-}\StringTok{ }\KeywordTok{createStudyPopulation}\NormalTok{(}\DataTypeTok{plpData =}\NormalTok{ plpData,}
-  \DataTypeTok{outcomeId =} \DecValTok{2}\NormalTok{,}
-  \DataTypeTok{washoutPeriod =} \DecValTok{1095}\NormalTok{,}
-  \DataTypeTok{firstExposureOnly =} \OtherTok{FALSE}\NormalTok{,}
-  \DataTypeTok{removeSubjectsWithPriorOutcome =} \OtherTok{FALSE}\NormalTok{,}
-  \DataTypeTok{priorOutcomeLookback =} \DecValTok{1}\NormalTok{,}
-  \DataTypeTok{riskWindowStart =} \DecValTok{1}\NormalTok{,}
-  \DataTypeTok{riskWindowEnd =} \DecValTok{365}\NormalTok{,}
-  \DataTypeTok{addExposureDaysToStart =} \OtherTok{FALSE}\NormalTok{,}
-  \DataTypeTok{addExposureDaysToEnd =} \OtherTok{FALSE}\NormalTok{,}
-  \DataTypeTok{minTimeAtRisk =} \DecValTok{364}\NormalTok{,}
-  \DataTypeTok{requireTimeAtRisk =} \OtherTok{TRUE}\NormalTok{,}
-  \DataTypeTok{includeAllOutcomes =} \OtherTok{TRUE}\NormalTok{,}
-  \DataTypeTok{verbosity =} \StringTok{"DEBUG"}
+\NormalTok{  populationSettings }\OtherTok{\textless{}{-}} \FunctionTok{createStudyPopulationSettings}\NormalTok{(}
+  \AttributeTok{washoutPeriod =} \DecValTok{1095}\NormalTok{,}
+  \AttributeTok{firstExposureOnly =} \ConstantTok{FALSE}\NormalTok{,}
+  \AttributeTok{removeSubjectsWithPriorOutcome =} \ConstantTok{FALSE}\NormalTok{,}
+  \AttributeTok{priorOutcomeLookback =} \DecValTok{1}\NormalTok{,}
+  \AttributeTok{riskWindowStart =} \DecValTok{1}\NormalTok{,}
+  \AttributeTok{riskWindowEnd =} \DecValTok{365}\NormalTok{,}
+  \AttributeTok{startAnchor =}  \StringTok{\textquotesingle{}cohort start\textquotesingle{}}\NormalTok{,}
+  \AttributeTok{endAnchor =}  \StringTok{\textquotesingle{}cohort start\textquotesingle{}}\NormalTok{,}
+  \AttributeTok{minTimeAtRisk =} \DecValTok{364}\NormalTok{,}
+  \AttributeTok{requireTimeAtRisk =} \ConstantTok{TRUE}\NormalTok{,}
+  \AttributeTok{includeAllOutcomes =} \ConstantTok{TRUE}
 \NormalTok{  )}
 \end{Highlighting}
 \end{Shaded}
 
+\hypertarget{spliting-the-data-into-trainingvalidationtesting-datasets}{%
+\subsubsection{Spliting the data into training/validation/testing
+datasets}\label{spliting-the-data-into-trainingvalidationtesting-datasets}}
+
+When developing a prediction model using supervised learning (when you
+have features paired with labels for a set of patients), the first step
+is to design the development/internal validation process. This requires
+specifying how to select the model hyper-parameters, how to learn the
+model parameters and how to fairly evaluate the model. In general, the
+validation set is used to pick hyper-parameters, the training set is
+used to learn the model parameters and the test set is used to perform
+fair internal validation. However, cross-validation can be implemented
+to pick the hyper-parameters on the training data (so a validation data
+set is not required). Cross validation can also be used to estimate
+internal validation (so a testing data set is not required).
+
+In small data the best approach for internal validation has been shown
+to be boostrapping. However, in big data (many patients and many
+features) bootstrapping is generally not feasible. In big data our
+research has shown that it is just important to have some form of fair
+evaluation (use a test set or cross validation). For full details see
+\href{add\%20link}{our BMJ open paper}.
+
+In the PatientLevelPrediction package, the splitSettings define how the
+plpData are partitioned into training/validation/testing data. Cross
+validation is always done, but using a test set is optional (when the
+data are small, it may be optimal to not use a test set). For the
+splitSettings we can use the type (stratified/time/subject) and
+testFraction parameters to split the data in a 75\%-25\% split and run
+the patient-level prediction pipeline:
+
+\begin{Shaded}
+\begin{Highlighting}[]
+\NormalTok{  splitSettings }\OtherTok{\textless{}{-}} \FunctionTok{createDefaultSplitSetting}\NormalTok{(}
+    \AttributeTok{trainFraction =} \FloatTok{0.75}\NormalTok{,}
+    \AttributeTok{testFraction =} \FloatTok{0.25}\NormalTok{,}
+    \AttributeTok{type =} \StringTok{\textquotesingle{}stratified\textquotesingle{}}\NormalTok{,}
+    \AttributeTok{nfold =} \DecValTok{2}\NormalTok{, }
+    \AttributeTok{splitSeed =} \DecValTok{1234}
+\NormalTok{    )}
+\end{Highlighting}
+\end{Shaded}
+
+Note: it is possible to add a custom method to specify how the plpData
+are partitioned into training/validation/testing data, see
+\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSplitting.pdf}{vignette
+for custom splitting}.
+
+\hypertarget{preprocessing-the-training-data}{%
+\subsubsection{Preprocessing the training
+data}\label{preprocessing-the-training-data}}
+
+There a numerous data processing settings that a user must specify when
+developing a prediction model. These are: * Whether to under-sample or
+over-sample the training data (this may be useful when there is class
+imballance (e.g., the outcome is very rare or very common)) * Whether to
+perform feature engineering or feature selection (e.g., create latent
+variables that are not observed in the data or reduce the dimensionality
+of the data) * Whether to remove redundant features and normalize the
+data (this is required for some models)
+
+The default sample settings does nothing, it simply returns the
+trainData as input, see below:
+
+\begin{Shaded}
+\begin{Highlighting}[]
+\NormalTok{  sampleSettings }\OtherTok{\textless{}{-}} \FunctionTok{createSampleSettings}\NormalTok{()}
+\end{Highlighting}
+\end{Shaded}
+
+However, the current package contains methods of under-sampling the
+non-outcome patients. To perform undersampling, the \texttt{type} input
+should be `underSample' and \texttt{numberOutcomestoNonOutcomes} must be
+specified (an integer specifying the number of non-outcomes per
+outcome). It is possible to add any custom function for over/under
+sampling, see
+\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSamples.pdf}{vignette
+for custom sampling}.
+
+It is possible to specify a combination of feature engineering functions
+that take as input the trainData and output a new trainData with
+different features. The default feature engineering setting does
+nothing:
+
+\begin{Shaded}
+\begin{Highlighting}[]
+\NormalTok{  featureEngineeringSettings }\OtherTok{\textless{}{-}} \FunctionTok{createFeatureEngineeringSettings}\NormalTok{()}
+\end{Highlighting}
+\end{Shaded}
+
+However, it is possible to add custom feature engineering functions into
+the pipeline, see
+\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomFeatureEngineering.pdf}{vignette
+for custom feature engineering}.
+
+Finally, the preprocessing setting is required. For this setting the
+user can define \texttt{minFraction}, this removes any features that is
+observed in the training data for less than 0.01 fraction of the
+patients. So, if \texttt{minFraction\ =\ 0.01} then any feature that is
+seen in less than 1 percent of the target population is removed. The
+input \texttt{normalize} specifies whether the features are scaled
+between 0 and 1, this is required for certain models (e.g., LASSO
+logistic regression). The input \texttt{removeRedundancy} specifies
+whether features that are observed in all of the target population are
+removed.
+
+\begin{Shaded}
+\begin{Highlighting}[]
+\NormalTok{  preprocessSettingsSettings }\OtherTok{\textless{}{-}} \FunctionTok{createPreprocessSettings}\NormalTok{(}
+    \AttributeTok{minFraction =} \FloatTok{0.01}\NormalTok{, }
+    \AttributeTok{normalize =}\NormalTok{ T, }
+    \AttributeTok{removeRedundancy =}\NormalTok{ T}
+\NormalTok{      )}
+\end{Highlighting}
+\end{Shaded}
+
 \hypertarget{model-development}{%
 \subsubsection{Model Development}\label{model-development}}
 
@@ -1131,19 +1152,41 @@ \subsubsection{Model Development}\label{model-development}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{lrModel <-}\StringTok{ }\KeywordTok{setLassoLogisticRegression}\NormalTok{()}
+\NormalTok{lrModel }\OtherTok{\textless{}{-}} \FunctionTok{setLassoLogisticRegression}\NormalTok{()}
 \end{Highlighting}
 \end{Shaded}
 
-The \texttt{runPlP} function uses the population, \texttt{plpData}, and
-model settings to train and evaluate the model. We can use the testSplit
-(person/time) and testFraction parameters to split the data in a
-75\%-25\% split and run the patient-level prediction pipeline:
+The \texttt{runPlP} function requires the \texttt{plpData}, the
+\texttt{outcomeId} specifying the outcome being predicted and the
+settings: \texttt{populationSettings}, \texttt{splitSettings},
+\texttt{sampleSettings}, \texttt{featureEngineeringSettings},
+\texttt{preprocessSettings} and \texttt{modelSettings} to train and
+evaluate the model.
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{  lrResults <-}\StringTok{ }\KeywordTok{runPlp}\NormalTok{(population, plpData, }\DataTypeTok{modelSettings =}\NormalTok{ lrModel, }\DataTypeTok{testSplit=}\StringTok{'stratified'}\NormalTok{, }
-  \DataTypeTok{testFraction=}\FloatTok{0.25}\NormalTok{, }\DataTypeTok{nfold=}\DecValTok{2}\NormalTok{, }\DataTypeTok{splitSeed =} \DecValTok{1234}\NormalTok{)}
+\NormalTok{  lrResults }\OtherTok{\textless{}{-}} \FunctionTok{runPlp}\NormalTok{(}
+    \AttributeTok{plpData =}\NormalTok{ plpData,}
+    \AttributeTok{outcomeId =} \DecValTok{2}\NormalTok{, }
+    \AttributeTok{analysisId =} \StringTok{\textquotesingle{}singleDemo\textquotesingle{}}\NormalTok{,}
+    \AttributeTok{analysisName =} \StringTok{\textquotesingle{}Demonstration of runPlp for training single PLP models\textquotesingle{}}\NormalTok{,}
+    \AttributeTok{populationSettings =}\NormalTok{ populationSettings, }
+    \AttributeTok{splitSettings =}\NormalTok{ splitSettings,}
+    \AttributeTok{sampleSettings =}\NormalTok{ sampleSettings, }
+    \AttributeTok{featureEngineeringSettings =}\NormalTok{ featureEngineeringSettings, }
+    \AttributeTok{preprocessSettings =}\NormalTok{ preprocessSettings,}
+    \AttributeTok{modelSettings =}\NormalTok{ lrModel,}
+    \AttributeTok{logSettings =} \FunctionTok{createLogSettings}\NormalTok{(), }
+    \AttributeTok{executeSettings =} \FunctionTok{createExecuteSettings}\NormalTok{(}
+      \AttributeTok{runSplitData =}\NormalTok{ T, }
+      \AttributeTok{runSampleData =}\NormalTok{ T, }
+      \AttributeTok{runfeatureEngineering =}\NormalTok{ T, }
+      \AttributeTok{runPreprocessData =}\NormalTok{ T, }
+      \AttributeTok{runModelDevelopment =}\NormalTok{ T, }
+      \AttributeTok{runCovariateSummary =}\NormalTok{ T}
+\NormalTok{    ), }
+    \AttributeTok{saveDirectory =} \FunctionTok{file.path}\NormalTok{(}\FunctionTok{getwd}\NormalTok{(), }\StringTok{\textquotesingle{}singlePlp\textquotesingle{}}\NormalTok{)}
+\NormalTok{    )}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1153,15 +1196,11 @@ \subsubsection{Model Development}\label{model-development}}
 evaluate the model on the remaining 25\%. A results data structure is
 returned containing information about the model, its performance etc.
 
-In the runPlp function there are several parameters to save the plpData,
-plpResults, plpPlots, evaluation etc. which are all set to True by
-default. However, there is also some functionality to this manually.
-
 You can save the model using:
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\KeywordTok{savePlpModel}\NormalTok{(lrResults}\OperatorTok{$}\NormalTok{model, }\DataTypeTok{dirPath =} \KeywordTok{file.path}\NormalTok{(}\KeywordTok{getwd}\NormalTok{(), }\StringTok{"model"}\NormalTok{))}
+\FunctionTok{savePlpModel}\NormalTok{(lrResults}\SpecialCharTok{$}\NormalTok{model, }\AttributeTok{dirPath =} \FunctionTok{file.path}\NormalTok{(}\FunctionTok{getwd}\NormalTok{(), }\StringTok{"model"}\NormalTok{))}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1169,7 +1208,7 @@ \subsubsection{Model Development}\label{model-development}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{plpModel <-}\StringTok{ }\KeywordTok{loadPlpModel}\NormalTok{(}\KeywordTok{file.path}\NormalTok{(}\KeywordTok{getwd}\NormalTok{(), }\StringTok{"model"}\NormalTok{))}
+\NormalTok{  plpModel }\OtherTok{\textless{}{-}} \FunctionTok{loadPlpModel}\NormalTok{(}\FunctionTok{file.path}\NormalTok{(}\FunctionTok{getwd}\NormalTok{(),}\StringTok{\textquotesingle{}model\textquotesingle{}}\NormalTok{))}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1177,7 +1216,7 @@ \subsubsection{Model Development}\label{model-development}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\KeywordTok{savePlpResult}\NormalTok{(lrResults, }\DataTypeTok{location =} \KeywordTok{file.path}\NormalTok{(}\KeywordTok{getwd}\NormalTok{(), }\StringTok{"lr"}\NormalTok{))}
+  \FunctionTok{savePlpResult}\NormalTok{(lrResults, }\AttributeTok{location =} \FunctionTok{file.path}\NormalTok{(}\FunctionTok{getwd}\NormalTok{(),}\StringTok{\textquotesingle{}lr\textquotesingle{}}\NormalTok{))}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1185,7 +1224,7 @@ \subsubsection{Model Development}\label{model-development}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{lrResults <-}\StringTok{ }\KeywordTok{loadPlpResult}\NormalTok{(}\KeywordTok{file.path}\NormalTok{(}\KeywordTok{getwd}\NormalTok{(), }\StringTok{"lr"}\NormalTok{))}
+\NormalTok{  lrResults }\OtherTok{\textless{}{-}} \FunctionTok{loadPlpResult}\NormalTok{(}\FunctionTok{file.path}\NormalTok{(}\FunctionTok{getwd}\NormalTok{(),}\StringTok{\textquotesingle{}lr\textquotesingle{}}\NormalTok{))}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1197,110 +1236,41 @@ \section{Example 2: Angioedema in ACE inhibitor users}\label{example2}}
 \hypertarget{study-specification-2}{%
 \subsection{Study Specification}\label{study-specification-2}}
 
-\begin{longtable}[]{@{}ll@{}}
-\toprule
-\begin{minipage}[b]{0.42\columnwidth}\raggedright
-Definition\strut
-\end{minipage} & \begin{minipage}[b]{0.52\columnwidth}\raggedright
-Value\strut
-\end{minipage}\tabularnewline
-\midrule
+\begin{longtable}[]{@{}
+  >{\raggedright\arraybackslash}p{(\columnwidth - 2\tabcolsep) * \real{0.3056}}
+  >{\raggedright\arraybackslash}p{(\columnwidth - 2\tabcolsep) * \real{0.6944}}@{}}
+\toprule\noalign{}
+\begin{minipage}[b]{\linewidth}\raggedright
+Definition
+\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright
+Value
+\end{minipage} \\
+\midrule\noalign{}
 \endhead
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-\textbf{Problem Definition}\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Target Cohort (T)\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-`Patients who are newly dispensed an ACE inhibitor' defined as the first
-drug record of any ACE inhibitor\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Outcome Cohort (O)\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-`Angioedema' defined as an angioedema condition record during an
-inpatient or ER visit\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Time-at-risk (TAR)\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-1 day till 365 days from cohort start\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-\textbf{Population Definition}\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Washout Period\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-365\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Enter the target cohort multiple times?\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-No\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Allow prior outcomes?\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-No\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Start of time-at-risk\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-1 day\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-End of time-at-risk\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-365 days\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Require a minimum amount of time-at-risk?\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-Yes (364 days)\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-\textbf{Model Development}\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Algorithm\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-Gradient Boosting Machine\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Hyper-parameters\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-ntree:5000, max depth:4 or 7 or 10 and learning rate: 0.001 or 0.01 or
-0.1 or 0.9\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Covariates\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-Gender, Age, Conditions (ever before, \textless365), Drugs Groups (ever
-before, \textless365), and Visit Count\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.42\columnwidth}\raggedright
-Data split\strut
-\end{minipage} & \begin{minipage}[t]{0.52\columnwidth}\raggedright
-75\% train, 25\% test. Randomly assigned by person\strut
-\end{minipage}\tabularnewline
-\bottomrule
+\bottomrule\noalign{}
+\endlastfoot
+\textbf{Problem Definition} & \\
+Target Cohort (T) & `Patients who are newly dispensed an ACE inhibitor'
+defined as the first drug record of any ACE inhibitor \\
+Outcome Cohort (O) & `Angioedema' defined as an angioedema condition
+record during an inpatient or ER visit \\
+Time-at-risk (TAR) & 1 day till 365 days from cohort start \\
+& \\
+\textbf{Population Definition} & \\
+Washout Period & 365 \\
+Enter the target cohort multiple times? & No \\
+Allow prior outcomes? & No \\
+Start of time-at-risk & 1 day \\
+End of time-at-risk & 365 days \\
+Require a minimum amount of time-at-risk? & Yes (364 days) \\
+& \\
+\textbf{Model Development} & \\
+Algorithm & Gradient Boosting Machine \\
+Hyper-parameters & ntree:5000, max depth:4 or 7 or 10 and learning rate:
+0.001 or 0.01 or 0.1 or 0.9 \\
+Covariates & Gender, Age, Conditions (ever before, \textless365), Drugs
+Groups (ever before, \textless365), and Visit Count \\
+Data split & 75\% train, 25\% test. Randomly assigned by person \\
 \end{longtable}
 
 According to the best practices we need to make a protocol that
@@ -1364,7 +1334,7 @@ \subsubsection{ATLAS cohort builder}\label{atlas-cohort-builder-1}}
 
 \begin{figure}
 \centering
-\includegraphics{example2/aceinhibitors.png}
+\includegraphics{example2/aceinhibitors.webp}
 \caption{Target Cohort ACE inhibitors}
 \end{figure}
 
@@ -1381,7 +1351,7 @@ \subsubsection{ATLAS cohort builder}\label{atlas-cohort-builder-1}}
 
 \begin{figure}
 \centering
-\includegraphics{example2/angioedema.png}
+\includegraphics{example2/angioedema.webp}
 \caption{Outcome Cohort Angioedema}
 \end{figure}
 
@@ -1425,67 +1395,67 @@ \subsubsection{Custom cohorts}\label{custom-cohorts-1}}
 \CommentTok{    Create a table to store the persons in the T and C cohort}
 \CommentTok{  */}
     
-    \ControlFlowTok{IF}\NormalTok{ OBJECT_ID(}\StringTok{'@resultsDatabaseSchema.PLPAceAngioCohort'}\NormalTok{, }\StringTok{'U'}\NormalTok{) }\KeywordTok{IS} \KeywordTok{NOT} \KeywordTok{NULL} 
+    \ControlFlowTok{IF}\NormalTok{ OBJECT\_ID(}\StringTok{\textquotesingle{}@resultsDatabaseSchema.PLPAceAngioCohort\textquotesingle{}}\NormalTok{, }\StringTok{\textquotesingle{}U\textquotesingle{}}\NormalTok{) }\KeywordTok{IS} \KeywordTok{NOT} \KeywordTok{NULL} 
   \KeywordTok{DROP} \KeywordTok{TABLE}\NormalTok{ @resultsDatabaseSchema.PLPAceAngioCohort;}
   
   \KeywordTok{CREATE} \KeywordTok{TABLE}\NormalTok{ @resultsDatabaseSchema.PLPAceAngioCohort }
 \NormalTok{  ( }
-\NormalTok{    cohort_definition_id }\DataTypeTok{INT}\NormalTok{, }
-\NormalTok{    subject_id BIGINT,}
-\NormalTok{    cohort_start_date }\DataTypeTok{DATE}\NormalTok{, }
-\NormalTok{    cohort_end_date }\DataTypeTok{DATE}
+\NormalTok{    cohort\_definition\_id }\DataTypeTok{INT}\NormalTok{, }
+\NormalTok{    subject\_id BIGINT,}
+\NormalTok{    cohort\_start\_date }\DataTypeTok{DATE}\NormalTok{, }
+\NormalTok{    cohort\_end\_date }\DataTypeTok{DATE}
 \NormalTok{  );}
   
   
   \CommentTok{/*}
 \CommentTok{    T cohort:  [PatientLevelPrediction vignette]:  T : patients who are newly }
 \CommentTok{  dispensed an ACE inhibitor}
-\CommentTok{  - persons with a drug exposure record of any 'ACE inhibitor' or }
+\CommentTok{  {-} persons with a drug exposure record of any \textquotesingle{}ACE inhibitor\textquotesingle{} or }
 \CommentTok{  any descendants, indexed at the first diagnosis}
-\CommentTok{  - who have >364 days of prior observation before their first dispensing}
+\CommentTok{  {-} who have \textgreater{}364 days of prior observation before their first dispensing}
 \CommentTok{  */}
-    \KeywordTok{INSERT} \KeywordTok{INTO}\NormalTok{ @resultsDatabaseSchema.AceAngioCohort (cohort_definition_id, }
-\NormalTok{                                                       subject_id, }
-\NormalTok{                                                       cohort_start_date, }
-\NormalTok{                                                       cohort_end_date)}
-  \KeywordTok{SELECT} \DecValTok{1} \KeywordTok{AS}\NormalTok{ cohort_definition_id,}
-\NormalTok{  Ace.person_id }\KeywordTok{AS}\NormalTok{ subject_id,}
-\NormalTok{  Ace.drug_start_date }\KeywordTok{AS}\NormalTok{ cohort_start_date,}
-\NormalTok{  observation_period.observation_period_end_date }\KeywordTok{AS}\NormalTok{ cohort_end_date}
+    \KeywordTok{INSERT} \KeywordTok{INTO}\NormalTok{ @resultsDatabaseSchema.AceAngioCohort (cohort\_definition\_id, }
+\NormalTok{                                                       subject\_id, }
+\NormalTok{                                                       cohort\_start\_date, }
+\NormalTok{                                                       cohort\_end\_date)}
+  \KeywordTok{SELECT} \DecValTok{1} \KeywordTok{AS}\NormalTok{ cohort\_definition\_id,}
+\NormalTok{  Ace.person\_id }\KeywordTok{AS}\NormalTok{ subject\_id,}
+\NormalTok{  Ace.drug\_start\_date }\KeywordTok{AS}\NormalTok{ cohort\_start\_date,}
+\NormalTok{  observation\_period.observation\_period\_end\_date }\KeywordTok{AS}\NormalTok{ cohort\_end\_date}
   \KeywordTok{FROM}
 \NormalTok{  (}
-    \KeywordTok{SELECT}\NormalTok{ person_id, }\FunctionTok{min}\NormalTok{(drug_exposure_date) }\KeywordTok{as}\NormalTok{ drug_start_date}
-    \KeywordTok{FROM}\NormalTok{ @cdmDatabaseSchema.drug_exposure}
-    \KeywordTok{WHERE}\NormalTok{ drug_concept_id }\KeywordTok{IN}\NormalTok{ (}\KeywordTok{SELECT}\NormalTok{ descendant_concept_id }\KeywordTok{FROM} 
-\NormalTok{                              @cdmDatabaseSchema.concept_ancestor }\KeywordTok{WHERE}\NormalTok{ ancestor_concept_id }\KeywordTok{IN} 
+    \KeywordTok{SELECT}\NormalTok{ person\_id, }\FunctionTok{min}\NormalTok{(drug\_exposure\_date) }\KeywordTok{as}\NormalTok{ drug\_start\_date}
+    \KeywordTok{FROM}\NormalTok{ @cdmDatabaseSchema.drug\_exposure}
+    \KeywordTok{WHERE}\NormalTok{ drug\_concept\_id }\KeywordTok{IN}\NormalTok{ (}\KeywordTok{SELECT}\NormalTok{ descendant\_concept\_id }\KeywordTok{FROM} 
+\NormalTok{                              @cdmDatabaseSchema.concept\_ancestor }\KeywordTok{WHERE}\NormalTok{ ancestor\_concept\_id }\KeywordTok{IN} 
 \NormalTok{                              (}\DecValTok{1342439}\NormalTok{,}\DecValTok{1334456}\NormalTok{, }\DecValTok{1331235}\NormalTok{, }\DecValTok{1373225}\NormalTok{, }\DecValTok{1310756}\NormalTok{, }\DecValTok{1308216}\NormalTok{, }\DecValTok{1363749}\NormalTok{, }\DecValTok{1341927}\NormalTok{, }\DecValTok{1340128}\NormalTok{, }\DecValTok{1335471} \CommentTok{/*ace inhibitors*/}\NormalTok{))}
-    \KeywordTok{GROUP} \KeywordTok{BY}\NormalTok{ person_id}
+    \KeywordTok{GROUP} \KeywordTok{BY}\NormalTok{ person\_id}
 \NormalTok{  ) Ace}
-  \KeywordTok{INNER} \KeywordTok{JOIN}\NormalTok{ @cdmDatabaseSchema.observation_period}
-  \KeywordTok{ON}\NormalTok{ Ace.person_id }\OperatorTok{=}\NormalTok{ observation_period.person_id}
-  \KeywordTok{AND}\NormalTok{ Ace.drug_start_date }\OperatorTok{>=}\NormalTok{ dateadd(dd,}\DecValTok{364}\NormalTok{, }
-\NormalTok{                                     observation_period.observation_period_start_date)}
-  \KeywordTok{AND}\NormalTok{ Ace.drug_start_date }\OperatorTok{<=}\NormalTok{ observation_period.observation_period_end_date}
+  \KeywordTok{INNER} \KeywordTok{JOIN}\NormalTok{ @cdmDatabaseSchema.observation\_period}
+  \KeywordTok{ON}\NormalTok{ Ace.person\_id }\OperatorTok{=}\NormalTok{ observation\_period.person\_id}
+  \KeywordTok{AND}\NormalTok{ Ace.drug\_start\_date }\OperatorTok{\textgreater{}=}\NormalTok{ dateadd(dd,}\DecValTok{364}\NormalTok{, }
+\NormalTok{                                     observation\_period.observation\_period\_start\_date)}
+  \KeywordTok{AND}\NormalTok{ Ace.drug\_start\_date }\OperatorTok{\textless{}=}\NormalTok{ observation\_period.observation\_period\_end\_date}
 \NormalTok{  ;}
   
   \CommentTok{/*}
 \CommentTok{    C cohort:  [PatientLevelPrediction vignette]:  O: Angioedema}
 \CommentTok{  */}
-    \KeywordTok{INSERT} \KeywordTok{INTO}\NormalTok{ @resultsDatabaseSchema.AceAngioCohort (cohort_definition_id, }
-\NormalTok{                                                       subject_id, }
-\NormalTok{                                                       cohort_start_date, }
-\NormalTok{                                                       cohort_end_date)}
-  \KeywordTok{SELECT} \DecValTok{2} \KeywordTok{AS}\NormalTok{ cohort_definition_id,}
-\NormalTok{  angioedema.person_id }\KeywordTok{AS}\NormalTok{ subject_id,}
-\NormalTok{  angioedema.condition_start_date }\KeywordTok{AS}\NormalTok{ cohort_start_date,}
-\NormalTok{  angioedema.condition_start_date }\KeywordTok{AS}\NormalTok{ cohort_end_date}
+    \KeywordTok{INSERT} \KeywordTok{INTO}\NormalTok{ @resultsDatabaseSchema.AceAngioCohort (cohort\_definition\_id, }
+\NormalTok{                                                       subject\_id, }
+\NormalTok{                                                       cohort\_start\_date, }
+\NormalTok{                                                       cohort\_end\_date)}
+  \KeywordTok{SELECT} \DecValTok{2} \KeywordTok{AS}\NormalTok{ cohort\_definition\_id,}
+\NormalTok{  angioedema.person\_id }\KeywordTok{AS}\NormalTok{ subject\_id,}
+\NormalTok{  angioedema.condition\_start\_date }\KeywordTok{AS}\NormalTok{ cohort\_start\_date,}
+\NormalTok{  angioedema.condition\_start\_date }\KeywordTok{AS}\NormalTok{ cohort\_end\_date}
   \KeywordTok{FROM}  
 \NormalTok{  (}
-    \KeywordTok{SELECT}\NormalTok{ person_id, condition_start_date}
-    \KeywordTok{FROM}\NormalTok{ @cdmDatabaseSchema.condition_occurrence}
-    \KeywordTok{WHERE}\NormalTok{ condition_concept_id }\KeywordTok{IN}\NormalTok{ (}\KeywordTok{SELECT} \KeywordTok{DISTINCT}\NormalTok{ descendant_concept_id }\KeywordTok{FROM} 
-\NormalTok{                                   @cdmDatabaseSchema.concept_ancestor }\KeywordTok{WHERE}\NormalTok{ ancestor_concept_id }\KeywordTok{IN} 
-\NormalTok{                                   (}\DecValTok{432791} \CommentTok{/*angioedema*/}\NormalTok{) }\KeywordTok{OR}\NormalTok{ descendant_concept_id }\KeywordTok{IN} 
+    \KeywordTok{SELECT}\NormalTok{ person\_id, condition\_start\_date}
+    \KeywordTok{FROM}\NormalTok{ @cdmDatabaseSchema.condition\_occurrence}
+    \KeywordTok{WHERE}\NormalTok{ condition\_concept\_id }\KeywordTok{IN}\NormalTok{ (}\KeywordTok{SELECT} \KeywordTok{DISTINCT}\NormalTok{ descendant\_concept\_id }\KeywordTok{FROM} 
+\NormalTok{                                   @cdmDatabaseSchema.concept\_ancestor }\KeywordTok{WHERE}\NormalTok{ ancestor\_concept\_id }\KeywordTok{IN} 
+\NormalTok{                                   (}\DecValTok{432791} \CommentTok{/*angioedema*/}\NormalTok{) }\KeywordTok{OR}\NormalTok{ descendant\_concept\_id }\KeywordTok{IN} 
 \NormalTok{                                   (}\DecValTok{432791} \CommentTok{/*angioedema*/}\NormalTok{)}
 \NormalTok{    ) angioedema}
     
@@ -1514,14 +1484,14 @@ \subsubsection{Custom cohorts}\label{custom-cohorts-1}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{    connectionDetails <-}\StringTok{ }\KeywordTok{createConnectionDetails}\NormalTok{(}\DataTypeTok{dbms =} \StringTok{"postgresql"}\NormalTok{, }
-                                                 \DataTypeTok{server =} \StringTok{"localhost/ohdsi"}\NormalTok{, }
-                                                 \DataTypeTok{user =} \StringTok{"joe"}\NormalTok{, }
-                                                 \DataTypeTok{password =} \StringTok{"supersecret"}\NormalTok{)}
+\NormalTok{    connectionDetails }\OtherTok{\textless{}{-}} \FunctionTok{createConnectionDetails}\NormalTok{(}\AttributeTok{dbms =} \StringTok{"postgresql"}\NormalTok{, }
+                                                 \AttributeTok{server =} \StringTok{"localhost/ohdsi"}\NormalTok{, }
+                                                 \AttributeTok{user =} \StringTok{"joe"}\NormalTok{, }
+                                                 \AttributeTok{password =} \StringTok{"supersecret"}\NormalTok{)}
     
-\NormalTok{    cdmDatabaseSchema <-}\StringTok{ "my_cdm_data"}
-\NormalTok{    cohortsDatabaseSchema <-}\StringTok{ "my_results"}
-\NormalTok{    cdmVersion <-}\StringTok{ "5"}
+\NormalTok{    cdmDatabaseSchema }\OtherTok{\textless{}{-}} \StringTok{"my\_cdm\_data"}
+\NormalTok{    cohortsDatabaseSchema }\OtherTok{\textless{}{-}} \StringTok{"my\_results"}
+\NormalTok{    cdmVersion }\OtherTok{\textless{}{-}} \StringTok{"5"}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1535,15 +1505,15 @@ \subsubsection{Custom cohorts}\label{custom-cohorts-1}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-    \KeywordTok{library}\NormalTok{(SqlRender)}
-\NormalTok{    sql <-}\StringTok{ }\KeywordTok{readSql}\NormalTok{(}\StringTok{"AceAngioCohorts.sql"}\NormalTok{)}
-\NormalTok{    sql <-}\StringTok{ }\KeywordTok{render}\NormalTok{(sql,}
-                  \DataTypeTok{cdmDatabaseSchema =}\NormalTok{ cdmDatabaseSchema,}
-                  \DataTypeTok{cohortsDatabaseSchema =}\NormalTok{ cohortsDatabaseSchema)}
-\NormalTok{    sql <-}\StringTok{ }\KeywordTok{translate}\NormalTok{(sql, }\DataTypeTok{targetDialect =}\NormalTok{ connectionDetails}\OperatorTok{$}\NormalTok{dbms)}
+    \FunctionTok{library}\NormalTok{(SqlRender)}
+\NormalTok{    sql }\OtherTok{\textless{}{-}} \FunctionTok{readSql}\NormalTok{(}\StringTok{"AceAngioCohorts.sql"}\NormalTok{)}
+\NormalTok{    sql }\OtherTok{\textless{}{-}} \FunctionTok{render}\NormalTok{(sql,}
+                  \AttributeTok{cdmDatabaseSchema =}\NormalTok{ cdmDatabaseSchema,}
+                  \AttributeTok{cohortsDatabaseSchema =}\NormalTok{ cohortsDatabaseSchema)}
+\NormalTok{    sql }\OtherTok{\textless{}{-}} \FunctionTok{translate}\NormalTok{(sql, }\AttributeTok{targetDialect =}\NormalTok{ connectionDetails}\SpecialCharTok{$}\NormalTok{dbms)}
     
-\NormalTok{    connection <-}\StringTok{ }\KeywordTok{connect}\NormalTok{(connectionDetails)}
-    \KeywordTok{executeSql}\NormalTok{(connection, sql)}
+\NormalTok{    connection }\OtherTok{\textless{}{-}} \FunctionTok{connect}\NormalTok{(connectionDetails)}
+    \FunctionTok{executeSql}\NormalTok{(connection, sql)}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1558,13 +1528,13 @@ \subsubsection{Custom cohorts}\label{custom-cohorts-1}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{    sql <-}\StringTok{ }\KeywordTok{paste}\NormalTok{(}\StringTok{"SELECT cohort_definition_id, COUNT(*) AS count"}\NormalTok{,}
+\NormalTok{    sql }\OtherTok{\textless{}{-}} \FunctionTok{paste}\NormalTok{(}\StringTok{"SELECT cohort\_definition\_id, COUNT(*) AS count"}\NormalTok{,}
                  \StringTok{"FROM @cohortsDatabaseSchema.AceAngioCohort"}\NormalTok{,}
-                 \StringTok{"GROUP BY cohort_definition_id"}\NormalTok{)}
-\NormalTok{    sql <-}\StringTok{ }\KeywordTok{render}\NormalTok{(sql, }\DataTypeTok{cohortsDatabaseSchema =}\NormalTok{ cohortsDatabaseSchema)}
-\NormalTok{    sql <-}\StringTok{ }\KeywordTok{translate}\NormalTok{(sql, }\DataTypeTok{targetDialect =}\NormalTok{ connectionDetails}\OperatorTok{$}\NormalTok{dbms)}
+                 \StringTok{"GROUP BY cohort\_definition\_id"}\NormalTok{)}
+\NormalTok{    sql }\OtherTok{\textless{}{-}} \FunctionTok{render}\NormalTok{(sql, }\AttributeTok{cohortsDatabaseSchema =}\NormalTok{ cohortsDatabaseSchema)}
+\NormalTok{    sql }\OtherTok{\textless{}{-}} \FunctionTok{translate}\NormalTok{(sql, }\AttributeTok{targetDialect =}\NormalTok{ connectionDetails}\SpecialCharTok{$}\NormalTok{dbms)}
     
-    \KeywordTok{querySql}\NormalTok{(connection, sql)}
+    \FunctionTok{querySql}\NormalTok{(connection, sql)}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1587,25 +1557,25 @@ \subsubsection{Data extraction}\label{data-extraction-1}}
 
 Now we can tell \texttt{PatientLevelPrediction} to extract all necessary
 data for our analysis. This is done using the
-\href{https://github.com/OHDSI/FeatureExtration}{\texttt{FeatureExtractionPackage}}.
+\href{https://github.com/OHDSI/FeatureExtraction}{\texttt{FeatureExtractionPackage}}.
 In short the FeatureExtractionPackage allows you to specify which
 features (covariates) need to be extracted, e.g.~all conditions and drug
 exposures. It also supports the creation of custom covariates. For more
 detailed information on the FeatureExtraction package see its
-\href{https://github.com/OHDSI/FeatureExtration}{vignettes}. For our
+\href{https://github.com/OHDSI/FeatureExtraction}{vignettes}. For our
 example study we decided to use these settings:
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{    covariateSettings <-}\StringTok{ }\KeywordTok{createCovariateSettings}\NormalTok{(}\DataTypeTok{useDemographicsGender =} \OtherTok{TRUE}\NormalTok{,}
-                                                 \DataTypeTok{useDemographicsAge =} \OtherTok{TRUE}\NormalTok{,}
-                                                 \DataTypeTok{useConditionGroupEraLongTerm =} \OtherTok{TRUE}\NormalTok{,}
-                                                 \DataTypeTok{useConditionGroupEraAnyTimePrior =} \OtherTok{TRUE}\NormalTok{,}
-                                                 \DataTypeTok{useDrugGroupEraLongTerm =} \OtherTok{TRUE}\NormalTok{,}
-                                                 \DataTypeTok{useDrugGroupEraAnyTimePrior =} \OtherTok{TRUE}\NormalTok{,}
-                                                 \DataTypeTok{useVisitConceptCountLongTerm =} \OtherTok{TRUE}\NormalTok{,}
-                                                 \DataTypeTok{longTermStartDays =} \DecValTok{-365}\NormalTok{,}
-                                                 \DataTypeTok{endDays =} \DecValTok{-1}\NormalTok{)}
+\NormalTok{    covariateSettings }\OtherTok{\textless{}{-}} \FunctionTok{createCovariateSettings}\NormalTok{(}\AttributeTok{useDemographicsGender =} \ConstantTok{TRUE}\NormalTok{,}
+                                                 \AttributeTok{useDemographicsAge =} \ConstantTok{TRUE}\NormalTok{,}
+                                                 \AttributeTok{useConditionGroupEraLongTerm =} \ConstantTok{TRUE}\NormalTok{,}
+                                                 \AttributeTok{useConditionGroupEraAnyTimePrior =} \ConstantTok{TRUE}\NormalTok{,}
+                                                 \AttributeTok{useDrugGroupEraLongTerm =} \ConstantTok{TRUE}\NormalTok{,}
+                                                 \AttributeTok{useDrugGroupEraAnyTimePrior =} \ConstantTok{TRUE}\NormalTok{,}
+                                                 \AttributeTok{useVisitConceptCountLongTerm =} \ConstantTok{TRUE}\NormalTok{,}
+                                                 \AttributeTok{longTermStartDays =} \SpecialCharTok{{-}}\DecValTok{365}\NormalTok{,}
+                                                 \AttributeTok{endDays =} \SpecialCharTok{{-}}\DecValTok{1}\NormalTok{)}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1619,17 +1589,26 @@ \subsubsection{Data extraction}\label{data-extraction-1}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{    plpData <-}\StringTok{ }\KeywordTok{getPlpData}\NormalTok{(}\DataTypeTok{connectionDetails =}\NormalTok{ connectionDetails,}
-                          \DataTypeTok{cdmDatabaseSchema =}\NormalTok{ cdmDatabaseSchema,}
-                          \DataTypeTok{cohortDatabaseSchema =}\NormalTok{ resultsDatabaseSchema,}
-                          \DataTypeTok{cohortTable =} \StringTok{'AceAngioCohort'}\NormalTok{,}
-                          \DataTypeTok{cohortId =} \DecValTok{1}\NormalTok{,}
-                          \DataTypeTok{covariateSettings =}\NormalTok{ covariateSettings,}
-                          \DataTypeTok{outcomeDatabaseSchema =}\NormalTok{ resultsDatabaseSchema,}
-                          \DataTypeTok{outcomeTable =} \StringTok{'AceAngioCohort'}\NormalTok{,}
-                          \DataTypeTok{outcomeIds =} \DecValTok{2}\NormalTok{,}
-                          \DataTypeTok{sampleSize =} \DecValTok{10000}
-\NormalTok{    )}
+\NormalTok{databaseDetails }\OtherTok{\textless{}{-}} \FunctionTok{createDatabaseDetails}\NormalTok{(}
+  \AttributeTok{connectionDetails =}\NormalTok{ connectionDetails,}
+  \AttributeTok{cdmDatabaseSchema =}\NormalTok{ cdmDatabaseSchema,}
+  \AttributeTok{cohortDatabaseSchema =}\NormalTok{ resultsDatabaseSchema,}
+  \AttributeTok{cohortTable =} \StringTok{\textquotesingle{}AceAngioCohort\textquotesingle{}}\NormalTok{,}
+  \AttributeTok{cohortId =} \DecValTok{1}\NormalTok{,}
+  \AttributeTok{outcomeDatabaseSchema =}\NormalTok{ resultsDatabaseSchema,}
+  \AttributeTok{outcomeTable =} \StringTok{\textquotesingle{}AceAngioCohort\textquotesingle{}}\NormalTok{,}
+  \AttributeTok{outcomeIds =} \DecValTok{2}
+\NormalTok{  )}
+
+\NormalTok{restrictPlpDataSettings }\OtherTok{\textless{}{-}} \FunctionTok{createRestrictPlpDataSettings}\NormalTok{(}
+  \AttributeTok{sampleSize =} \DecValTok{10000}
+\NormalTok{  )}
+
+\NormalTok{plpData }\OtherTok{\textless{}{-}} \FunctionTok{getPlpData}\NormalTok{(}
+  \AttributeTok{databaseDetails =}\NormalTok{ databaseDetails, }
+  \AttributeTok{covariateSettings =}\NormalTok{ covariateSettings, }
+  \AttributeTok{restrictPlpDataSettings =}\NormalTok{ restrictPlpDataSettings}
+\NormalTok{  )}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1649,7 +1628,7 @@ \subsubsection{Data extraction}\label{data-extraction-1}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\KeywordTok{savePlpData}\NormalTok{(plpData, }\StringTok{"angio_in_ace_data"}\NormalTok{)}
+    \FunctionTok{savePlpData}\NormalTok{(plpData, }\StringTok{"angio\_in\_ace\_data"}\NormalTok{)}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1682,24 +1661,138 @@ \subsubsection{Additional inclusion
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{    population <-}\StringTok{ }\KeywordTok{createStudyPopulation}\NormalTok{(}\DataTypeTok{plpData =}\NormalTok{ plpData,}
-                                        \DataTypeTok{outcomeId =} \DecValTok{2}\NormalTok{,}
-                                        \DataTypeTok{washoutPeriod =} \DecValTok{364}\NormalTok{,}
-                                        \DataTypeTok{firstExposureOnly =} \OtherTok{FALSE}\NormalTok{,}
-                                        \DataTypeTok{removeSubjectsWithPriorOutcome =} \OtherTok{TRUE}\NormalTok{,}
-                                        \DataTypeTok{priorOutcomeLookback =} \DecValTok{9999}\NormalTok{,}
-                                        \DataTypeTok{riskWindowStart =} \DecValTok{1}\NormalTok{,}
-                                        \DataTypeTok{riskWindowEnd =} \DecValTok{365}\NormalTok{,}
-                                        \DataTypeTok{addExposureDaysToStart =} \OtherTok{FALSE}\NormalTok{,}
-                                        \DataTypeTok{addExposureDaysToEnd =} \OtherTok{FALSE}\NormalTok{,}
-                                        \DataTypeTok{minTimeAtRisk =} \DecValTok{364}\NormalTok{,}
-                                        \DataTypeTok{requireTimeAtRisk =} \OtherTok{TRUE}\NormalTok{,}
-                                        \DataTypeTok{includeAllOutcomes =} \OtherTok{TRUE}\NormalTok{,}
-                                        \DataTypeTok{verbosity =} \StringTok{"DEBUG"}
+\NormalTok{    populationSettings }\OtherTok{\textless{}{-}} \FunctionTok{createStudyPopulationSettings}\NormalTok{(}
+      \AttributeTok{washoutPeriod =} \DecValTok{364}\NormalTok{,}
+      \AttributeTok{firstExposureOnly =} \ConstantTok{FALSE}\NormalTok{,}
+      \AttributeTok{removeSubjectsWithPriorOutcome =} \ConstantTok{TRUE}\NormalTok{,}
+      \AttributeTok{priorOutcomeLookback =} \DecValTok{9999}\NormalTok{,}
+      \AttributeTok{riskWindowStart =} \DecValTok{1}\NormalTok{,}
+      \AttributeTok{riskWindowEnd =} \DecValTok{365}\NormalTok{, }
+      \AttributeTok{minTimeAtRisk =} \DecValTok{364}\NormalTok{,}
+      \AttributeTok{startAnchor =} \StringTok{\textquotesingle{}cohort start\textquotesingle{}}\NormalTok{,}
+      \AttributeTok{endAnchor =} \StringTok{\textquotesingle{}cohort start\textquotesingle{}}\NormalTok{,}
+      \AttributeTok{requireTimeAtRisk =} \ConstantTok{TRUE}\NormalTok{,}
+      \AttributeTok{includeAllOutcomes =} \ConstantTok{TRUE}
 \NormalTok{    )}
 \end{Highlighting}
 \end{Shaded}
 
+\hypertarget{spliting-the-data-into-trainingvalidationtesting-datasets-1}{%
+\subsubsection{Spliting the data into training/validation/testing
+datasets}\label{spliting-the-data-into-trainingvalidationtesting-datasets-1}}
+
+When developing a prediction model using supervised learning (when you
+have features paired with labels for a set of patients), the first step
+is to design the development/internal validation process. This requires
+specifying how to select the model hyper-parameters, how to learn the
+model parameters and how to fairly evaluate the model. In general, the
+validation set is used to pick hyper-parameters, the training set is
+used to learn the model parameters and the test set is used to perform
+fair internal validation. However, cross-validation can be implemented
+to pick the hyper-parameters on the training data (so a validation data
+set is not required). Cross validation can also be used to estimate
+internal validation (so a testing data set is not required).
+
+In small data the best approach for internal validation has been shown
+to be boostrapping. However, in big data (many patients and many
+features) bootstrapping is generally not feasible. In big data our
+research has shown that it is just important to have some form of fair
+evaluation (use a test set or cross validation). For full details see
+\href{add\%20link}{our BMJ open paper}.
+
+In the PatientLevelPrediction package, the splitSettings define how the
+plpData are partitioned into training/validation/testing data. Cross
+validation is always done, but using a test set is optional (when the
+data are small, it may be optimal to not use a test set). For the
+splitSettings we can use the type (stratified/time/subject) and
+testFraction parameters to split the data in a 75\%-25\% split and run
+the patient-level prediction pipeline:
+
+\begin{Shaded}
+\begin{Highlighting}[]
+\NormalTok{  splitSettings }\OtherTok{\textless{}{-}} \FunctionTok{createDefaultSplitSetting}\NormalTok{(}
+    \AttributeTok{trainFraction =} \FloatTok{0.75}\NormalTok{,}
+    \AttributeTok{testFraction =} \FloatTok{0.25}\NormalTok{,}
+    \AttributeTok{type =} \StringTok{\textquotesingle{}stratified\textquotesingle{}}\NormalTok{,}
+    \AttributeTok{nfold =} \DecValTok{2}\NormalTok{, }
+    \AttributeTok{splitSeed =} \DecValTok{1234}
+\NormalTok{    )}
+\end{Highlighting}
+\end{Shaded}
+
+Note: it is possible to add a custom method to specify how the plpData
+are partitioned into training/validation/testing data, see
+\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSplitting.pdf}{vignette
+for custom splitting}.
+
+\hypertarget{preprocessing-the-training-data-1}{%
+\subsubsection{Preprocessing the training
+data}\label{preprocessing-the-training-data-1}}
+
+There a numerous data processing settings that a user must specify when
+developing a prediction model. These are: * Whether to under-sample or
+over-sample the training data (this may be useful when there is class
+imballance (e.g., the outcome is very rare or very common)) * Whether to
+perform feature engineering or feature selection (e.g., create latent
+variables that are not observed in the data or reduce the dimensionality
+of the data) * Whether to remove redundant features and normalize the
+data (this is required for some models)
+
+The default sample settings does nothing, it simply returns the
+trainData as input, see below:
+
+\begin{Shaded}
+\begin{Highlighting}[]
+\NormalTok{  sampleSettings }\OtherTok{\textless{}{-}} \FunctionTok{createSampleSettings}\NormalTok{()}
+\end{Highlighting}
+\end{Shaded}
+
+However, the current package contains methods of under-sampling the
+non-outcome patients. To perform undersampling, the \texttt{type} input
+should be `underSample' and \texttt{numberOutcomestoNonOutcomes} must be
+specified (an integer specifying the number of non-outcomes per
+outcome). It is possible to add any custom function for over/under
+sampling, see
+\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSamples.pdf}{vignette
+for custom sampling}.
+
+It is possible to specify a combination of feature engineering functions
+that take as input the trainData and output a new trainData with
+different features. The default feature engineering setting does
+nothing:
+
+\begin{Shaded}
+\begin{Highlighting}[]
+\NormalTok{  featureEngineeringSettings }\OtherTok{\textless{}{-}} \FunctionTok{createFeatureEngineeringSettings}\NormalTok{()}
+\end{Highlighting}
+\end{Shaded}
+
+However, it is possible to add custom feature engineering functions into
+the pipeline, see
+\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomfeatureEngineering.pdf}{vignette
+for custom feature engineering}.
+
+Finally, the preprocessing setting is required. For this setting the
+user can define \texttt{minFraction}, this removes any features that is
+observed in the training data for less than 0.01 fraction of the
+patients. So, if \texttt{minFraction\ =\ 0.01} then any feature that is
+seen in less than 1 percent of the target population is removed. The
+input \texttt{normalize} specifies whether the features are scaled
+between 0 and 1, this is required for certain models (e.g., LASSO
+logistic regression). The input \texttt{removeRedundancy} specifies
+whether features that are observed in all of the target population are
+removed.
+
+\begin{Shaded}
+\begin{Highlighting}[]
+\NormalTok{  preprocessSettingsSettings }\OtherTok{\textless{}{-}} \FunctionTok{createPreprocessSettings}\NormalTok{(}
+    \AttributeTok{minFraction =} \FloatTok{0.01}\NormalTok{, }
+    \AttributeTok{normalize =}\NormalTok{ T, }
+    \AttributeTok{removeRedundancy =}\NormalTok{ T}
+\NormalTok{      )}
+\end{Highlighting}
+\end{Shaded}
+
 \hypertarget{model-development-1}{%
 \subsubsection{Model Development}\label{model-development-1}}
 
@@ -1721,20 +1814,45 @@ \subsubsection{Model Development}\label{model-development-1}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{gbmModel <-}\StringTok{ }\KeywordTok{setGradientBoostingMachine}\NormalTok{(}\DataTypeTok{ntrees =} \DecValTok{5000}\NormalTok{, }\DataTypeTok{maxDepth =} \KeywordTok{c}\NormalTok{(}\DecValTok{4}\NormalTok{, }\DecValTok{7}\NormalTok{, }\DecValTok{10}\NormalTok{), }\DataTypeTok{learnRate =} \KeywordTok{c}\NormalTok{(}\FloatTok{0.001}\NormalTok{, }
-    \FloatTok{0.01}\NormalTok{, }\FloatTok{0.1}\NormalTok{, }\FloatTok{0.9}\NormalTok{))}
+\NormalTok{    gbmModel }\OtherTok{\textless{}{-}} \FunctionTok{setGradientBoostingMachine}\NormalTok{(}
+      \AttributeTok{ntrees =} \DecValTok{5000}\NormalTok{, }
+      \AttributeTok{maxDepth =} \FunctionTok{c}\NormalTok{(}\DecValTok{4}\NormalTok{,}\DecValTok{7}\NormalTok{,}\DecValTok{10}\NormalTok{), }
+      \AttributeTok{learnRate =} \FunctionTok{c}\NormalTok{(}\FloatTok{0.001}\NormalTok{,}\FloatTok{0.01}\NormalTok{,}\FloatTok{0.1}\NormalTok{,}\FloatTok{0.9}\NormalTok{)}
+\NormalTok{      )}
 \end{Highlighting}
 \end{Shaded}
 
-The \texttt{runPlP} function uses the population, \texttt{plpData}, and
-model settings to train and evaluate the model. We can use the testSplit
-(person/time) and testFraction parameters to split the data in a
-75\%-25\% split and run the patient-level prediction pipeline:
+The \texttt{runPlP} function requires the \texttt{plpData}, the
+\texttt{outcomeId} specifying the outcome being predicted and the
+settings: \texttt{populationSettings}, \texttt{splitSettings},
+\texttt{sampleSettings}, \texttt{featureEngineeringSettings},
+\texttt{preprocessSettings} and \texttt{modelSettings} to train and
+evaluate the model.
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{    gbmResults <-}\StringTok{ }\KeywordTok{runPlp}\NormalTok{(population, plpData, }\DataTypeTok{modelSettings =}\NormalTok{ gbmModel, }\DataTypeTok{testSplit=}\StringTok{'stratified'}\NormalTok{, }
-                         \DataTypeTok{testFraction=}\FloatTok{0.25}\NormalTok{, }\DataTypeTok{nfold=}\DecValTok{2}\NormalTok{, }\DataTypeTok{splitSeed =} \DecValTok{1234}\NormalTok{)}
+\NormalTok{  gbmResults }\OtherTok{\textless{}{-}} \FunctionTok{runPlp}\NormalTok{(}
+    \AttributeTok{plpData =}\NormalTok{ plpData,}
+    \AttributeTok{outcomeId =} \DecValTok{2}\NormalTok{, }
+    \AttributeTok{analysisId =} \StringTok{\textquotesingle{}singleDemo2\textquotesingle{}}\NormalTok{,}
+    \AttributeTok{analysisName =} \StringTok{\textquotesingle{}Demonstration of runPlp for training single PLP models\textquotesingle{}}\NormalTok{,}
+    \AttributeTok{populationSettings =}\NormalTok{ populationSettings, }
+    \AttributeTok{splitSettings =}\NormalTok{ splitSettings,}
+    \AttributeTok{sampleSettings =}\NormalTok{ sampleSettings, }
+    \AttributeTok{featureEngineeringSettings =}\NormalTok{ featureEngineeringSettings, }
+    \AttributeTok{preprocessSettings =}\NormalTok{ preprocessSettings,}
+    \AttributeTok{modelSettings =}\NormalTok{ gbmModel,}
+    \AttributeTok{logSettings =} \FunctionTok{createLogSettings}\NormalTok{(), }
+    \AttributeTok{executeSettings =} \FunctionTok{createExecuteSettings}\NormalTok{(}
+      \AttributeTok{runSplitData =}\NormalTok{ T, }
+      \AttributeTok{runSampleData =}\NormalTok{ T, }
+      \AttributeTok{runfeatureEngineering =}\NormalTok{ T, }
+      \AttributeTok{runPreprocessData =}\NormalTok{ T, }
+      \AttributeTok{runModelDevelopment =}\NormalTok{ T, }
+      \AttributeTok{runCovariateSummary =}\NormalTok{ T}
+\NormalTok{    ), }
+    \AttributeTok{saveDirectory =} \FunctionTok{file.path}\NormalTok{(}\FunctionTok{getwd}\NormalTok{(), }\StringTok{\textquotesingle{}singlePlpExample2\textquotesingle{}}\NormalTok{)}
+\NormalTok{    )}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1743,15 +1861,11 @@ \subsubsection{Model Development}\label{model-development-1}}
 the model on the remaining 25\%. A results data structure is returned
 containing information about the model, its performance etc.
 
-In the runPlp function there are several parameters to save the plpData,
-plpResults, plpPlots, evaluation etc. which are all set to True by
-default. However, there is also some functionality to this manually.
-
 You can save the model using:
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\KeywordTok{savePlpModel}\NormalTok{(gbmResults}\OperatorTok{$}\NormalTok{model, }\DataTypeTok{dirPath =} \KeywordTok{file.path}\NormalTok{(}\KeywordTok{getwd}\NormalTok{(), }\StringTok{"model"}\NormalTok{))}
+    \FunctionTok{savePlpModel}\NormalTok{(gbmResults}\SpecialCharTok{$}\NormalTok{model, }\AttributeTok{dirPath =} \FunctionTok{file.path}\NormalTok{(}\FunctionTok{getwd}\NormalTok{(), }\StringTok{"model"}\NormalTok{))}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1759,7 +1873,7 @@ \subsubsection{Model Development}\label{model-development-1}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{plpModel <-}\StringTok{ }\KeywordTok{loadPlpModel}\NormalTok{(}\KeywordTok{file.path}\NormalTok{(}\KeywordTok{getwd}\NormalTok{(), }\StringTok{"model"}\NormalTok{))}
+\NormalTok{    plpModel }\OtherTok{\textless{}{-}} \FunctionTok{loadPlpModel}\NormalTok{(}\FunctionTok{file.path}\NormalTok{(}\FunctionTok{getwd}\NormalTok{(),}\StringTok{\textquotesingle{}model\textquotesingle{}}\NormalTok{))}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1767,7 +1881,7 @@ \subsubsection{Model Development}\label{model-development-1}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\KeywordTok{savePlpResult}\NormalTok{(gbmResults, }\DataTypeTok{location =} \KeywordTok{file.path}\NormalTok{(}\KeywordTok{getwd}\NormalTok{(), }\StringTok{"gbm"}\NormalTok{))}
+    \FunctionTok{savePlpResult}\NormalTok{(gbmResults, }\AttributeTok{location =} \FunctionTok{file.path}\NormalTok{(}\FunctionTok{getwd}\NormalTok{(),}\StringTok{\textquotesingle{}gbm\textquotesingle{}}\NormalTok{))}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1775,7 +1889,7 @@ \subsubsection{Model Development}\label{model-development-1}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{gbmResults <-}\StringTok{ }\KeywordTok{loadPlpResult}\NormalTok{(}\KeywordTok{file.path}\NormalTok{(}\KeywordTok{getwd}\NormalTok{(), }\StringTok{"gbm"}\NormalTok{))}
+\NormalTok{    gbmResults }\OtherTok{\textless{}{-}} \FunctionTok{loadPlpResult}\NormalTok{(}\FunctionTok{file.path}\NormalTok{(}\FunctionTok{getwd}\NormalTok{(),}\StringTok{\textquotesingle{}gbm\textquotesingle{}}\NormalTok{))}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1799,24 +1913,24 @@ \section{Study package creation}\label{study-package-creation}}
   Create a new prediction study and select your target and outcome
   cohorts.
 
-  \includegraphics{atlasplp1.png}
+  \includegraphics{atlasplp1.webp}
 \item
   Specify one or more analysis settings.
 
-  \includegraphics{atlasplp2.png}
+  \includegraphics{atlasplp2.web}
 
   \newpage
 \item
   Specify the trainings settigns
 
-  \includegraphics{atlasplp3.png}
+  \includegraphics{atlasplp3.webp}
 \item
   Specify the execution settings
 
-  \includegraphics{atlasplp4.png}
-\end{enumerate}
+  \includegraphics{atlasplp4.web}
 
-\newpage
+  {]} \newpage
+\end{enumerate}
 
 ATLAS can build a R package for you that will execute the full study
 against you CDM. Below the steps are explained how to do this in ATLAS.
@@ -1829,7 +1943,7 @@ \section{Study package creation}\label{study-package-creation}}
 
   \begin{figure}
   \centering
-  \includegraphics{atlasdownload1.png}
+  \includegraphics{atlasdownload1.webp}
   \caption{R package download functionality in ATLAS}
   \end{figure}
 \item
@@ -1838,7 +1952,7 @@ \section{Study package creation}\label{study-package-creation}}
 
   \begin{figure}
   \centering
-  \includegraphics{atlasdownload2.png}
+  \includegraphics{atlasdownload2.webp}
   \caption{R package download functionality in ATLAS}
   \end{figure}
 \item
@@ -1862,20 +1976,28 @@ \section{Internal validation}\label{internal-validation}}
 your browser in which you can view all performance measures created by
 the framework as shown in the figure below.
 
-\includegraphics{shinysummary.png}
+\begin{figure}
+\centering
+\includegraphics{shinysummary.webp}
+\caption{Summary of all the performance measures of the analyses}
+\end{figure}
 
 Furthermore, many interactive plots are available in the Shiny App, for
 example the ROC curve in which you can move over the plot to see the
 threshold and the corresponding sensitivity and specificity values.
 
-\includegraphics{shinyroc.png}
+\begin{figure}
+\centering
+\includegraphics{shinyroc.webp}
+\caption{Example of the interactive ROC curve}
+\end{figure}
 
 To generate and save all the evaluation plots to a folder run the
 following code:
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\KeywordTok{plotPlp}\NormalTok{(lrResults, }\DataTypeTok{dirPath =} \KeywordTok{getwd}\NormalTok{())}
+\FunctionTok{plotPlp}\NormalTok{(lrResults, }\AttributeTok{dirPath=}\FunctionTok{getwd}\NormalTok{())}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1894,9 +2016,15 @@ \subsection{Discrimination}\label{discrimination}}
 plot the better the discrimination of the model. The plot is created by
 changing the probability threshold to assign the positive class.
 
-\includegraphics{sparseRoc.png}
+\begin{figure}
+\centering
+\includegraphics{sparseRoc.webp}
+\caption{Receiver Operating Characteristic Plot}
+\end{figure}
+
+\newpage
 
-\newpage \#\# Calibration
+\#\# Calibration
 
 The calibration plot shows how close the predicted risk is to the
 observed risk. The diagonal dashed line thus indicates a perfectly
@@ -1908,7 +2036,11 @@ \subsection{Discrimination}\label{discrimination}}
 represented the 95\% lower and upper confidence intervals of the slope
 of the fitted line.
 
-\includegraphics{sparseCalibration.png}
+\begin{figure}
+\centering
+\includegraphics{sparseCalibration.webp}
+\caption{Calibration Plot}
+\end{figure}
 
 \newpage
 
@@ -1932,7 +2064,7 @@ \subsection{Smooth Calibration}\label{smooth-calibration}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\KeywordTok{plotSmoothCalibration}\NormalTok{(lrResults)}
+\FunctionTok{plotSmoothCalibration}\NormalTok{(lrResults)}
 \end{Highlighting}
 \end{Shaded}
 
@@ -1944,17 +2076,29 @@ \subsection{Smooth Calibration}\label{smooth-calibration}}
 not highlight the miss-calibration at the lower predicted probability
 levels that well.
 
+\begin{figure}
+\centering
 \includegraphics{smoothCalibration.jpeg}
+\caption{Smooth Calibration plot}
+\end{figure}
+
+\newpage
 
-\newpage \#\# Preference distribution
+\#\# Preference distribution
 
 The preference distribution plots are the preference score distributions
 corresponding to i) people in the test set with the outcome (red) and
 ii) people in the test set without the outcome (blue).
 
-\includegraphics{preferencePDF.png}
+\begin{figure}
+\centering
+\includegraphics{preferencePDF.webp}
+\caption{Preference Plot}
+\end{figure}
 
-\newpage \#\# Predicted probability distribution
+\newpage
+
+\#\# Predicted probability distribution
 
 The prediction distribution box plots are for the predicted risks of the
 people in the test set with the outcome (class 1: blue) and without the
@@ -1965,9 +2109,15 @@ \subsection{Smooth Calibration}\label{smooth-calibration}}
 overlap between the two distribution which lead to an imperfect
 discrimination.
 
-\includegraphics{predictionDistribution.png}
+\begin{figure}
+\centering
+\includegraphics{predictionDistribution.wwebp}
+\caption{Prediction Distribution Box Plot}
+\end{figure}
+
+\newpage
 
-\newpage \#\# Test-Train similarity
+\#\# Test-Train similarity
 
 The test-train similarity is assessed by plotting the mean covariate
 values in the train set against those in the test set for people with
@@ -1976,9 +2126,15 @@ \subsection{Smooth Calibration}\label{smooth-calibration}}
 The results for our example of look very promising since the mean values
 of the covariates are on the diagonal.
 
-\includegraphics{generalizability.png}
+\begin{figure}
+\centering
+\includegraphics{generalizability.webp}
+\caption{Similarity plots of train and test set}
+\end{figure}
 
-\newpage \#\# Variable scatter plot
+\newpage
+
+\#\# Variable scatter plot
 
 The variable scatter plot shows the mean covariate value for the people
 with the outcome against the mean covariate value for the people without
@@ -1990,16 +2146,22 @@ \subsection{Smooth Calibration}\label{smooth-calibration}}
 The plot shows that the mean of most of the covariates is higher for
 subjects with the outcome compared to those without.
 
-\includegraphics{variableScatterplot.png}
+\begin{figure}
+\centering
+\includegraphics{variableScatterplot.webp}
+\caption{Variabel scatter Plot}
+\end{figure}
+
+\newpage
 
-\newpage \#\# Precision recall
+\#\# Precision recall
 
 Precision (P) is defined as the number of true positives (Tp) over the
 number of true positives plus the number of false positives (Fp).
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{P <-}\StringTok{ }\NormalTok{Tp}\OperatorTok{/}\NormalTok{(Tp }\OperatorTok{+}\StringTok{ }\NormalTok{Fp)}
+\NormalTok{P }\OtherTok{\textless{}{-}}\NormalTok{ Tp}\SpecialCharTok{/}\NormalTok{(Tp}\SpecialCharTok{+}\NormalTok{Fp)}
 \end{Highlighting}
 \end{Shaded}
 
@@ -2008,7 +2170,7 @@ \subsection{Smooth Calibration}\label{smooth-calibration}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{R <-}\StringTok{ }\NormalTok{Tp}\OperatorTok{/}\NormalTok{(Tp }\OperatorTok{+}\StringTok{ }\NormalTok{Fn)}
+\NormalTok{R }\OtherTok{\textless{}{-}}\NormalTok{ Tp}\SpecialCharTok{/}\NormalTok{(Tp }\SpecialCharTok{+}\NormalTok{ Fn)}
 \end{Highlighting}
 \end{Shaded}
 
@@ -2017,7 +2179,7 @@ \subsection{Smooth Calibration}\label{smooth-calibration}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\NormalTok{F1 <-}\StringTok{ }\DecValTok{2} \OperatorTok{*}\StringTok{ }\NormalTok{P }\OperatorTok{*}\StringTok{ }\NormalTok{R}\OperatorTok{/}\NormalTok{(P }\OperatorTok{+}\StringTok{ }\NormalTok{R)}
+\NormalTok{F1 }\OtherTok{\textless{}{-}} \DecValTok{2}\SpecialCharTok{*}\NormalTok{P}\SpecialCharTok{*}\NormalTok{R}\SpecialCharTok{/}\NormalTok{(P}\SpecialCharTok{+}\NormalTok{R)}
 \end{Highlighting}
 \end{Shaded}
 
@@ -2035,9 +2197,15 @@ \subsection{Smooth Calibration}\label{smooth-calibration}}
 It is also possible that lowering the threshold may leave recall
 unchanged, while the precision fluctuates.
 
-\includegraphics{precisionRecall.png}
+\begin{figure}
+\centering
+\includegraphics{precisionRecall.webp}
+\caption{Precision Recall Plot}
+\end{figure}
+
+\newpage
 
-\newpage \#\# Demographic summary
+\#\# Demographic summary
 
 This plot shows for females and males the expected and observed risk in
 different age groups together with a confidence area.
@@ -2045,60 +2213,36 @@ \subsection{Smooth Calibration}\label{smooth-calibration}}
 The results show that our model is well calibrated across gender and age
 groups.
 
-\includegraphics{demographicSummary.png}
+\begin{figure}
+\centering
+\includegraphics{demographicSummary.webp}
+\caption{Demographic Summary Plot}
+\end{figure}
 
-\newpage \# External validation
+\newpage
+
+\# External validation
 
 We recommend to always perform external validation, i.e.~apply the final
 model on as much new datasets as feasible and evaluate its performance.
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\CommentTok{# load the trained model}
-\NormalTok{plpModel <-}\StringTok{ }\KeywordTok{loadPlpModel}\NormalTok{(}\KeywordTok{getwd}\NormalTok{(),}\StringTok{'model'}\NormalTok{)}
-
-\CommentTok{#load the new plpData and create the population}
-\NormalTok{plpData <-}\StringTok{ }\KeywordTok{loadPlpData}\NormalTok{(}\KeywordTok{getwd}\NormalTok{(),}\StringTok{'data'}\NormalTok{)}
-\NormalTok{population <-}\StringTok{ }\KeywordTok{createStudyPopulation}\NormalTok{(plpData, }
-                                    \DataTypeTok{outcomeId =} \DecValTok{2}\NormalTok{, }
-                                    \DataTypeTok{includeAllOutcomes =} \OtherTok{TRUE}\NormalTok{, }
-                                    \DataTypeTok{firstExposureOnly =} \OtherTok{TRUE}\NormalTok{, }
-                                    \DataTypeTok{washoutPeriod =} \DecValTok{365}\NormalTok{, }
-                                    \DataTypeTok{removeSubjectsWithPriorOutcome =} \OtherTok{TRUE}\NormalTok{, }
-                                    \DataTypeTok{priorOutcomeLookback =} \DecValTok{365}\NormalTok{,}
-                                    \DataTypeTok{riskWindowStart =} \DecValTok{1}\NormalTok{,}
-                                    \DataTypeTok{requireTimeAtRisk =} \OtherTok{FALSE}\NormalTok{,}
-                                    \DataTypeTok{riskWindowEnd =} \DecValTok{365}
-\NormalTok{)}
-
-\CommentTok{# apply the trained model on the new data}
-\NormalTok{validationResults <-}\StringTok{ }\KeywordTok{applyModel}\NormalTok{(population,plpData,plpModel)}
-\end{Highlighting}
-\end{Shaded}
-
-To make things easier we also provide a function for performing external
-validation of a model across one or multiple datasets:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\CommentTok{# load the trained model}
-\NormalTok{plpResult <-}\StringTok{ }\KeywordTok{loadPlpResult}\NormalTok{(}\KeywordTok{getwd}\NormalTok{(),}\StringTok{'plpResult'}\NormalTok{)}
-
-\NormalTok{connectionDetails <-}\StringTok{ }\KeywordTok{createConnectionDetails}\NormalTok{(}\DataTypeTok{dbms =} \StringTok{"postgresql"}\NormalTok{, }
-                                             \DataTypeTok{server =} \StringTok{"localhost/ohdsi"}\NormalTok{, }
-                                             \DataTypeTok{user =} \StringTok{"joe"}\NormalTok{, }
-                                             \DataTypeTok{password =} \StringTok{"supersecret"}\NormalTok{)}
-
-\NormalTok{validation <-}\StringTok{ }\KeywordTok{externalValidatePlp}\NormalTok{(}\DataTypeTok{plpResult =}\NormalTok{ plpResult, }
-                                  \DataTypeTok{connectionDetails =}\NormalTok{ connectionDetails,}
-                                  \DataTypeTok{validationSchemaTarget =} \StringTok{'new_cohort_schema'}\NormalTok{,}
-                                  \DataTypeTok{validationSchemaOutcome =} \StringTok{'new_cohort_schema'}\NormalTok{,}
-                                  \DataTypeTok{validationSchemaCdm =} \StringTok{'new_cdm_schema'}\NormalTok{, }
-                                  \DataTypeTok{validationTableTarget =} \StringTok{'cohort_table'}\NormalTok{,}
-                                  \DataTypeTok{validationTableOutcome =} \StringTok{'cohort_table'}\NormalTok{, }
-                                  \DataTypeTok{validationIdTarget =} \StringTok{'cohort_id'}\NormalTok{, }
-                                  \DataTypeTok{validationIdOutcome =} \StringTok{'outcome_id'}\NormalTok{, }
-                                  \DataTypeTok{keepPrediction =}\NormalTok{ T}
+\CommentTok{\# load the trained model}
+\NormalTok{plpModel }\OtherTok{\textless{}{-}} \FunctionTok{loadPlpModel}\NormalTok{(}\FunctionTok{getwd}\NormalTok{(),}\StringTok{\textquotesingle{}model\textquotesingle{}}\NormalTok{)}
+
+\CommentTok{\# add details of new database}
+\NormalTok{validationDatabaseDetails }\OtherTok{\textless{}{-}} \FunctionTok{createDatabaseDetails}\NormalTok{()}
+
+\CommentTok{\# to externally validate the model and perform recalibration run:}
+\FunctionTok{externalValidateDbPlp}\NormalTok{(}
+  \AttributeTok{plpModel =}\NormalTok{ plpModel,}
+  \AttributeTok{validationDatabaseDetails =}\NormalTok{ validationDatabaseDetails,}
+  \AttributeTok{validationRestrictPlpDataSettings =}\NormalTok{ plpModel}\SpecialCharTok{$}\NormalTok{settings}\SpecialCharTok{$}\NormalTok{plpDataSettings,}
+  \AttributeTok{settings =} \FunctionTok{createValidationSettings}\NormalTok{(}
+    \AttributeTok{recalibrate =} \StringTok{\textquotesingle{}weakRecalibration\textquotesingle{}}
+\NormalTok{    ),}
+  \AttributeTok{outputFolder =} \FunctionTok{getwd}\NormalTok{()}
 \NormalTok{)}
 \end{Highlighting}
 \end{Shaded}
@@ -2106,75 +2250,12 @@ \subsection{Smooth Calibration}\label{smooth-calibration}}
 This will extract the new plpData from the specified schemas and cohort
 tables. It will then apply the same population settings and the trained
 plp model. Finally, it will evaluate the performance and return the
-standard output as \texttt{validation\$performance} and if
-keepPrediction is TRUE then it will also return the prediction on the
-population as \texttt{validation\$prediction}. They can be inserted into
-the shiny app for viewing the model and validation by running:
+standard output as \texttt{validation\$performanceEvaluation} and it
+will also return the prediction on the population as
+\texttt{validation\$prediction}. They can be inserted into the shiny app
+for viewing the model and validation by running:
 \texttt{viewPlp(runPlp=plpResult,\ validatePlp=validation\ )}.
 
-If you want to validate on multiple databases available you can insert
-the new schemas and cohort tables as a list:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\CommentTok{# load the trained model}
-\NormalTok{plpResult <-}\StringTok{ }\KeywordTok{loadPlpResult}\NormalTok{(}\KeywordTok{getwd}\NormalTok{(),}\StringTok{'plpResult'}\NormalTok{)}
-
-\NormalTok{connectionDetails <-}\StringTok{ }\KeywordTok{createConnectionDetails}\NormalTok{(}\DataTypeTok{dbms =} \StringTok{"postgresql"}\NormalTok{, }
-                                             \DataTypeTok{server =} \StringTok{"localhost/ohdsi"}\NormalTok{, }
-                                             \DataTypeTok{user =} \StringTok{"joe"}\NormalTok{, }
-                                             \DataTypeTok{password =} \StringTok{"supersecret"}\NormalTok{)}
-
-\NormalTok{validation <-}\StringTok{ }\KeywordTok{externalValidatePlp}\NormalTok{(}\DataTypeTok{plpResult =}\NormalTok{ plpResult, }
-                                  \DataTypeTok{connectionDetails =}\NormalTok{ connectionDetails,}
-                                  \DataTypeTok{validationSchemaTarget =} \KeywordTok{list}\NormalTok{(}\StringTok{'new_cohort_schema1'}\NormalTok{,}
-                                                                \StringTok{'new_cohort_schema2'}\NormalTok{),}
-                                  \DataTypeTok{validationSchemaOutcome =} \KeywordTok{list}\NormalTok{(}\StringTok{'new_cohort_schema1'}\NormalTok{,}
-                                                                 \StringTok{'new_cohort_schema2'}\NormalTok{),}
-                                  \DataTypeTok{validationSchemaCdm =} \KeywordTok{list}\NormalTok{(}\StringTok{'new_cdm_schema1'}\NormalTok{,}
-                                                             \StringTok{'new_cdm_schema2'}\NormalTok{), }
-                                  \DataTypeTok{validationTableTarget =} \KeywordTok{list}\NormalTok{(}\StringTok{'new_cohort_table1'}\NormalTok{,}
-                                                               \StringTok{'new_cohort_table2'}\NormalTok{),}
-                                  \DataTypeTok{validationTableOutcome =} \KeywordTok{list}\NormalTok{(}\StringTok{'new_cohort_table1'}\NormalTok{,}
-                                                                \StringTok{'new_cohort_table2'}\NormalTok{),}
-                                  \DataTypeTok{validationIdTarget =} \StringTok{'cohort_id'}\NormalTok{, }
-                                  \DataTypeTok{validationIdOutcome =} \StringTok{'outcome_id'}\NormalTok{, }
-                                  \DataTypeTok{keepPrediction =}\NormalTok{ T}
-\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{journal-paper-generation}{%
-\section{Journal paper generation}\label{journal-paper-generation}}
-
-We have added functionality to automatically generate a word document
-you can use as start of a journal paper. It contains many of the
-generated study details and results. If you have performed external
-validation these results will can be added as well. Optionally, you can
-add a ``Table 1'' that contains data on many covariates for the target
-population.
-
-You can create the draft journal paper by running this function:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{createPlpJournalDocument}\NormalTok{(}\DataTypeTok{plpResult =} \OperatorTok{<}\NormalTok{your plp results}\OperatorTok{>}\NormalTok{, }
-                         \DataTypeTok{plpValidation =} \OperatorTok{<}\NormalTok{your validation results}\OperatorTok{>}\NormalTok{,}
-                         \DataTypeTok{plpData =} \OperatorTok{<}\NormalTok{your plp data}\OperatorTok{>}\NormalTok{, }
-                         \DataTypeTok{targetName =} \StringTok{"<target population>"}\NormalTok{,}
-                         \DataTypeTok{outcomeName =} \StringTok{"<outcome>"}\NormalTok{, }
-                         \DataTypeTok{table1 =}\NormalTok{ F, }
-                         \DataTypeTok{connectionDetails =} \OtherTok{NULL}\NormalTok{,}
-                         \DataTypeTok{includeTrain =} \OtherTok{FALSE}\NormalTok{,}
-                         \DataTypeTok{includeTest =} \OtherTok{TRUE}\NormalTok{,}
-                         \DataTypeTok{includePredictionPicture =} \OtherTok{TRUE}\NormalTok{,}
-                         \DataTypeTok{includeAttritionPlot =} \OtherTok{TRUE}\NormalTok{,}
-                         \DataTypeTok{outputLocation =} \StringTok{"<your location>"}\NormalTok{)}\ErrorTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-For more details see the help page of the function.
-
 \newpage
 
 \hypertarget{other-functionality}{%
@@ -2184,69 +2265,49 @@ \section{Other functionality}\label{other-functionality}}
 and contributions have been made my many persons in the OHDSI community.
 The table below provides an overview:
 
-\begin{longtable}[]{@{}lll@{}}
-\toprule
-\begin{minipage}[b]{0.18\columnwidth}\raggedright
-Functionality\strut
-\end{minipage} & \begin{minipage}[b]{0.55\columnwidth}\raggedright
-Description\strut
-\end{minipage} & \begin{minipage}[b]{0.18\columnwidth}\raggedright
-Vignette\strut
-\end{minipage}\tabularnewline
-\midrule
+\begin{longtable}[]{@{}
+  >{\raggedright\arraybackslash}p{(\columnwidth - 4\tabcolsep) * \real{0.2361}}
+  >{\raggedright\arraybackslash}p{(\columnwidth - 4\tabcolsep) * \real{0.5278}}
+  >{\raggedright\arraybackslash}p{(\columnwidth - 4\tabcolsep) * \real{0.2361}}@{}}
+\toprule\noalign{}
+\begin{minipage}[b]{\linewidth}\raggedright
+Functionality
+\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright
+Description
+\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright
+Vignette
+\end{minipage} \\
+\midrule\noalign{}
 \endhead
-\begin{minipage}[t]{0.18\columnwidth}\raggedright
-Builing Multiple Models\strut
-\end{minipage} & \begin{minipage}[t]{0.55\columnwidth}\raggedright
-This vignette describes how you can run multiple models
-automatically\strut
-\end{minipage} & \begin{minipage}[t]{0.18\columnwidth}\raggedright
-\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingMultiplePredictiveModels.pdf}{\texttt{Vignette}}\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.18\columnwidth}\raggedright
-Custom algorithms\strut
-\end{minipage} & \begin{minipage}[t]{0.55\columnwidth}\raggedright
-This vignette describes how you can add your own custom algorithms in
-the framework\strut
-\end{minipage} & \begin{minipage}[t]{0.18\columnwidth}\raggedright
-\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomAlgorithms.pdf}{\texttt{Vignette}}\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.18\columnwidth}\raggedright
-Ensemble models\strut
-\end{minipage} & \begin{minipage}[t]{0.55\columnwidth}\raggedright
-This vignette describes how you can use the framework to build ensemble
-models, i.e combine multiple models in a super learner\strut
-\end{minipage} & \begin{minipage}[t]{0.18\columnwidth}\raggedright
-\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingEnsembleModels.pdf}{\texttt{Vignette}}\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.18\columnwidth}\raggedright
-Deep Learning Models\strut
-\end{minipage} & \begin{minipage}[t]{0.55\columnwidth}\raggedright
-We have added extensive functionality for Deep Learning including
-several architectures in both pyTorch and Keras. These algorithms can be
-trained using GPU power\strut
-\end{minipage} & \begin{minipage}[t]{0.18\columnwidth}\raggedright
-\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingDeepLearningModels.pdf}{\texttt{Vignette}}\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.18\columnwidth}\raggedright
-Learning curves\strut
-\end{minipage} & \begin{minipage}[t]{0.55\columnwidth}\raggedright
-Learning curves assess the effect of training set size on model
-performance by training a sequence of prediction models on successively
-larger subsets of the training set. A learning curve plot can also help
-in diagnosing a bias or variance problem as explained below.\strut
-\end{minipage} & \begin{minipage}[t]{0.18\columnwidth}\raggedright
-\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/GeneratingLearningCurves.pdf}{\texttt{Vignette}}\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.18\columnwidth}\raggedright
-Implementing existing models\strut
-\end{minipage} & \begin{minipage}[t]{0.55\columnwidth}\raggedright
-This vignette describes how you can implement existing logistic
-regression models in the framework, e.g.~as found in literature\strut
-\end{minipage} & \begin{minipage}[t]{0.18\columnwidth}\raggedright
-\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/ImplementingExistingModels.pdf}{\texttt{Vignette}}\strut
-\end{minipage}\tabularnewline
-\bottomrule
+\bottomrule\noalign{}
+\endlastfoot
+Builing Multiple Models & This vignette describes how you can run
+multiple models automatically &
+\href{https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingMultiplePredictiveModels.pdf}{\texttt{Vignette}} \\
+Custom Models & This vignette describes how you can add your own custom
+algorithms in the framework &
+\href{https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/AddingCustomModels.pdf}{\texttt{Vignette}} \\
+Custom Splitting Functions & This vignette describes how you can add
+your own custom training/validation/testing splitting functions in the
+framework &
+\href{https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/AddingCustomSplitting.pdf}{\texttt{Vignette}} \\
+Custom Sampling Functions & This vignette describes how you can add your
+own custom sampling functions in the framework &
+\href{https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/AddingCustomSamples.pdf}{\texttt{Vignette}} \\
+Custom Feature Engineering/Selection & This vignette describes how you
+can add your own custom feature engineering and selection functions in
+the framework &
+\href{https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/AddingCustomFeatureEngineering.pdf}{\texttt{Vignette}} \\
+Ensemble models & This vignette describes how you can use the framework
+to build ensemble models, i.e combine multiple models in a super learner
+&
+\href{https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingEnsembleModels.pdf}{\texttt{Vignette}} \\
+Learning curves & Learning curves assess the effect of training set size
+on model performance by training a sequence of prediction models on
+successively larger subsets of the training set. A learning curve plot
+can also help in diagnosing a bias or variance problem as explained
+below. &
+\href{https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/GeneratingLearningCurves.pdf}{\texttt{Vignette}} \\
 \end{longtable}
 
 \hypertarget{demos}{%
@@ -2256,11 +2317,11 @@ \section{Demos}\label{demos}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\CommentTok{# Show all demos in our package: }
-\KeywordTok{demo}\NormalTok{(}\DataTypeTok{package =} \StringTok{"PatientLevelPrediction"}\NormalTok{)}
+\CommentTok{\# Show all demos in our package: }
+\FunctionTok{demo}\NormalTok{(}\AttributeTok{package =} \StringTok{"PatientLevelPrediction"}\NormalTok{)}
 
-\CommentTok{# For example, to run the SingleModelDemo that runs Lasso and shows you how to run the Shiny App use this call}
-\KeywordTok{demo}\NormalTok{(}\StringTok{"SingleModelDemo"}\NormalTok{, }\DataTypeTok{package =} \StringTok{"PatientLevelPrediction"}\NormalTok{)}
+\CommentTok{\# For example, to run the SingleModelDemo that runs Lasso and shows you how to run the Shiny App use this call}
+\FunctionTok{demo}\NormalTok{(}\StringTok{"SingleModelDemo"}\NormalTok{, }\AttributeTok{package =} \StringTok{"PatientLevelPrediction"}\NormalTok{)}
 \end{Highlighting}
 \end{Shaded}
 
@@ -2274,19 +2335,18 @@ \section{Acknowledgments}\label{acknowledgments}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\KeywordTok{citation}\NormalTok{(}\StringTok{"PatientLevelPrediction"}\NormalTok{)}
+\FunctionTok{citation}\NormalTok{(}\StringTok{"PatientLevelPrediction"}\NormalTok{)}
 \end{Highlighting}
 \end{Shaded}
 
 \begin{verbatim}
-## 
 ## To cite PatientLevelPrediction in publications use:
 ## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design and
-## implementation of a standardized framework to generate and evaluate patient-level
-## prediction models using observational healthcare data." _Journal of the American
-## Medical Informatics Association_, *25*(8), 969-975. <URL:
-## https://doi.org/10.1093/jamia/ocy032>.
+##   Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design and
+##   implementation of a standardized framework to generate and evaluate patient-level
+##   prediction models using observational healthcare data." _Journal of the American
+##   Medical Informatics Association_, *25*(8), 969-975.
+##   <https://doi.org/10.1093/jamia/ocy032>.
 ## 
 ## A BibTeX entry for LaTeX users is
 ## 
@@ -2307,18 +2367,17 @@ \section{Acknowledgments}\label{acknowledgments}}
 
 \begin{Shaded}
 \begin{Highlighting}[]
-\KeywordTok{citation}\NormalTok{(}\StringTok{"Cyclops"}\NormalTok{)}
+\FunctionTok{citation}\NormalTok{(}\StringTok{"Cyclops"}\NormalTok{)}
 \end{Highlighting}
 \end{Shaded}
 
 \begin{verbatim}
-## 
 ## To cite Cyclops in publications use:
 ## 
-## Suchard MA, Simpson SE, Zorych I, Ryan P, Madigan D (2013). "Massive
-## parallelization of serial inference algorithms for complex generalized linear
-## models." _ACM Transactions on Modeling and Computer Simulation_, *23*, 10. <URL:
-## http://dl.acm.org/citation.cfm?id=2414791>.
+##   Suchard MA, Simpson SE, Zorych I, Ryan P, Madigan D (2013). "Massive parallelization of
+##   serial inference algorithms for complex generalized linear models." _ACM Transactions
+##   on Modeling and Computer Simulation_, *23*, 10.
+##   <https://dl.acm.org/doi/10.1145/2414416.2414791>.
 ## 
 ## A BibTeX entry for LaTeX users is
 ## 
@@ -2329,7 +2388,7 @@ \section{Acknowledgments}\label{acknowledgments}}
 ##     volume = {23},
 ##     pages = {10},
 ##     year = {2013},
-##     url = {http://dl.acm.org/citation.cfm?id=2414791},
+##     url = {https://dl.acm.org/doi/10.1145/2414416.2414791},
 ##   }
 \end{verbatim}
 
@@ -2365,20 +2424,23 @@ \section*{Appendix 1: Study population settings
 \item
   Require minimum time-at-risk for all person in the target cohort
 
-  \includegraphics{popdef1.png}
+  \includegraphics{popdef1.webp}
 \item
   Require minumum time-at-risk for target cohort, except for persons
   with outcomes during time-at-risk.
 
-  \includegraphics{popdef2.png}
+  \includegraphics{popdef2.webp}
 \end{enumerate}
 
-\newpage 3)
+\newpage
+3
+
+)
 
 Include all persons in the target cohort exclude persons with prior
 outcomes
 
-\includegraphics{popdef3.png}
+\includegraphics{popdef3.webp}
 
 \begin{enumerate}
 \def\labelenumi{\arabic{enumi})}
@@ -2387,23 +2449,25 @@ \section*{Appendix 1: Study population settings
   Require minimum time-at-risk for target cohort, except for persons
   with outcomes during time-at-risk, exclude persons with prior outcomes
 
-  \includegraphics{popdef4.png}
+  \includegraphics{popdef4.webp}
 \end{enumerate}
 
 \newpage
+5
+
+)
+
+Include all persons in target cohort exclude persons with prior outcomes
+
+\includegraphics{popdef5.webp}
 
 \begin{enumerate}
 \def\labelenumi{\arabic{enumi})}
-\setcounter{enumi}{4}
-\item
-  Include all persons in target cohort exclude persons with prior
-  outcomes
-
-  \includegraphics{popdef5.png}
+\setcounter{enumi}{5}
 \item
   Include all persons in target cohort
 
-  \includegraphics{popdef6.png}
+  \includegraphics{popdef6.webp}
 \end{enumerate}
 
 \end{document}
diff --git a/inst/doc/CreatingLearningCurves.tex b/inst/doc/CreatingLearningCurves.tex
deleted file mode 100644
index 79283a598..000000000
--- a/inst/doc/CreatingLearningCurves.tex
+++ /dev/null
@@ -1,408 +0,0 @@
-% Options for packages loaded elsewhere
-\PassOptionsToPackage{unicode}{hyperref}
-\PassOptionsToPackage{hyphens}{url}
-%
-\documentclass[
-]{article}
-\usepackage{lmodern}
-\usepackage{amssymb,amsmath}
-\usepackage{ifxetex,ifluatex}
-\ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex
-  \usepackage[T1]{fontenc}
-  \usepackage[utf8]{inputenc}
-  \usepackage{textcomp} % provide euro and other symbols
-\else % if luatex or xetex
-  \usepackage{unicode-math}
-  \defaultfontfeatures{Scale=MatchLowercase}
-  \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
-\fi
-% Use upquote if available, for straight quotes in verbatim environments
-\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
-\IfFileExists{microtype.sty}{% use microtype if available
-  \usepackage[]{microtype}
-  \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
-}{}
-\makeatletter
-\@ifundefined{KOMAClassName}{% if non-KOMA class
-  \IfFileExists{parskip.sty}{%
-    \usepackage{parskip}
-  }{% else
-    \setlength{\parindent}{0pt}
-    \setlength{\parskip}{6pt plus 2pt minus 1pt}}
-}{% if KOMA class
-  \KOMAoptions{parskip=half}}
-\makeatother
-\usepackage{xcolor}
-\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
-\IfFileExists{bookmark.sty}{\usepackage{bookmark}}{\usepackage{hyperref}}
-\hypersetup{
-  pdftitle={Creating Learning Curves},
-  pdfauthor={Luis H. John, Jenna M. Reps, Peter R. Rijnbeek},
-  hidelinks,
-  pdfcreator={LaTeX via pandoc}}
-\urlstyle{same} % disable monospaced font for URLs
-\usepackage[margin=1in]{geometry}
-\usepackage{color}
-\usepackage{fancyvrb}
-\newcommand{\VerbBar}{|}
-\newcommand{\VERB}{\Verb[commandchars=\\\{\}]}
-\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
-% Add ',fontsize=\small' for more characters per line
-\usepackage{framed}
-\definecolor{shadecolor}{RGB}{248,248,248}
-\newenvironment{Shaded}{\begin{snugshade}}{\end{snugshade}}
-\newcommand{\AlertTok}[1]{\textcolor[rgb]{0.94,0.16,0.16}{#1}}
-\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.77,0.63,0.00}{#1}}
-\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\BuiltInTok}[1]{#1}
-\newcommand{\CharTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
-\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
-\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{#1}}
-\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\ErrorTok}[1]{\textcolor[rgb]{0.64,0.00,0.00}{\textbf{#1}}}
-\newcommand{\ExtensionTok}[1]{#1}
-\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\ImportTok}[1]{#1}
-\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
-\newcommand{\NormalTok}[1]{#1}
-\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.81,0.36,0.00}{\textbf{#1}}}
-\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{#1}}
-\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
-\newcommand{\RegionMarkerTok}[1]{#1}
-\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\StringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\usepackage{graphicx,grffile}
-\makeatletter
-\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
-\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
-\makeatother
-% Scale images if necessary, so that they will not overflow the page
-% margins by default, and it is still possible to overwrite the defaults
-% using explicit options in \includegraphics[width, height, ...]{}
-\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
-% Set default figure placement to htbp
-\makeatletter
-\def\fps@figure{htbp}
-\makeatother
-\setlength{\emergencystretch}{3em} % prevent overfull lines
-\providecommand{\tightlist}{%
-  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
-\setcounter{secnumdepth}{-\maxdimen} % remove section numbering
-\usepackage{fancyhdr}
-\pagestyle{fancy}
-\fancyhead{}
-\fancyhead[CO,CE]{Generating Learning Curves}
-\fancyfoot[LE,RO]{\thepage}
-\renewcommand{\headrulewidth}{0.4pt}
-\renewcommand{\footrulewidth}{0.4pt}
-\fancyfoot[CO,CE]{PatientLevelPrediction Package Version 4.0.6}
-
-\title{Creating Learning Curves}
-\author{Luis H. John, Jenna M. Reps, Peter R. Rijnbeek}
-\date{2020-08-19}
-
-\begin{document}
-\maketitle
-
-\hypertarget{introduction}{%
-\section{Introduction}\label{introduction}}
-
-This vignette describes how you can use the Observational Health Data
-Sciences and Informatics (OHDSI)
-\href{http://github.com/OHDSI/PatientLevelPrediction}{\texttt{PatientLevelPrediction}}
-package to create learning curves. This vignette assumes you have read
-and are comfortable with building patient level prediction models as
-described in the
-\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf}{\texttt{BuildingPredictiveModels}
-vignette}.
-
-Prediction models will show overly-optimistic performance when
-predicting on the same data as used for training. Therefore,
-best-practice is to partition our data into a training set and testing
-set. We then train our prediction model on the training set portion and
-asses its ability to generalize to unseen data by measuring its
-performance on the testing set.
-
-Learning curves assess the effect of training set size on model
-performance by training a sequence of prediction models on successively
-larger subsets of the training set. A learning curve plot can also help
-in diagnosing a bias or variance problem as explained below.
-
-\begin{figure}
-\centering
-\includegraphics{learningCurve.png}
-\caption{Learning curve example.}
-\end{figure}
-
-Figure 1, shows an example of learning curve plot in which the vertical
-axis represents the model performance and the horizontal axis the
-training set size. If training set size is small, the performance on the
-training set is high, because a model can often be fitted well to a
-limited number of training examples. At the same time, the performance
-on the testing set will be poor, because the model trained on such a
-limited number of training examples will not generalize well to unseen
-data in the testing set. As the training set size increases, the
-performance of the model on the training set will decrease. It becomes
-more difficult for the model to find a good fit through all the training
-examples. Also, the model will be trained on a more representative
-portion of training examples, making it generalize better to unseen
-data. This can be observed by the increasin testing set performance.
-
-The learning curve can help us in diagnosing bias and variance problems
-with our classifier which will provide guidance on how to further
-improve our model. We can observe high variance (overfitting) in a
-prediction model if it performs well on the training set, but poorly on
-the testing set (Figure 2). Adding additional data is a common approach
-to counteract high variance. From the learning curve it becomes
-apparent, that adding additional data may improve performance on the
-testing set a little further, as the learning curve has not yet
-plateaued and, thus, the model is not saturated yet. Therefore, adding
-more data will decrease the gap between training set and testing set,
-which is the main indicator for a high variance problem.
-
-\begin{figure}
-\centering
-\includegraphics{learningCurveVariance.png}
-\caption{Prediction model suffering from high variance.}
-\end{figure}
-
-Furthermore, we can observe high bias (underfitting) if a prediction
-model performs poorly on the training set as well as on the testing set
-(Figure 3). The learning curves of training set and testing set have
-flattened on a low performance with only a small gap in between them.
-Adding additional data will in this case have little to no impact on the
-model performance. Choosing another prediction algorithm that can find
-more complex (for example non-linear) relationships in the data may be
-an alternative approach to consider in this high bias situation.
-
-\begin{figure}
-\centering
-\includegraphics{learningCurveBias.png}
-\caption{Prediction model suffering from high bias.}
-\end{figure}
-
-\hypertarget{creating-the-learning-curve}{%
-\section{Creating the learning
-curve}\label{creating-the-learning-curve}}
-
-Use the
-\href{http://github.com/OHDSI/PatientLevelPrediction}{\texttt{PatientLevelPrediction}}
-package to create a \texttt{population} and \texttt{plpData} object.
-Alternatively, you can make use of the data simulator. The following
-code snippet creates a population of 12000 patients.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{set.seed}\NormalTok{(}\DecValTok{1234}\NormalTok{)}
-\KeywordTok{data}\NormalTok{(plpDataSimulationProfile)}
-\NormalTok{sampleSize <-}\StringTok{ }\DecValTok{12000}
-\NormalTok{plpData <-}\StringTok{ }\KeywordTok{simulatePlpData}\NormalTok{(}
-\NormalTok{  plpDataSimulationProfile,}
-  \DataTypeTok{n =}\NormalTok{ sampleSize}
-\NormalTok{)}
-
-\NormalTok{population <-}\StringTok{ }\KeywordTok{createStudyPopulation}\NormalTok{(}
-\NormalTok{  plpData,}
-  \DataTypeTok{outcomeId =} \DecValTok{2}\NormalTok{,}
-  \DataTypeTok{binary =} \OtherTok{TRUE}\NormalTok{,}
-  \DataTypeTok{firstExposureOnly =} \OtherTok{FALSE}\NormalTok{,}
-  \DataTypeTok{washoutPeriod =} \DecValTok{0}\NormalTok{,}
-  \DataTypeTok{removeSubjectsWithPriorOutcome =} \OtherTok{FALSE}\NormalTok{,}
-  \DataTypeTok{priorOutcomeLookback =} \DecValTok{99999}\NormalTok{,}
-  \DataTypeTok{requireTimeAtRisk =} \OtherTok{FALSE}\NormalTok{,}
-  \DataTypeTok{minTimeAtRisk =} \DecValTok{0}\NormalTok{,}
-  \DataTypeTok{riskWindowStart =} \DecValTok{0}\NormalTok{,}
-  \DataTypeTok{riskWindowEnd =} \DecValTok{365}\NormalTok{,}
-  \DataTypeTok{verbosity =} \StringTok{"INFO"}
-\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-Specify the prediction algorithm to be used.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\CommentTok{# Use LASSO logistic regression}
-\NormalTok{modelSettings <-}\StringTok{ }\KeywordTok{setLassoLogisticRegression}\NormalTok{()}
-\end{Highlighting}
-\end{Shaded}
-
-Specify a test fraction and a sequence of training set fractions.
-Alternatively, you can provide a sequence of training events instead of
-the training set fractions. This is recommended, because events are
-determinant of model performance. Make sure that your training set can
-provide the number of events specified.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{testFraction <-}\StringTok{ }\FloatTok{0.2}
-\NormalTok{trainFractions <-}\StringTok{ }\KeywordTok{seq}\NormalTok{(}\FloatTok{0.1}\NormalTok{, }\FloatTok{0.8}\NormalTok{, }\FloatTok{0.1}\NormalTok{) }\CommentTok{# Create eight training set fractions}
-
-\CommentTok{# alternatively use a sequence of training events by uncommenting the line below.}
-\CommentTok{# trainEvents <- seq(100, 5000, 100)}
-\end{Highlighting}
-\end{Shaded}
-
-Specify the test split to be used.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\CommentTok{# Use a split by person, alternatively a time split is possible}
-\NormalTok{testSplit <-}\StringTok{ 'stratified'}
-\end{Highlighting}
-\end{Shaded}
-
-Create the learning curve object.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{learningCurve <-}\StringTok{ }\KeywordTok{createLearningCurve}\NormalTok{(population,}
-                                     \DataTypeTok{plpData =}\NormalTok{ plpData,}
-                                     \DataTypeTok{modelSettings =}\NormalTok{ modelSettings,}
-                                     \DataTypeTok{testFraction =} \FloatTok{0.2}\NormalTok{,}
-                                     \DataTypeTok{verbosity =} \StringTok{"TRACE"}\NormalTok{,}
-                                     \DataTypeTok{trainFractions =}\NormalTok{ trainFractions,}
-                                     \CommentTok{# trainEvents = trainEvents,}
-                                     \DataTypeTok{splitSeed =} \DecValTok{1000}\NormalTok{,}
-                                     \DataTypeTok{saveModel =} \OtherTok{TRUE}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-Plot the learning curve object (Figure 4). Specify one of the available
-metrics: \texttt{AUROC}, \texttt{AUPRC}, \texttt{sBrier}. Moreover, you
-can specify what metric to put on the abscissa, number of
-\texttt{observations} or number of \texttt{events}. We recommend the
-latter, because \texttt{events} are determinant of model performance and
-allow you to better compare learning curves across different prediction
-problems and databases.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{plotLearningCurve}\NormalTok{(}
-\NormalTok{  learningCurve,}
-  \DataTypeTok{metric =} \StringTok{'AUROC'}\NormalTok{,}
-  \DataTypeTok{abscissa =} \StringTok{'events'}\NormalTok{,}
-  \DataTypeTok{plotTitle =} \StringTok{'Learning Curve'}\NormalTok{,}
-  \DataTypeTok{plotSubtitle =} \StringTok{'AUROC performance'}
-\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-\begin{figure}
-\centering
-\includegraphics{learningCurvePlot.png}
-\caption{Learning curve plot.}
-\end{figure}
-
-\hypertarget{parallel-processing}{%
-\section{Parallel processing}\label{parallel-processing}}
-
-The learning curve object can be created in parallel, which can reduce
-computation time significantly. Currently this functionality is only
-available for LASSO logistic regression and gradient boosting machines.
-Depending on the number of parallel workers it may require a significant
-amount of memory. We advise to use the parallelized learning curve
-function for parameter search and exploratory data analysis.
-
-Use the parallelized version of the learning curve function to create
-the learning curve object in parallel. R will find the number of
-available processing cores automatically and register the required
-parallel backend. Alternatively, you can provide the number of cores you
-wish to use.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{learningCurvePar <-}\StringTok{ }\KeywordTok{createLearningCurvePar}\NormalTok{(}
-\NormalTok{  population,}
-  \DataTypeTok{plpData =}\NormalTok{  plpData,}
-  \DataTypeTok{modelSettings =}\NormalTok{ modelSettings,}
-  \DataTypeTok{testSplit =}\NormalTok{ testSplit,}
-  \DataTypeTok{testFraction =}\NormalTok{ testFraction,}
-  \DataTypeTok{trainEvents =}\NormalTok{ trainEvents,}
-  \DataTypeTok{cores =} \DecValTok{4}\NormalTok{,}
-  \DataTypeTok{splitSeed =} \DecValTok{1000}
-\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{demo}{%
-\section{Demo}\label{demo}}
-
-We have added a demo of the learningcurve:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\CommentTok{# Show all demos in our package: }
- \KeywordTok{demo}\NormalTok{(}\DataTypeTok{package =} \StringTok{"PatientLevelPrediction"}\NormalTok{)}
-
-\CommentTok{# Run the learning curve}
- \KeywordTok{demo}\NormalTok{(}\StringTok{"LearningCurveDemo"}\NormalTok{, }\DataTypeTok{package =} \StringTok{"PatientLevelPrediction"}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-Do note that running this demo can take a considerable amount of time
-(15 min on Quad core running in parallel)!
-
-\hypertarget{publication}{%
-\section{Publication}\label{publication}}
-
-A publication titled `How little data do we need for patient-level
-prediction?' uses the learning curve functionality in this package and
-can be accessed as preprint in the arXiv archives at
-\url{https://arxiv.org/abs/2008.07361}.
-
-\hypertarget{acknowledgments}{%
-\section{Acknowledgments}\label{acknowledgments}}
-
-Considerable work has been dedicated to provide the
-\texttt{PatientLevelPrediction} package.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{citation}\NormalTok{(}\StringTok{"PatientLevelPrediction"}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-\begin{verbatim}
-## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design and implementation of a standardized framework
-## to generate and evaluate patient-level prediction models using observational healthcare data." _Journal of the
-## American Medical Informatics Association_, *25*(8), 969-975. <URL: https://doi.org/10.1093/jamia/ocy032>.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }
-\end{verbatim}
-
-\textbf{Please reference this paper if you use the PLP Package in your
-work:}
-
-\href{http://dx.doi.org/10.1093/jamia/ocy032}{Reps JM, Schuemie MJ,
-Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a
-standardized framework to generate and evaluate patient-level prediction
-models using observational healthcare data. J Am Med Inform Assoc.
-2018;25(8):969-975.}
-
-\end{document}
diff --git a/inst/doc/CreatingNetworkstudies.tex b/inst/doc/CreatingNetworkstudies.tex
deleted file mode 100644
index 62ec2f7e2..000000000
--- a/inst/doc/CreatingNetworkstudies.tex
+++ /dev/null
@@ -1,420 +0,0 @@
-% Options for packages loaded elsewhere
-\PassOptionsToPackage{unicode}{hyperref}
-\PassOptionsToPackage{hyphens}{url}
-%
-\documentclass[
-]{article}
-\usepackage{lmodern}
-\usepackage{amssymb,amsmath}
-\usepackage{ifxetex,ifluatex}
-\ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex
-  \usepackage[T1]{fontenc}
-  \usepackage[utf8]{inputenc}
-  \usepackage{textcomp} % provide euro and other symbols
-\else % if luatex or xetex
-  \usepackage{unicode-math}
-  \defaultfontfeatures{Scale=MatchLowercase}
-  \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
-\fi
-% Use upquote if available, for straight quotes in verbatim environments
-\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
-\IfFileExists{microtype.sty}{% use microtype if available
-  \usepackage[]{microtype}
-  \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
-}{}
-\makeatletter
-\@ifundefined{KOMAClassName}{% if non-KOMA class
-  \IfFileExists{parskip.sty}{%
-    \usepackage{parskip}
-  }{% else
-    \setlength{\parindent}{0pt}
-    \setlength{\parskip}{6pt plus 2pt minus 1pt}}
-}{% if KOMA class
-  \KOMAoptions{parskip=half}}
-\makeatother
-\usepackage{xcolor}
-\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
-\IfFileExists{bookmark.sty}{\usepackage{bookmark}}{\usepackage{hyperref}}
-\hypersetup{
-  pdftitle={Making patient-level predictive network study packages},
-  pdfauthor={Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek},
-  hidelinks,
-  pdfcreator={LaTeX via pandoc}}
-\urlstyle{same} % disable monospaced font for URLs
-\usepackage[margin=1in]{geometry}
-\usepackage{color}
-\usepackage{fancyvrb}
-\newcommand{\VerbBar}{|}
-\newcommand{\VERB}{\Verb[commandchars=\\\{\}]}
-\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
-% Add ',fontsize=\small' for more characters per line
-\usepackage{framed}
-\definecolor{shadecolor}{RGB}{248,248,248}
-\newenvironment{Shaded}{\begin{snugshade}}{\end{snugshade}}
-\newcommand{\AlertTok}[1]{\textcolor[rgb]{0.94,0.16,0.16}{#1}}
-\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.77,0.63,0.00}{#1}}
-\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\BuiltInTok}[1]{#1}
-\newcommand{\CharTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
-\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
-\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{#1}}
-\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\ErrorTok}[1]{\textcolor[rgb]{0.64,0.00,0.00}{\textbf{#1}}}
-\newcommand{\ExtensionTok}[1]{#1}
-\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\ImportTok}[1]{#1}
-\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
-\newcommand{\NormalTok}[1]{#1}
-\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.81,0.36,0.00}{\textbf{#1}}}
-\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{#1}}
-\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
-\newcommand{\RegionMarkerTok}[1]{#1}
-\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\StringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\usepackage{longtable,booktabs}
-% Correct order of tables after \paragraph or \subparagraph
-\usepackage{etoolbox}
-\makeatletter
-\patchcmd\longtable{\par}{\if@noskipsec\mbox{}\fi\par}{}{}
-\makeatother
-% Allow footnotes in longtable head/foot
-\IfFileExists{footnotehyper.sty}{\usepackage{footnotehyper}}{\usepackage{footnote}}
-\makesavenoteenv{longtable}
-\usepackage{graphicx,grffile}
-\makeatletter
-\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
-\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
-\makeatother
-% Scale images if necessary, so that they will not overflow the page
-% margins by default, and it is still possible to overwrite the defaults
-% using explicit options in \includegraphics[width, height, ...]{}
-\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
-% Set default figure placement to htbp
-\makeatletter
-\def\fps@figure{htbp}
-\makeatother
-\setlength{\emergencystretch}{3em} % prevent overfull lines
-\providecommand{\tightlist}{%
-  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
-\setcounter{secnumdepth}{5}
-
-\title{Making patient-level predictive network study packages}
-\author{Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek}
-\date{2020-06-03}
-
-\begin{document}
-\maketitle
-
-{
-\setcounter{tocdepth}{2}
-\tableofcontents
-}
-\newpage
-
-\hypertarget{introduction}{%
-\section{Introduction}\label{introduction}}
-
-The OHDSI Patient Level Prediction (PLP) package provides the framework
-to implement prediction models at scale. This can range from developing
-a large number of models across sites (methodology and study design
-insight) to extensive external validation of existing models in the
-OHDSI PLP framework (model insight). This vignette describes how you can
-use the \texttt{PatientLevelPrediction} package to create a network
-study package.
-
-\hypertarget{running-a-network-study-process}{%
-\section{Running a Network study
-Process}\label{running-a-network-study-process}}
-
-\hypertarget{step-1-developing-the-study}{%
-\subsection{Step 1 -- developing the
-study}\label{step-1-developing-the-study}}
-
-\begin{itemize}
-\tightlist
-\item
-  Design the study: target/outcome cohort logic, concept sets for
-  medical definitions, settings for developing new model or validation
-  of adding existing models to framework. Suggestion: look in literature
-  for validated definitions.
-\item
-  Write a protocol that motivates the study and provides full details
-  (sufficient for people to replicate the study in the future).
-\item
-  Write an R package for implementing the study across diverse
-  computational environments {[}see guidance below for structure of
-  package and use the skeleton github package here: \ldots{} {]}
-\end{itemize}
-
-\hypertarget{step-2-implementing-the-study-part-1}{%
-\subsection{Step 2 -- implementing the study part
-1}\label{step-2-implementing-the-study-part-1}}
-
-\begin{itemize}
-\tightlist
-\item
-  Get contributors to install the package and dependencies. Ensure the
-  package is installed correctly by running the checkInstall functions.
-\item
-  Get contributors to run the createCohort function to inspect the
-  target/outcome definitions. If the definitions are not suitable for a
-  site, go back to step 1 and revise the cohort definitions.
-\end{itemize}
-
-\hypertarget{step-3-implementing-the-study-part-2-make-sure-package-checks-outputs-the-package-is-functioning-as-planned-and-the-definitions-are-valid-across-sites}{%
-\subsection{Step 3 -- implementing the study part 2 {[}make sure package
-checks outputs the package is functioning as planned and the definitions
-are valid across
-sites{]}}\label{step-3-implementing-the-study-part-2-make-sure-package-checks-outputs-the-package-is-functioning-as-planned-and-the-definitions-are-valid-across-sites}}
-
-\begin{itemize}
-\tightlist
-\item
-  Get contributors to run the main.R with the settings configured to
-  their environment
-\item
-  Get the contributors to submit the results
-\end{itemize}
-
-\hypertarget{step-4-publication}{%
-\subsection{Step 4 -- Publication}\label{step-4-publication}}
-
-\begin{itemize}
-\tightlist
-\item
-  The study creator has the first option to be first author, if he/she
-  does not wish to be first author then he/she can pick the most
-  suitable person from the contributors. All contributors will be listed
-  as authors on the paper. The last author will be the person who
-  lead/managed the study, if this was the first author then the first
-  author can pick the most suitable last author. All authors between the
-  first and last author will be alphabetical by last name.
-\end{itemize}
-
-\hypertarget{package-skeleton---file-structure}{%
-\section{Package Skeleton - File
-Structure}\label{package-skeleton---file-structure}}
-
-\begin{itemize}
-\tightlist
-\item
-  DESCRIPTION -- This file describes the R package and the dependencies
-\item
-  NAMESPACE -- This file is created automatically by Roxygen
-\item
-  Readme.md -- This file should provide the step by step guidance on
-  implementing the package
-\item
-  R
-\item
-  helpers.r -- all the custom functions used by the package should be in
-  this file (e.g., checkInstall)
-\item
-  main.r -- this file will call the functions in helpers.r to execute
-  the full study
-\item
-  submit.r -- this file will be called at the end the submit the
-  compressed folder to the study creator/manager.
-\item
-  Man -- this folder will contain the documentation for the functions in
-  helpers.r (this should be automatically generated by roxygen)
-\item
-  Inst
-\item
-  sql/sql\_sever
-
-  \begin{itemize}
-  \tightlist
-  \item
-    targetCohort -- the target cohort parameterised sql code
-  \item
-    outcomeCohort -- the outcome cohort parameterised sql code
-  \end{itemize}
-\item
-  extdata -- place any data required for the package here
-\item
-  plp\_models -- place any PLP models here
-\item
-  existing\_models -- place the files for existing models here
-\item
-  Extras
-\end{itemize}
-
-\hypertarget{package-skeleton---output-of-running-package}{%
-\section{Package Skeleton - Output of Running
-Package}\label{package-skeleton---output-of-running-package}}
-
-The output should contain three folders inside the study directory such
-as
-\texttt{outputLoc\ \textless{}-\ (file.path(getwd(),\ paste0(studyName\_database\_date)))}:
-* Plots -- containing the test/train or validation ROC plot, calibration
-plot, precision recall plot and optionally the demographic calibration
-plot. * Results -- The output of running savePlpResult * Summary -- a
-summary csv of performance and the table 1 csv
-
-Then there should also be a zip file of the folder in the working
-directory containing the same folders and files but with sensitive
-results removed (this will be created using the packageResults
-function). Once the contributor has inspected the zipped file and is
-happy with the content being shared, he/she can then finally run the
-submit function with the details provided in the readme.md.
-
-\hypertarget{example-code-to-make-package-for-external-validation-of-plp-model}{%
-\section{Example Code To Make Package For External Validation of PLP
-Model}\label{example-code-to-make-package-for-external-validation-of-plp-model}}
-
-First you need to make a copy of the PatientLevelPrediciton skeleton
-package found here:
-
-Assuming you ran a sucessful PatientLevelPrediction model development
-and saved the output of \texttt{runPlp()} to to location `goodModel' in
-your working directory then:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{library}\NormalTok{(PatientLevelPrediction)}
-\NormalTok{plpResult <-}\StringTok{ }\KeywordTok{loadPlpResult}\NormalTok{(}\StringTok{"goodModel"}\NormalTok{)}
-
-\CommentTok{# add the model to the skeleton package with sensitive information removed}
-\KeywordTok{exportPlpResult}\NormalTok{(}\DataTypeTok{plpResult =}\NormalTok{ plpResult, }\DataTypeTok{modelName =} \StringTok{"Model Name"}\NormalTok{, }\DataTypeTok{packageName =} \StringTok{"Your Package Name"}\NormalTok{, }
-    \DataTypeTok{gitHubLocation =} \StringTok{"location/of/github"}\NormalTok{, }\DataTypeTok{includeEvaluationStatistics =}\NormalTok{ T, }\DataTypeTok{includeThresholdSummary =}\NormalTok{ T, }
-    \DataTypeTok{includeDemographicSummary =}\NormalTok{ T, }\DataTypeTok{includeCalibrationSummary =}\NormalTok{ T, }\DataTypeTok{includePredictionDistribution =}\NormalTok{ T, }
-    \DataTypeTok{includeCovariateSummary =}\NormalTok{ F)}
-\end{Highlighting}
-\end{Shaded}
-
-Now you want to add the cohorts (generally the parameterized sql
-required to create one or more target and outcome cohorts). This should
-be added into the inst/sql/sql\_server directory of your package. If you
-are using atlas to create the cohorts then you can use:
-\texttt{OhdsiRTools::insertCirceDefinitionInPackage()}. The settings for
-the cohort creation are defined in the inst/extdata directory in the
-file cohort\_details.csv. this file contains two columns: cohortName and
-cohortId. The cohortName should contain the name of the sql file of the
-cohort in inst/sql/sql\_server (e.g., a file called ``targetCohort.sql''
-has the name ``targetCohort'') and the cohortId is the default
-cohort\_definition\_id that will be used when people run the study
-corresponding to this cohort. The main.R file in the extras directory
-contains the vanilla code to run a study with the model eported into the
-package and the cohort files added.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{library}\NormalTok{(PatientLevelPrediction)}
-\CommentTok{# input settings for person running the study}
-\NormalTok{connectionDetails <-}\StringTok{ " "}
-\NormalTok{cdmDatabaseSchema <-}\StringTok{ "their_cdm_database"}
-\NormalTok{databaseName <-}\StringTok{ "Name for database"}
-\NormalTok{cohortDatabaseSchema <-}\StringTok{ "a_database_with_write_priv"}
-\NormalTok{cohortTable <-}\StringTok{ "package_table"}
-\NormalTok{outputLocation <-}\StringTok{ "location to save results"}
-
-\NormalTok{cohortDetails <-}\StringTok{ }\KeywordTok{createCohort}\NormalTok{(}\DataTypeTok{connectionDetails =}\NormalTok{ connectionDetails, }\DataTypeTok{cdmDatabaseSchema =}\NormalTok{ cdmDatabaseSchema, }
-    \DataTypeTok{cohortDatabaseSchema =}\NormalTok{ cohortDatabaseSchema, }\DataTypeTok{cohortTable =}\NormalTok{ cohortTable, }\DataTypeTok{package =} \StringTok{"Your Package Name"}\NormalTok{)}
-
-\NormalTok{plpResult <-}\StringTok{ }\KeywordTok{loadPlpResult}\NormalTok{(}\KeywordTok{system.file}\NormalTok{(}\StringTok{"model"}\NormalTok{, }\DataTypeTok{package =} \StringTok{"Your Package Name"}\NormalTok{))}
-\NormalTok{result <-}\StringTok{ }\KeywordTok{externalValidatePlp}\NormalTok{(}\DataTypeTok{plpResult =}\NormalTok{ plpResult, }\DataTypeTok{connectionDetails =}\NormalTok{ connectionDetails, }\DataTypeTok{validationSchemaTarget =}\NormalTok{ cohortDatabaseSchema, }
-    \DataTypeTok{validationSchemaOutcome =}\NormalTok{ cohortDatabaseSchema, }\DataTypeTok{validationSchemaCdm =}\NormalTok{ cdmDatabaseSchema, }\DataTypeTok{validationTableTarget =}\NormalTok{ cohortTable, }
-    \DataTypeTok{validationTableOutcome =}\NormalTok{ cohortTable, }\DataTypeTok{validationIdTarget =}\NormalTok{ target_cohort_id, }\DataTypeTok{validationIdOutcome =}\NormalTok{ outcome_cohort_id)}
-
-\CommentTok{# save results to standard output}
-\NormalTok{resultLoc <-}\StringTok{ }\KeywordTok{standardOutput}\NormalTok{(}\DataTypeTok{result =}\NormalTok{ result, }\DataTypeTok{outputLocation =}\NormalTok{ outputLocation, }\DataTypeTok{studyName =} \StringTok{"external validation of ... model"}\NormalTok{, }
-    \DataTypeTok{databaseName =}\NormalTok{ databaseName, }\DataTypeTok{cohortName =} \StringTok{"your cohortName"}\NormalTok{, }\DataTypeTok{outcomeName =} \StringTok{"your outcomeName"}\NormalTok{)}
-
-\CommentTok{# package results ready to submit}
-\KeywordTok{packageResults}\NormalTok{(}\DataTypeTok{mainFolder =}\NormalTok{ resultLoc, }\DataTypeTok{includeROCplot =}\NormalTok{ T, }\DataTypeTok{includeCalibrationPlot =}\NormalTok{ T, }\DataTypeTok{includePRPlot =}\NormalTok{ T, }
-    \DataTypeTok{includeTable1 =}\NormalTok{ F, }\DataTypeTok{includeThresholdSummary =}\NormalTok{ T, }\DataTypeTok{includeDemographicSummary =}\NormalTok{ T, }\DataTypeTok{includeCalibrationSummary =}\NormalTok{ T, }
-    \DataTypeTok{includePredictionDistribution =}\NormalTok{ T, }\DataTypeTok{includeCovariateSummary =}\NormalTok{ F, }\DataTypeTok{removeLessThanN =}\NormalTok{ F, }\DataTypeTok{N =} \DecValTok{10}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-Where the target\_cohort\_id and outcome\_cohort\_id should correspond
-to the cohort\_details.csv file.
-
-We recommend getting the network implementors to submit their results of
-\texttt{createCohort()} before continuing with the study to ensure
-definitions run across the network. After running the rest of main.R the
-implementor should inspect the files in the export folder created by the
-package to ensure there isn't sensitive data remaining. Once checked the
-implementor can run submit.R to send the results to the study organisor.
-The submit.R file is:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{submitResults}\NormalTok{(}\DataTypeTok{exportFolder =}\NormalTok{ outputLocation, }\DataTypeTok{dbName =}\NormalTok{ databaseName, key, secret)}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{useful-patientlevelprediction-functions}{%
-\section{Useful PatientLevelPrediction
-Functions}\label{useful-patientlevelprediction-functions}}
-
-The functions to aid the creation of a network study are:
-
-\begin{longtable}[]{@{}lll@{}}
-\toprule
-\begin{minipage}[b]{0.26\columnwidth}\raggedright
-Function\strut
-\end{minipage} & \begin{minipage}[b]{0.42\columnwidth}\raggedright
-Description\strut
-\end{minipage} & \begin{minipage}[b]{0.23\columnwidth}\raggedright
-Usage\strut
-\end{minipage}\tabularnewline
-\midrule
-\endhead
-\begin{minipage}[t]{0.26\columnwidth}\raggedright
-\texttt{checkPlpInstall()}\strut
-\end{minipage} & \begin{minipage}[t]{0.42\columnwidth}\raggedright
-This function checks the connection, and various aspects of the PLP
-package to check it is set up correctly\strut
-\end{minipage} & \begin{minipage}[t]{0.23\columnwidth}\raggedright
-This should be run with the appropriate settings to check the
-contributor is set up correctly for the study\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.26\columnwidth}\raggedright
-\texttt{getPlpData()}\strut
-\end{minipage} & \begin{minipage}[t]{0.42\columnwidth}\raggedright
-This function extracts the data from the cdm for model development\strut
-\end{minipage} & \begin{minipage}[t]{0.23\columnwidth}\raggedright
-This should be used if developing new models\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.26\columnwidth}\raggedright
-\texttt{runPlp()}\strut
-\end{minipage} & \begin{minipage}[t]{0.42\columnwidth}\raggedright
-This function trains and tests a new PLP model\strut
-\end{minipage} & \begin{minipage}[t]{0.23\columnwidth}\raggedright
-This should be used if developing new models\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.26\columnwidth}\raggedright
-\texttt{transportPlp()}\strut
-\end{minipage} & \begin{minipage}[t]{0.42\columnwidth}\raggedright
-This function exports the output of runPlp into an R package while
-removing sensitive objects\strut
-\end{minipage} & \begin{minipage}[t]{0.23\columnwidth}\raggedright
-This should be used when saving a model into a study package to validate
-the model\strut
-\end{minipage}\tabularnewline
-\begin{minipage}[t]{0.26\columnwidth}\raggedright
-\texttt{externalValidatePlp()}\strut
-\end{minipage} & \begin{minipage}[t]{0.42\columnwidth}\raggedright
-This function requires the user to inpute an existing model and then
-extracts the required data on a new database and applies/evaluates the
-model.\strut
-\end{minipage} & \begin{minipage}[t]{0.23\columnwidth}\raggedright
-This should be used if validating a PLP model\strut
-\end{minipage}\tabularnewline
-\bottomrule
-\end{longtable}
-
-\end{document}
diff --git a/inst/doc/CreatingShinyApp.tex b/inst/doc/CreatingShinyApp.tex
deleted file mode 100644
index 1e6884a8a..000000000
--- a/inst/doc/CreatingShinyApp.tex
+++ /dev/null
@@ -1,502 +0,0 @@
-% Options for packages loaded elsewhere
-\PassOptionsToPackage{unicode}{hyperref}
-\PassOptionsToPackage{hyphens}{url}
-%
-\documentclass[
-]{article}
-\usepackage{lmodern}
-\usepackage{amssymb,amsmath}
-\usepackage{ifxetex,ifluatex}
-\ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex
-  \usepackage[T1]{fontenc}
-  \usepackage[utf8]{inputenc}
-  \usepackage{textcomp} % provide euro and other symbols
-\else % if luatex or xetex
-  \usepackage{unicode-math}
-  \defaultfontfeatures{Scale=MatchLowercase}
-  \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
-\fi
-% Use upquote if available, for straight quotes in verbatim environments
-\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
-\IfFileExists{microtype.sty}{% use microtype if available
-  \usepackage[]{microtype}
-  \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
-}{}
-\makeatletter
-\@ifundefined{KOMAClassName}{% if non-KOMA class
-  \IfFileExists{parskip.sty}{%
-    \usepackage{parskip}
-  }{% else
-    \setlength{\parindent}{0pt}
-    \setlength{\parskip}{6pt plus 2pt minus 1pt}}
-}{% if KOMA class
-  \KOMAoptions{parskip=half}}
-\makeatother
-\usepackage{xcolor}
-\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
-\IfFileExists{bookmark.sty}{\usepackage{bookmark}}{\usepackage{hyperref}}
-\hypersetup{
-  pdftitle={Creating Shiny App},
-  pdfauthor={Jenna Reps},
-  hidelinks,
-  pdfcreator={LaTeX via pandoc}}
-\urlstyle{same} % disable monospaced font for URLs
-\usepackage[margin=1in]{geometry}
-\usepackage{color}
-\usepackage{fancyvrb}
-\newcommand{\VerbBar}{|}
-\newcommand{\VERB}{\Verb[commandchars=\\\{\}]}
-\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
-% Add ',fontsize=\small' for more characters per line
-\usepackage{framed}
-\definecolor{shadecolor}{RGB}{248,248,248}
-\newenvironment{Shaded}{\begin{snugshade}}{\end{snugshade}}
-\newcommand{\AlertTok}[1]{\textcolor[rgb]{0.94,0.16,0.16}{#1}}
-\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.77,0.63,0.00}{#1}}
-\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\BuiltInTok}[1]{#1}
-\newcommand{\CharTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
-\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
-\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{#1}}
-\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\ErrorTok}[1]{\textcolor[rgb]{0.64,0.00,0.00}{\textbf{#1}}}
-\newcommand{\ExtensionTok}[1]{#1}
-\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\ImportTok}[1]{#1}
-\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
-\newcommand{\NormalTok}[1]{#1}
-\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.81,0.36,0.00}{\textbf{#1}}}
-\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{#1}}
-\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
-\newcommand{\RegionMarkerTok}[1]{#1}
-\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\StringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\usepackage{graphicx,grffile}
-\makeatletter
-\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
-\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
-\makeatother
-% Scale images if necessary, so that they will not overflow the page
-% margins by default, and it is still possible to overwrite the defaults
-% using explicit options in \includegraphics[width, height, ...]{}
-\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
-% Set default figure placement to htbp
-\makeatletter
-\def\fps@figure{htbp}
-\makeatother
-\setlength{\emergencystretch}{3em} % prevent overfull lines
-\providecommand{\tightlist}{%
-  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
-\setcounter{secnumdepth}{5}
-\usepackage{fancyhdr}
-\pagestyle{fancy}
-\fancyhead{}
-\fancyhead[CO,CE]{Installation Guide}
-\fancyfoot[CO,CE]{PatientLevelPrediction Package Version 3.1.0}
-\fancyfoot[LE,RO]{\thepage}
-\renewcommand{\headrulewidth}{0.4pt}
-\renewcommand{\footrulewidth}{0.4pt}
-
-\title{Creating Shiny App}
-\author{Jenna Reps}
-\date{2020-06-03}
-
-\begin{document}
-\maketitle
-
-{
-\setcounter{tocdepth}{2}
-\tableofcontents
-}
-\hypertarget{introduction}{%
-\section{Introduction}\label{introduction}}
-
-In this vignette we will show with example code how to create a shiny
-app and add the shiny app online for other researcher around the whole
-to explore.
-
-There are two ways to create the shiny app: 1) Using the atlas R
-generated prediction package 2) Manually using the
-PatientLevelPrediction functions in a script
-
-We assume you have experience with using the OHDSI
-PatientLevelPrediction package to develop and externally validate
-prediction models using data in the OMOP CDM. If you do not have
-experience with this then please first read our general vignette
-\href{https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf}{\texttt{BuildingPredictiveModels}
-vignette}.
-
-\hypertarget{atlas-development-shiny-app}{%
-\section{Atlas Development Shiny
-App}\label{atlas-development-shiny-app}}
-
-\hypertarget{step-1-run-the-model-development-package-to-get-results}{%
-\subsection{Step 1: Run the model development package to get
-results}\label{step-1-run-the-model-development-package-to-get-results}}
-
-To create a shiny app project via the Atlas auto-generated prediction R
-package you named `myPackage' you need to run the execute function:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{library}\NormalTok{(myPackage)}
-\NormalTok{myPackage}\OperatorTok{::}\KeywordTok{execute}\NormalTok{(}\DataTypeTok{connectionDetails =}\NormalTok{ connectionDetails,}
-        \DataTypeTok{cdmDatabaseSchema =} \StringTok{'myDatabaseSchema.dbo'}\NormalTok{,}
-        \DataTypeTok{cdmDatabaseName =} \StringTok{'MyDatabase'}\NormalTok{,}
-        \DataTypeTok{cohortDatabaseSchema =} \StringTok{'myDatabaseSchema.ohdsi_results'}\NormalTok{,}
-        \DataTypeTok{cohortTable =} \StringTok{'cohort'}\NormalTok{,}
-        \DataTypeTok{outputFolder =} \StringTok{'C:/myResults'}\NormalTok{,}
-        \DataTypeTok{createProtocol =}\NormalTok{ F,}
-        \DataTypeTok{createCohorts =}\NormalTok{ F,}
-        \DataTypeTok{runAnalyses =}\NormalTok{ T,}
-        \DataTypeTok{createResultsDoc =}\NormalTok{ F,}
-        \DataTypeTok{packageResults =}\NormalTok{ F,}
-        \DataTypeTok{createValidationPackage =}\NormalTok{ F, }
-        \DataTypeTok{minCellCount=} \DecValTok{5}\NormalTok{,}
-        \DataTypeTok{createShiny =}\NormalTok{ F,}
-        \DataTypeTok{createJournalDocument =}\NormalTok{ F,}
-        \DataTypeTok{analysisIdDocument =} \DecValTok{1}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-This will extract data based on the settings you supplied in the Atlas
-prediction design from cohort tables already generated in your CDM
-database schema. The PatientLevelPrediction framework will then run and
-develop/evaluate models saving the results to the location specified by
-outputFolder (e.g., `C:/myResults').
-
-\hypertarget{step-2-create-the-shiny-app}{%
-\subsection{Step 2: Create the shiny
-app}\label{step-2-create-the-shiny-app}}
-
-To create a shiny app project with these results you can then simply
-run:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{myPackage}\OperatorTok{::}\KeywordTok{execute}\NormalTok{(}\DataTypeTok{connectionDetails =}\NormalTok{ connectionDetails,}
-        \DataTypeTok{cdmDatabaseSchema =} \StringTok{'myDatabaseSchema.dbo'}\NormalTok{,}
-        \DataTypeTok{cdmDatabaseName =} \StringTok{'MyDatabase'}\NormalTok{,}
-        \DataTypeTok{cohortDatabaseSchema =} \StringTok{'myDatabaseSchema.ohdsi_results'}\NormalTok{,}
-        \DataTypeTok{cohortTable =} \StringTok{'cohort'}\NormalTok{,}
-        \DataTypeTok{outputFolder =} \StringTok{'C:/myResults'}\NormalTok{,}
-        \DataTypeTok{minCellCount=} \DecValTok{5}\NormalTok{,}
-        \DataTypeTok{createShiny =}\NormalTok{ T)}
-\end{Highlighting}
-\end{Shaded}
-
-making sure the outputFolder is the same location used when you ran the
-analysis. This code populates a shiny app project with the results but
-removes sensitive date such as connection settings, the
-cdmDatabaseSchema setting, the predicton matrix and any sensitive counts
-less than `minCellCount' from the covariate summary and performance
-evalaution.
-
-The shiny app project populated with the model development results can
-then be found at `{[}outputFolder{]}/ShinyApp' e.g.,
-`C:/myResults/ShinyApp'.
-
-\hypertarget{testing-optional-but-recommended}{%
-\subsubsection{Testing (Optional but
-recommended)}\label{testing-optional-but-recommended}}
-
-You can test the app by opening the shiny project within the
-{[}outputFolder{]}/ShinyApp' folder, double click on the file named
-`PLPViewer.Rproj'. This will open an R studio session with the shiny app
-project loaded. Now load the `ui.R' files within this R studio session
-and you will see a green arrow with the words `Run App' at the top right
-of the script. Click on this and the shiny app with open. Note: You may
-need to install some R pacakge dependancies for the shiny app to work.
-
-\hypertarget{step-3-sharing-the-shiny-app}{%
-\subsection{Step 3: Sharing the shiny
-app}\label{step-3-sharing-the-shiny-app}}
-
-Once you are happy with your app, you can publish it onto
-\url{https://data.ohdsi.org} by adding the folder `ShinyApp' to the
-OHDSI githib ShinyDeploy (\url{https://github.com/OHDSI/ShinyDeploy/}).
-Continuing the example, we would copy the folder
-`{[}outputFolder{]}/ShinyApp' and paste it to the local github clone of
-ShinyDeploy. We recommend renaming the folder from `ShinyApp' to a name
-that describes your prediction, e.g., `StrokeinAF'. Then commit the
-changes and make a pull request to ShinyDeploy. Once accepted your shiny
-app will be viewable at `\url{https://data.ohdsi.org}'. If you commited
-the folder named `StrokeInAF' then the shiny app will be hosted at
-`\url{https://data.ohdsi.org/StrokeInAF}'.
-
-\hypertarget{atlas-external-validation}{%
-\section{Atlas External Validation}\label{atlas-external-validation}}
-
-To include external validation results you can use the Atlas generated R
-study package to create the external validation package:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{myPackage}\OperatorTok{::}\KeywordTok{execute}\NormalTok{(}\DataTypeTok{connectionDetails =}\NormalTok{ connectionDetails,}
-        \DataTypeTok{cdmDatabaseSchema =} \StringTok{'myDatabaseSchema.dbo'}\NormalTok{,}
-        \DataTypeTok{cdmDatabaseName =} \StringTok{'MyDatabase'}\NormalTok{,}
-        \DataTypeTok{cohortDatabaseSchema =} \StringTok{'myDatabaseSchema.ohdsi_results'}\NormalTok{,}
-        \DataTypeTok{cohortTable =} \StringTok{'cohort'}\NormalTok{,}
-        \DataTypeTok{outputFolder =} \StringTok{'C:/myResults'}\NormalTok{,}
-        \DataTypeTok{createValidationPackage =}\NormalTok{ T)}
-\end{Highlighting}
-\end{Shaded}
-
-This will create a new R package inside the `outputFolder' location with
-the word `Validation' appended the name of your development package. For
-example, if your `outputFolder' was `C:/myResults' and your development
-package was named `myPackage' then the validation package will be found
-at: `C:/myResults/myPackageValidation'. When running the valdiation
-package make sure to set the `outputFolder' to the Validation folder
-within your model development outputFolder location:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{myPackageValidation}\OperatorTok{::}\KeywordTok{execute}\NormalTok{(}\DataTypeTok{connectionDetails =}\NormalTok{ connectionDetails,}
-                 \DataTypeTok{databaseName =}\NormalTok{ databaseName,}
-                 \DataTypeTok{cdmDatabaseSchema =}\NormalTok{ cdmDatabaseSchema,}
-                 \DataTypeTok{cohortDatabaseSchema =}\NormalTok{ cohortDatabaseSchema,}
-                 \DataTypeTok{oracleTempSchema =}\NormalTok{ oracleTempSchema,}
-                 \DataTypeTok{cohortTable =}\NormalTok{ cohortTable,}
-                 \DataTypeTok{outputFolder =} \StringTok{'C:/myResults/Validation'}\NormalTok{,}
-                 \DataTypeTok{createCohorts =}\NormalTok{ T,}
-                 \DataTypeTok{runValidation =}\NormalTok{ T,}
-                 \DataTypeTok{packageResults =}\NormalTok{ F,}
-                 \DataTypeTok{minCellCount =} \DecValTok{5}\NormalTok{,}
-                 \DataTypeTok{sampleSize =} \OtherTok{NULL}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-Now you can rerun Steps 2-3 to populate the shiny app project that will
-also include the validation results (as long as the validation results
-are in the Validation folder found in the Step 1 outputFolder location
-e.g., in `C:/myResults/Validation').
-
-\hypertarget{combining-multiple-atlas-results-into-one-shiny-app}{%
-\section{Combining multiple atlas results into one shiny
-app:}\label{combining-multiple-atlas-results-into-one-shiny-app}}
-
-The code below can be used to combine multiple Atlas packages' results
-into one shiny app:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{populateMultipleShinyApp <-}\StringTok{ }\ControlFlowTok{function}\NormalTok{(shinyDirectory,}
-\NormalTok{                             resultDirectory,}
-                             \DataTypeTok{minCellCount =} \DecValTok{10}\NormalTok{,}
-                             \DataTypeTok{databaseName =} \StringTok{'sharable name of development data'}\NormalTok{)\{}
-  
-  \CommentTok{#check inputs}
-  \ControlFlowTok{if}\NormalTok{(}\KeywordTok{missing}\NormalTok{(shinyDirectory))\{}
-\NormalTok{    shinyDirectory <-}\StringTok{ }\KeywordTok{system.file}\NormalTok{(}\StringTok{"shiny"}\NormalTok{, }\StringTok{"PLPViewer"}\NormalTok{, }\DataTypeTok{package =} \StringTok{"SkeletonPredictionStudy"}\NormalTok{)}
-\NormalTok{  \}}
-  \ControlFlowTok{if}\NormalTok{(}\KeywordTok{missing}\NormalTok{(resultDirectory))\{}
-    \KeywordTok{stop}\NormalTok{(}\StringTok{'Need to enter the resultDirectory'}\NormalTok{)}
-\NormalTok{  \}}
-  
-
-    \ControlFlowTok{for}\NormalTok{(i }\ControlFlowTok{in} \DecValTok{1}\OperatorTok{:}\KeywordTok{length}\NormalTok{(resultDirectory))\{}
-      \ControlFlowTok{if}\NormalTok{(}\OperatorTok{!}\KeywordTok{dir.exists}\NormalTok{(resultDirectory[i]))\{}
-        \KeywordTok{stop}\NormalTok{(}\KeywordTok{paste}\NormalTok{(}\StringTok{'resultDirectory '}\NormalTok{,i,}\StringTok{' does not exist'}\NormalTok{))}
-\NormalTok{      \}}
-\NormalTok{    \}}
-  
-\NormalTok{  outputDirectory <-}\StringTok{ }\KeywordTok{file.path}\NormalTok{(shinyDirectory,}\StringTok{'data'}\NormalTok{)}
-  
-  \CommentTok{# create the shiny data folder}
-  \ControlFlowTok{if}\NormalTok{(}\OperatorTok{!}\KeywordTok{dir.exists}\NormalTok{(outputDirectory))\{}
-    \KeywordTok{dir.create}\NormalTok{(outputDirectory, }\DataTypeTok{recursive =}\NormalTok{ T)}
-\NormalTok{  \}}
-  
-  
-  \CommentTok{# need to edit settings ...}
-\NormalTok{  files <-}\StringTok{ }\KeywordTok{c}\NormalTok{()}
-  \ControlFlowTok{for}\NormalTok{(i }\ControlFlowTok{in} \DecValTok{1}\OperatorTok{:}\KeywordTok{length}\NormalTok{(resultDirectory))\{}
-  \CommentTok{# copy the settings csv}
-\NormalTok{  file <-}\StringTok{ }\NormalTok{utils}\OperatorTok{::}\KeywordTok{read.csv}\NormalTok{(}\KeywordTok{file.path}\NormalTok{(resultDirectory[i],}\StringTok{'settings.csv'}\NormalTok{))}
-\NormalTok{  file}\OperatorTok{$}\NormalTok{analysisId <-}\StringTok{ }\DecValTok{1000}\OperatorTok{*}\KeywordTok{as.double}\NormalTok{(file}\OperatorTok{$}\NormalTok{analysisId)}\OperatorTok{+}\NormalTok{i}
-\NormalTok{  files <-}\StringTok{ }\KeywordTok{rbind}\NormalTok{(files, file)}
-\NormalTok{  \}}
-\NormalTok{  utils}\OperatorTok{::}\KeywordTok{write.csv}\NormalTok{(files, }\KeywordTok{file.path}\NormalTok{(outputDirectory,}\StringTok{'settings.csv'}\NormalTok{), }\DataTypeTok{row.names =}\NormalTok{ F)}
-  
-  \ControlFlowTok{for}\NormalTok{(i }\ControlFlowTok{in} \DecValTok{1}\OperatorTok{:}\KeywordTok{length}\NormalTok{(resultDirectory))\{}
-  \CommentTok{# copy each analysis as a rds file and copy the log}
-\NormalTok{  files <-}\StringTok{ }\KeywordTok{dir}\NormalTok{(resultDirectory[i], }\DataTypeTok{full.names =}\NormalTok{ F)}
-\NormalTok{  files <-}\StringTok{ }\NormalTok{files[}\KeywordTok{grep}\NormalTok{(}\StringTok{'Analysis'}\NormalTok{, files)]}
-  \ControlFlowTok{for}\NormalTok{(file }\ControlFlowTok{in}\NormalTok{ files)\{}
-    
-    \ControlFlowTok{if}\NormalTok{(}\OperatorTok{!}\KeywordTok{dir.exists}\NormalTok{(}\KeywordTok{file.path}\NormalTok{(outputDirectory,}\KeywordTok{paste0}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\DecValTok{1000}\OperatorTok{*}\KeywordTok{as.double}\NormalTok{(}\KeywordTok{gsub}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\StringTok{''}\NormalTok{,file))}\OperatorTok{+}\NormalTok{i))))\{}
-      \KeywordTok{dir.create}\NormalTok{(}\KeywordTok{file.path}\NormalTok{(outputDirectory,}\KeywordTok{paste0}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\DecValTok{1000}\OperatorTok{*}\KeywordTok{as.double}\NormalTok{(}\KeywordTok{gsub}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\StringTok{''}\NormalTok{,file))}\OperatorTok{+}\NormalTok{i)))}
-\NormalTok{    \}}
-    
-    \ControlFlowTok{if}\NormalTok{(}\KeywordTok{dir.exists}\NormalTok{(}\KeywordTok{file.path}\NormalTok{(resultDirectory[i],file, }\StringTok{'plpResult'}\NormalTok{)))\{}
-\NormalTok{      res <-}\StringTok{ }\NormalTok{PatientLevelPrediction}\OperatorTok{::}\KeywordTok{loadPlpResult}\NormalTok{(}\KeywordTok{file.path}\NormalTok{(resultDirectory[i],file, }\StringTok{'plpResult'}\NormalTok{))}
-\NormalTok{      res <-}\StringTok{ }\NormalTok{PatientLevelPrediction}\OperatorTok{::}\KeywordTok{transportPlp}\NormalTok{(res, }\DataTypeTok{n=}\NormalTok{ minCellCount, }
-                                                  \DataTypeTok{save =}\NormalTok{ F, }\DataTypeTok{dataName =}\NormalTok{ databaseName[i])}
-      
-\NormalTok{      res}\OperatorTok{$}\NormalTok{covariateSummary <-}\StringTok{ }\NormalTok{res}\OperatorTok{$}\NormalTok{covariateSummary[res}\OperatorTok{$}\NormalTok{covariateSummary}\OperatorTok{$}\NormalTok{covariateValue}\OperatorTok{!=}\DecValTok{0}\NormalTok{,]}
-\NormalTok{      covSet <-}\StringTok{ }\NormalTok{res}\OperatorTok{$}\NormalTok{model}\OperatorTok{$}\NormalTok{metaData}\OperatorTok{$}\NormalTok{call}\OperatorTok{$}\NormalTok{covariateSettings}
-\NormalTok{      res}\OperatorTok{$}\NormalTok{model}\OperatorTok{$}\NormalTok{metaData <-}\StringTok{ }\OtherTok{NULL}
-\NormalTok{      res}\OperatorTok{$}\NormalTok{model}\OperatorTok{$}\NormalTok{metaData}\OperatorTok{$}\NormalTok{call}\OperatorTok{$}\NormalTok{covariateSettings <-}\StringTok{ }\NormalTok{covSet}
-\NormalTok{      res}\OperatorTok{$}\NormalTok{model}\OperatorTok{$}\NormalTok{predict <-}\StringTok{ }\OtherTok{NULL}
-      \ControlFlowTok{if}\NormalTok{(}\OperatorTok{!}\KeywordTok{is.null}\NormalTok{(res}\OperatorTok{$}\NormalTok{performanceEvaluation}\OperatorTok{$}\NormalTok{evaluationStatistics))\{}
-\NormalTok{      res}\OperatorTok{$}\NormalTok{performanceEvaluation}\OperatorTok{$}\NormalTok{evaluationStatistics[,}\DecValTok{1}\NormalTok{] <-}\StringTok{ }\KeywordTok{paste0}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\DecValTok{1000}\OperatorTok{*}\KeywordTok{as.double}\NormalTok{(}\KeywordTok{gsub}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\StringTok{''}\NormalTok{,file))}\OperatorTok{+}\NormalTok{i)}
-\NormalTok{      \} }\ControlFlowTok{else}\NormalTok{\{}
-        \KeywordTok{writeLines}\NormalTok{(}\KeywordTok{paste0}\NormalTok{(resultDirectory[i],file, }\StringTok{'-ev'}\NormalTok{))}
-\NormalTok{      \}}
-      \ControlFlowTok{if}\NormalTok{(}\OperatorTok{!}\KeywordTok{is.null}\NormalTok{(res}\OperatorTok{$}\NormalTok{performanceEvaluation}\OperatorTok{$}\NormalTok{thresholdSummary))\{}
-\NormalTok{      res}\OperatorTok{$}\NormalTok{performanceEvaluation}\OperatorTok{$}\NormalTok{thresholdSummary[,}\DecValTok{1}\NormalTok{] <-}\StringTok{ }\KeywordTok{paste0}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\DecValTok{1000}\OperatorTok{*}\KeywordTok{as.double}\NormalTok{(}\KeywordTok{gsub}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\StringTok{''}\NormalTok{,file))}\OperatorTok{+}\NormalTok{i)}
-\NormalTok{      \}}\ControlFlowTok{else}\NormalTok{\{}
-        \KeywordTok{writeLines}\NormalTok{(}\KeywordTok{paste0}\NormalTok{(resultDirectory[i],file, }\StringTok{'-thres'}\NormalTok{))}
-\NormalTok{      \}}
-      \ControlFlowTok{if}\NormalTok{(}\OperatorTok{!}\KeywordTok{is.null}\NormalTok{(res}\OperatorTok{$}\NormalTok{performanceEvaluation}\OperatorTok{$}\NormalTok{demographicSummary))\{}
-\NormalTok{      res}\OperatorTok{$}\NormalTok{performanceEvaluation}\OperatorTok{$}\NormalTok{demographicSummary[,}\DecValTok{1}\NormalTok{] <-}\StringTok{ }\KeywordTok{paste0}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\DecValTok{1000}\OperatorTok{*}\KeywordTok{as.double}\NormalTok{(}\KeywordTok{gsub}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\StringTok{''}\NormalTok{,file))}\OperatorTok{+}\NormalTok{i)}
-\NormalTok{      \} }\ControlFlowTok{else}\NormalTok{\{}
-        \KeywordTok{writeLines}\NormalTok{(}\KeywordTok{paste0}\NormalTok{(resultDirectory[i],file, }\StringTok{'-dem'}\NormalTok{))}
-\NormalTok{      \}}
-      \ControlFlowTok{if}\NormalTok{(}\OperatorTok{!}\KeywordTok{is.null}\NormalTok{(res}\OperatorTok{$}\NormalTok{performanceEvaluation}\OperatorTok{$}\NormalTok{calibrationSummary))\{}
-\NormalTok{      res}\OperatorTok{$}\NormalTok{performanceEvaluation}\OperatorTok{$}\NormalTok{calibrationSummary[,}\DecValTok{1}\NormalTok{] <-}\StringTok{ }\KeywordTok{paste0}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\DecValTok{1000}\OperatorTok{*}\KeywordTok{as.double}\NormalTok{(}\KeywordTok{gsub}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\StringTok{''}\NormalTok{,file))}\OperatorTok{+}\NormalTok{i)}
-\NormalTok{      \}}\ControlFlowTok{else}\NormalTok{\{}
-        \KeywordTok{writeLines}\NormalTok{(}\KeywordTok{paste0}\NormalTok{(resultDirectory[i],file, }\StringTok{'-cal'}\NormalTok{))}
-\NormalTok{      \}}
-      \ControlFlowTok{if}\NormalTok{(}\OperatorTok{!}\KeywordTok{is.null}\NormalTok{(res}\OperatorTok{$}\NormalTok{performanceEvaluation}\OperatorTok{$}\NormalTok{predictionDistribution))\{}
-\NormalTok{      res}\OperatorTok{$}\NormalTok{performanceEvaluation}\OperatorTok{$}\NormalTok{predictionDistribution[,}\DecValTok{1}\NormalTok{] <-}\StringTok{ }\KeywordTok{paste0}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\DecValTok{1000}\OperatorTok{*}\KeywordTok{as.double}\NormalTok{(}\KeywordTok{gsub}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\StringTok{''}\NormalTok{,file))}\OperatorTok{+}\NormalTok{i)}
-\NormalTok{      \}}\ControlFlowTok{else}\NormalTok{\{}
-        \KeywordTok{writeLines}\NormalTok{(}\KeywordTok{paste0}\NormalTok{(resultDirectory[i],file, }\StringTok{'-dist'}\NormalTok{))}
-\NormalTok{      \}}
-      \KeywordTok{saveRDS}\NormalTok{(res, }\KeywordTok{file.path}\NormalTok{(outputDirectory,}\KeywordTok{paste0}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\DecValTok{1000}\OperatorTok{*}\KeywordTok{as.double}\NormalTok{(}\KeywordTok{gsub}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\StringTok{''}\NormalTok{,file))}\OperatorTok{+}\NormalTok{i), }\StringTok{'plpResult.rds'}\NormalTok{))}
-\NormalTok{    \}}
-    \ControlFlowTok{if}\NormalTok{(}\KeywordTok{file.exists}\NormalTok{(}\KeywordTok{file.path}\NormalTok{(resultDirectory[i],file, }\StringTok{'plpLog.txt'}\NormalTok{)))\{}
-      \KeywordTok{file.copy}\NormalTok{(}\DataTypeTok{from =} \KeywordTok{file.path}\NormalTok{(resultDirectory[i],file, }\StringTok{'plpLog.txt'}\NormalTok{), }
-                \DataTypeTok{to =} \KeywordTok{file.path}\NormalTok{(outputDirectory,}\KeywordTok{paste0}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\DecValTok{1000}\OperatorTok{*}\KeywordTok{as.double}\NormalTok{(}\KeywordTok{gsub}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\StringTok{''}\NormalTok{,file))}\OperatorTok{+}\NormalTok{i), }\StringTok{'plpLog.txt'}\NormalTok{))}
-\NormalTok{    \}}
-\NormalTok{  \}}
-\NormalTok{  \}}
-  
-  
-  
-  \ControlFlowTok{for}\NormalTok{(i }\ControlFlowTok{in} \DecValTok{1}\OperatorTok{:}\KeywordTok{length}\NormalTok{(resultDirectory))\{}
-  \CommentTok{# copy any validation results}
-  \ControlFlowTok{if}\NormalTok{(}\KeywordTok{dir.exists}\NormalTok{(}\KeywordTok{file.path}\NormalTok{(resultDirectory[i],}\StringTok{'Validation'}\NormalTok{)))\{}
-\NormalTok{    valFolders <-}\StringTok{  }\KeywordTok{dir}\NormalTok{(}\KeywordTok{file.path}\NormalTok{(resultDirectory[i],}\StringTok{'Validation'}\NormalTok{), }\DataTypeTok{full.names =}\NormalTok{ F)}
-    
-    \ControlFlowTok{if}\NormalTok{(}\KeywordTok{length}\NormalTok{(valFolders)}\OperatorTok{>}\DecValTok{0}\NormalTok{)\{}
-      \CommentTok{# move each of the validation rds}
-      \ControlFlowTok{for}\NormalTok{(valFolder }\ControlFlowTok{in}\NormalTok{ valFolders)\{}
-        
-        \CommentTok{# get the analysisIds}
-\NormalTok{        valSubfolders <-}\StringTok{ }\KeywordTok{dir}\NormalTok{(}\KeywordTok{file.path}\NormalTok{(resultDirectory[i],}\StringTok{'Validation'}\NormalTok{,valFolder), }\DataTypeTok{full.names =}\NormalTok{ F)}
-        \ControlFlowTok{if}\NormalTok{(}\KeywordTok{length}\NormalTok{(valSubfolders)}\OperatorTok{!=}\DecValTok{0}\NormalTok{)\{}
-          \ControlFlowTok{for}\NormalTok{(valSubfolder }\ControlFlowTok{in}\NormalTok{ valSubfolders )\{}
-\NormalTok{            valSubfolderUpdate <-}\StringTok{ }\KeywordTok{paste0}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{, }\KeywordTok{as.double}\NormalTok{(}\KeywordTok{gsub}\NormalTok{(}\StringTok{'Analysis_'}\NormalTok{,}\StringTok{''}\NormalTok{, valSubfolder))}\OperatorTok{*}\DecValTok{1000}\OperatorTok{+}\NormalTok{i)}
-\NormalTok{            valOut <-}\StringTok{ }\KeywordTok{file.path}\NormalTok{(valFolder,valSubfolderUpdate)}
-\NormalTok{            valOutOld <-}\StringTok{ }\KeywordTok{file.path}\NormalTok{(valFolder,valSubfolder)}
-            \ControlFlowTok{if}\NormalTok{(}\OperatorTok{!}\KeywordTok{dir.exists}\NormalTok{(}\KeywordTok{file.path}\NormalTok{(outputDirectory,}\StringTok{'Validation'}\NormalTok{,valOut)))\{}
-              \KeywordTok{dir.create}\NormalTok{(}\KeywordTok{file.path}\NormalTok{(outputDirectory,}\StringTok{'Validation'}\NormalTok{,valOut), }\DataTypeTok{recursive =}\NormalTok{ T)}
-\NormalTok{            \}}
-            
-            
-            \ControlFlowTok{if}\NormalTok{(}\KeywordTok{file.exists}\NormalTok{(}\KeywordTok{file.path}\NormalTok{(resultDirectory[i],}\StringTok{'Validation'}\NormalTok{,valOutOld, }\StringTok{'validationResult.rds'}\NormalTok{)))\{}
-\NormalTok{              res <-}\StringTok{ }\KeywordTok{readRDS}\NormalTok{(}\KeywordTok{file.path}\NormalTok{(resultDirectory[i],}\StringTok{'Validation'}\NormalTok{,valOutOld, }\StringTok{'validationResult.rds'}\NormalTok{))}
-\NormalTok{              res <-}\StringTok{ }\NormalTok{PatientLevelPrediction}\OperatorTok{::}\KeywordTok{transportPlp}\NormalTok{(res, }\DataTypeTok{n=}\NormalTok{ minCellCount, }
-                                                          \DataTypeTok{save =}\NormalTok{ F, }\DataTypeTok{dataName =}\NormalTok{ databaseName[i])}
-\NormalTok{              res}\OperatorTok{$}\NormalTok{covariateSummary <-}\StringTok{ }\NormalTok{res}\OperatorTok{$}\NormalTok{covariateSummary[res}\OperatorTok{$}\NormalTok{covariateSummary}\OperatorTok{$}\NormalTok{covariateValue}\OperatorTok{!=}\DecValTok{0}\NormalTok{,]}
-              \KeywordTok{saveRDS}\NormalTok{(res, }\KeywordTok{file.path}\NormalTok{(outputDirectory,}\StringTok{'Validation'}\NormalTok{,valOut, }\StringTok{'validationResult.rds'}\NormalTok{))}
-\NormalTok{            \}}
-\NormalTok{          \}}
-\NormalTok{        \}}
-        
-\NormalTok{      \}}
-      
-\NormalTok{    \}}
-    
-\NormalTok{  \}}
-\NormalTok{  \}}
-  
-  \KeywordTok{return}\NormalTok{(outputDirectory)}
-  
-\NormalTok{\}}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{example-code-to-combine-multiple-results}{%
-\subsection{Example code to combine multiple
-results}\label{example-code-to-combine-multiple-results}}
-
-The following code will combine the results found in `C:/myResults',
-`C:/myResults2' and `C:/myResults3' into the shiny project at
-`C:/R/library/myPackage/shiny/PLPViewer':
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{populateMultipleShinyApp}\NormalTok{(}\DataTypeTok{shinyDirectory =} \StringTok{'C:/R/library/myPackage/shiny/PLPViewer'}\NormalTok{,}
-                                     \DataTypeTok{resultDirectory =} \KeywordTok{c}\NormalTok{(}\StringTok{'C:/myResults'}\NormalTok{,}
-                                                         \StringTok{'C:/myResults2'}\NormalTok{,}
-                                                         \StringTok{'C:/myResults3'}\NormalTok{),}
-                                     \DataTypeTok{minCellCount =} \DecValTok{0}\NormalTok{,}
-                                     \DataTypeTok{databaseName =} \KeywordTok{c}\NormalTok{(}\StringTok{'database1'}\NormalTok{,}\StringTok{'database2'}\NormalTok{,}\StringTok{'database3'}\NormalTok{))}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{manual-app-creation}{%
-\section{Manual App Creation}\label{manual-app-creation}}
-
-{[}instructions coming soon{]}
-
-\hypertarget{acknowledgments}{%
-\section{Acknowledgments}\label{acknowledgments}}
-
-Considerable work has been dedicated to provide the
-\texttt{PatientLevelPrediction} package.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{citation}\NormalTok{(}\StringTok{"PatientLevelPrediction"}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-\begin{verbatim}
-## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design and
-## implementation of a standardized framework to generate and evaluate patient-level
-## prediction models using observational healthcare data." _Journal of the American
-## Medical Informatics Association_, *25*(8), 969-975. <URL:
-## https://doi.org/10.1093/jamia/ocy032>.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }
-\end{verbatim}
-
-\textbf{Please reference this paper if you use the PLP Package in your
-work:}
-
-\href{http://dx.doi.org/10.1093/jamia/ocy032}{Reps JM, Schuemie MJ,
-Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a
-standardized framework to generate and evaluate patient-level prediction
-models using observational healthcare data. J Am Med Inform Assoc.
-2018;25(8):969-975.}
-
-\end{document}
diff --git a/inst/doc/Figure1.png b/inst/doc/Figure1.png
deleted file mode 100644
index 878a509e2..000000000
Binary files a/inst/doc/Figure1.png and /dev/null differ
diff --git a/inst/doc/InstallationGuide.pdf b/inst/doc/InstallationGuide.pdf
index 4734063a1..605054361 100644
Binary files a/inst/doc/InstallationGuide.pdf and b/inst/doc/InstallationGuide.pdf differ
diff --git a/inst/doc/InstallationGuide.tex b/inst/doc/InstallationGuide.tex
deleted file mode 100644
index 937f4c3e7..000000000
--- a/inst/doc/InstallationGuide.tex
+++ /dev/null
@@ -1,369 +0,0 @@
-% Options for packages loaded elsewhere
-\PassOptionsToPackage{unicode}{hyperref}
-\PassOptionsToPackage{hyphens}{url}
-%
-\documentclass[
-]{article}
-\usepackage{lmodern}
-\usepackage{amssymb,amsmath}
-\usepackage{ifxetex,ifluatex}
-\ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex
-  \usepackage[T1]{fontenc}
-  \usepackage[utf8]{inputenc}
-  \usepackage{textcomp} % provide euro and other symbols
-\else % if luatex or xetex
-  \usepackage{unicode-math}
-  \defaultfontfeatures{Scale=MatchLowercase}
-  \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
-\fi
-% Use upquote if available, for straight quotes in verbatim environments
-\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
-\IfFileExists{microtype.sty}{% use microtype if available
-  \usepackage[]{microtype}
-  \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
-}{}
-\makeatletter
-\@ifundefined{KOMAClassName}{% if non-KOMA class
-  \IfFileExists{parskip.sty}{%
-    \usepackage{parskip}
-  }{% else
-    \setlength{\parindent}{0pt}
-    \setlength{\parskip}{6pt plus 2pt minus 1pt}}
-}{% if KOMA class
-  \KOMAoptions{parskip=half}}
-\makeatother
-\usepackage{xcolor}
-\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
-\IfFileExists{bookmark.sty}{\usepackage{bookmark}}{\usepackage{hyperref}}
-\hypersetup{
-  pdftitle={Patient-Level Prediction Installation Guide},
-  pdfauthor={Jenna Reps, Peter R. Rijnbeek},
-  hidelinks,
-  pdfcreator={LaTeX via pandoc}}
-\urlstyle{same} % disable monospaced font for URLs
-\usepackage[margin=1in]{geometry}
-\usepackage{color}
-\usepackage{fancyvrb}
-\newcommand{\VerbBar}{|}
-\newcommand{\VERB}{\Verb[commandchars=\\\{\}]}
-\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
-% Add ',fontsize=\small' for more characters per line
-\usepackage{framed}
-\definecolor{shadecolor}{RGB}{248,248,248}
-\newenvironment{Shaded}{\begin{snugshade}}{\end{snugshade}}
-\newcommand{\AlertTok}[1]{\textcolor[rgb]{0.94,0.16,0.16}{#1}}
-\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.77,0.63,0.00}{#1}}
-\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\BuiltInTok}[1]{#1}
-\newcommand{\CharTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
-\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
-\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{#1}}
-\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\ErrorTok}[1]{\textcolor[rgb]{0.64,0.00,0.00}{\textbf{#1}}}
-\newcommand{\ExtensionTok}[1]{#1}
-\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
-\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\ImportTok}[1]{#1}
-\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
-\newcommand{\NormalTok}[1]{#1}
-\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.81,0.36,0.00}{\textbf{#1}}}
-\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{#1}}
-\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
-\newcommand{\RegionMarkerTok}[1]{#1}
-\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\StringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
-\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
-\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
-\usepackage{graphicx,grffile}
-\makeatletter
-\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
-\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
-\makeatother
-% Scale images if necessary, so that they will not overflow the page
-% margins by default, and it is still possible to overwrite the defaults
-% using explicit options in \includegraphics[width, height, ...]{}
-\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
-% Set default figure placement to htbp
-\makeatletter
-\def\fps@figure{htbp}
-\makeatother
-\setlength{\emergencystretch}{3em} % prevent overfull lines
-\providecommand{\tightlist}{%
-  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
-\setcounter{secnumdepth}{5}
-\usepackage{fancyhdr}
-\pagestyle{fancy}
-\fancyhead{}
-\fancyhead[CO,CE]{Installation Guide}
-\fancyfoot[CO,CE]{PatientLevelPrediction Package Version 3.1.0}
-\fancyfoot[LE,RO]{\thepage}
-\renewcommand{\headrulewidth}{0.4pt}
-\renewcommand{\footrulewidth}{0.4pt}
-
-\title{Patient-Level Prediction Installation Guide}
-\author{Jenna Reps, Peter R. Rijnbeek}
-\date{2020-06-03}
-
-\begin{document}
-\maketitle
-
-{
-\setcounter{tocdepth}{2}
-\tableofcontents
-}
-\hypertarget{introduction}{%
-\section{Introduction}\label{introduction}}
-
-This vignette describes how you need to install the Observational Health
-Data Sciencs and Informatics (OHDSI)
-\href{http://github.com/OHDSI/PatientLevelPrediction}{\texttt{PatientLevelPrediction}}
-package under Windows, Mac, and Linux.
-
-\hypertarget{software-prerequisites}{%
-\section{Software Prerequisites}\label{software-prerequisites}}
-
-\hypertarget{windows-users}{%
-\subsection{Windows Users}\label{windows-users}}
-
-Under Windows the OHDSI Patient Level Prediction (PLP) package requires
-installing:
-
-\begin{itemize}
-\tightlist
-\item
-  R (\url{https://cran.cnr.berkeley.edu/} ) - (R \textgreater= 3.3.0,
-  but latest is recommended)
-\item
-  Rstudio (\url{https://www.rstudio.com/} )
-\item
-  Java (\url{http://www.java.com} )
-\item
-  RTools (\url{https://cran.r-project.org/bin/windows/Rtools/})
-\end{itemize}
-
-\hypertarget{maclinux-users}{%
-\subsection{Mac/Linux Users}\label{maclinux-users}}
-
-Under Mac and Linux the OHDSI Patient Level Prediction (PLP) package
-requires installing:
-
-\begin{itemize}
-\tightlist
-\item
-  R (\url{https://cran.cnr.berkeley.edu/} ) - (R \textgreater= 3.3.0,
-  but latest is recommended)
-\item
-  Rstudio (\url{https://www.rstudio.com/} )
-\item
-  Java (\url{http://www.java.com} )
-\item
-  Xcode command line tools(run in terminal: xcode-select --install)
-  {[}MAC USERS ONLY{]}
-\end{itemize}
-
-\hypertarget{installing-the-package}{%
-\section{Installing the Package}\label{installing-the-package}}
-
-The preferred way to install the package is by using drat, which will
-automatically install the latest release and all the latest
-dependencies. If the drat code fails or you do not want the official
-release you could use devtools to install the bleading edge version of
-the package (latest master). Note that the latest master could contain
-bugs, please report them to us if you experience problems.
-
-\hypertarget{installing-patientlevelprediction-using-drat}{%
-\subsection{Installing PatientLevelPrediction using
-drat}\label{installing-patientlevelprediction-using-drat}}
-
-To install using drat run:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{install.packages}\NormalTok{(}\StringTok{"drat"}\NormalTok{)}
-\NormalTok{drat}\OperatorTok{::}\KeywordTok{addRepo}\NormalTok{(}\StringTok{"OHDSI"}\NormalTok{)}
-\KeywordTok{install.packages}\NormalTok{(}\StringTok{"PatientLevelPrediction"}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{installing-patientlevelprediction-using-devtools}{%
-\subsection{Installing PatientLevelPrediction using
-devtools}\label{installing-patientlevelprediction-using-devtools}}
-
-To install using devtools run:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{install.packages}\NormalTok{(}\StringTok{'devtools'}\NormalTok{)}
-\NormalTok{devtools}\OperatorTok{::}\KeywordTok{install_github}\NormalTok{(}\StringTok{"OHDSI/FeatureExtraction"}\NormalTok{)}
-\NormalTok{devtools}\OperatorTok{::}\KeywordTok{install_github}\NormalTok{(}\StringTok{'ohdsi/PatientLevelPrediction'}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-When installing using devtools make sure to close any other Rstudio
-sessions that are using PatientLevelPrediction or any dependency.
-Keeping Rstudio sessions open can cause locks that prevent the package
-installing.
-
-\hypertarget{creating-python-reticulate-environment}{%
-\section{Creating Python Reticulate
-Environment}\label{creating-python-reticulate-environment}}
-
-Many of the classifiers in the PatientLevelPrediction use a Python back
-end. To set up a python environment run:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{library}\NormalTok{(PatientLevelPrediction)}
-\NormalTok{reticulate}\OperatorTok{::}\KeywordTok{install_miniconda}\NormalTok{()}
-\KeywordTok{configurePython}\NormalTok{(}\DataTypeTok{envname=}\StringTok{'r-reticulate'}\NormalTok{, }\DataTypeTok{envtype=}\StringTok{'conda'}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-To add the R keras interface, in Rstudio run:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{devtools}\OperatorTok{::}\KeywordTok{install_github}\NormalTok{(}\StringTok{"rstudio/keras"}\NormalTok{)}
-\KeywordTok{library}\NormalTok{(keras)}
-\KeywordTok{install_keras}\NormalTok{()}
-\end{Highlighting}
-\end{Shaded}
-
-Some of the less frequently used classifiers are not installed during
-this set-up to add them run:
-
-For GBM survival:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{reticulate}\OperatorTok{::}\KeywordTok{conda_install}\NormalTok{(}\DataTypeTok{envname=}\StringTok{'r-reticulate'}\NormalTok{, }\DataTypeTok{packages =} \KeywordTok{c}\NormalTok{(}\StringTok{'scikit-survival'}\NormalTok{), }\DataTypeTok{forge =} \OtherTok{TRUE}\NormalTok{, }\DataTypeTok{pip =} \OtherTok{FALSE}\NormalTok{, }\DataTypeTok{pip_ignore_installed =} \OtherTok{TRUE}\NormalTok{, }\DataTypeTok{conda =} \StringTok{"auto"}\NormalTok{, }\DataTypeTok{channel =} \StringTok{'sebp'}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-For any of the torch models:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\NormalTok{reticulate}\OperatorTok{::}\KeywordTok{conda_install}\NormalTok{(}\DataTypeTok{envname=}\StringTok{'r-reticulate'}\NormalTok{, }\DataTypeTok{packages =} \KeywordTok{c}\NormalTok{(}\StringTok{'pytorch'}\NormalTok{, }\StringTok{'torchvision'}\NormalTok{, }\StringTok{'cpuonly'}\NormalTok{), }\DataTypeTok{forge =} \OtherTok{TRUE}\NormalTok{, }\DataTypeTok{pip =} \OtherTok{FALSE}\NormalTok{, }\DataTypeTok{channel =} \StringTok{'pytorch'}\NormalTok{, }\DataTypeTok{pip_ignore_installed =} \OtherTok{TRUE}\NormalTok{, }\DataTypeTok{conda =} \StringTok{'auto'}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-\hypertarget{testing-installation}{%
-\section{Testing installation}\label{testing-installation}}
-
-To test whether the package is installed correctly run:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{library}\NormalTok{(DatabaseConnector)}
-\NormalTok{connectionDetails <-}\StringTok{ }\KeywordTok{createConnectionDetails}\NormalTok{(}\DataTypeTok{dbms =} \StringTok{'sql_server'}\NormalTok{, }
-                                             \DataTypeTok{user =} \StringTok{'username'}\NormalTok{, }
-                                             \DataTypeTok{password =} \StringTok{'hidden'}\NormalTok{, }
-                                             \DataTypeTok{server =} \StringTok{'your server'}\NormalTok{, }
-                                             \DataTypeTok{port =} \StringTok{'your port'}\NormalTok{)}
-\NormalTok{PatientLevelPrediction}\OperatorTok{::}\KeywordTok{checkPlpInstallation}\NormalTok{(}\DataTypeTok{connectionDetails =}\NormalTok{ connectionDetails, }
-                                             \DataTypeTok{python =}\NormalTok{ T)}
-\end{Highlighting}
-\end{Shaded}
-
-To test the installation (excluding python) run:
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{library}\NormalTok{(DatabaseConnector)}
-\NormalTok{connectionDetails <-}\StringTok{ }\KeywordTok{createConnectionDetails}\NormalTok{(}\DataTypeTok{dbms =} \StringTok{'sql_server'}\NormalTok{, }
-                                           \DataTypeTok{user =} \StringTok{'username'}\NormalTok{, }
-                                           \DataTypeTok{password =} \StringTok{'hidden'}\NormalTok{, }
-                                           \DataTypeTok{server =} \StringTok{'your server'}\NormalTok{, }
-                                           \DataTypeTok{port =} \StringTok{'your port'}\NormalTok{)}
-\NormalTok{PatientLevelPrediction}\OperatorTok{::}\KeywordTok{checkPlpInstallation}\NormalTok{(}\DataTypeTok{connectionDetails =}\NormalTok{ connectionDetails, }
-                                             \DataTypeTok{python =}\NormalTok{ F)}
-\end{Highlighting}
-\end{Shaded}
-
-The check can take a while to run since it will build the following
-models in sequence on simulated \url{data:Logistic} Regression,
-RandomForest, MLP, AdaBoost, Decision Tree, Naive Bayes, KNN, Gradient
-Boosting. Moreover, it will test the database connection.
-
-\hypertarget{installation-issues}{%
-\section{Installation issues}\label{installation-issues}}
-
-Installation issues need to be posted in our issue tracker:
-\url{http://github.com/OHDSI/PatientLevelPrediction/issues}
-
-The list below provides solutions for some common issues:
-
-\begin{enumerate}
-\def\labelenumi{\arabic{enumi}.}
-\item
-  If you have an error when trying to install a package in R saying
-  \textbf{`Dependancy X not available \ldots{}'} then this can sometimes
-  be fixed by running
-  \texttt{install.packages(\textquotesingle{}X\textquotesingle{})} and
-  then once that completes trying to reinstall the package that had the
-  error.
-\item
-  I have found that using the github devtools to install packages can be
-  impacted if you have \textbf{multiple R sessions} open as one session
-  with a library open can causethe library to be locked and this can
-  prevent an install of a package that depends on that library.
-\end{enumerate}
-
-\hypertarget{acknowledgments}{%
-\section{Acknowledgments}\label{acknowledgments}}
-
-Considerable work has been dedicated to provide the
-\texttt{PatientLevelPrediction} package.
-
-\begin{Shaded}
-\begin{Highlighting}[]
-\KeywordTok{citation}\NormalTok{(}\StringTok{"PatientLevelPrediction"}\NormalTok{)}
-\end{Highlighting}
-\end{Shaded}
-
-\begin{verbatim}
-## 
-## To cite PatientLevelPrediction in publications use:
-## 
-## Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek P (2018). "Design and
-## implementation of a standardized framework to generate and evaluate patient-level
-## prediction models using observational healthcare data." _Journal of the American
-## Medical Informatics Association_, *25*(8), 969-975. <URL:
-## https://doi.org/10.1093/jamia/ocy032>.
-## 
-## A BibTeX entry for LaTeX users is
-## 
-##   @Article{,
-##     author = {J. M. Reps and M. J. Schuemie and M. A. Suchard and P. B. Ryan and P. Rijnbeek},
-##     title = {Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data},
-##     journal = {Journal of the American Medical Informatics Association},
-##     volume = {25},
-##     number = {8},
-##     pages = {969-975},
-##     year = {2018},
-##     url = {https://doi.org/10.1093/jamia/ocy032},
-##   }
-\end{verbatim}
-
-\textbf{Please reference this paper if you use the PLP Package in your
-work:}
-
-\href{http://dx.doi.org/10.1093/jamia/ocy032}{Reps JM, Schuemie MJ,
-Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a
-standardized framework to generate and evaluate patient-level prediction
-models using observational healthcare data. J Am Med Inform Assoc.
-2018;25(8):969-975.}
-
-This work is supported in part through the National Science Foundation
-grant IIS 1251151.
-
-\end{document}
diff --git a/inst/settings/rEnvironmentSnapshot.csv b/inst/settings/rEnvironmentSnapshot.csv
deleted file mode 100644
index 269bb3316..000000000
--- a/inst/settings/rEnvironmentSnapshot.csv
+++ /dev/null
@@ -1,115 +0,0 @@
-"package","version"
-"R","3.5.0"
-"grDevices","3.5.0"
-"graphics","3.5.0"
-"utils","3.5.0"
-"stats","3.5.0"
-"methods","3.5.0"
-"tools","3.5.0"
-"assertthat","0.2.0"
-"crayon","1.3.4"
-"cli","1.0.0"
-"rlang","0.2.0"
-"utf8","1.1.3"
-"grid","3.5.0"
-"pillar","1.2.2"
-"Rcpp","0.12.19"
-"colorspace","1.3-2"
-"glue","1.2.0"
-"later","0.7.2"
-"lattice","0.20-35"
-"magrittr","1.5"
-"R6","2.2.2"
-"stringi","1.1.7"
-"tibble","1.4.2"
-"bindr","0.1.1"
-"bit","1.1-14"
-"dichromat","2.0-0"
-"digest","0.6.15"
-"jsonlite","1.5"
-"labeling","0.3"
-"Matrix","1.2-14"
-"munsell","0.4.3"
-"plyr","1.8.4"
-"promises","1.0.1"
-"purrr","0.2.4"
-"RColorBrewer","1.1-2"
-"stringr","1.3.1"
-"viridisLite","0.3.0"
-"yaml","2.1.19"
-"base64enc","0.1-3"
-"bindrcpp","0.2.2"
-"config","0.3"
-"debugme","1.1.0"
-"fastmatch","1.1-0"
-"ff","2.2-14"
-"gtable","0.2.0"
-"htmltools","0.3.6"
-"httpuv","1.4.3"
-"lazyeval","0.2.1"
-"MASS","7.3-49"
-"mime","0.5"
-"pkgconfig","2.0.1"
-"reshape2","1.4.3"
-"reticulate","1.7"
-"rJava","0.9-10"
-"rstudioapi","0.7"
-"scales","0.5.0"
-"sourcetools","0.1.7"
-"tidyselect","0.2.4"
-"triebeard","0.3.0"
-"whisker","0.3-2"
-"xtable","1.8-2"
-"codetools","0.2-15"
-"compiler","3.5.0"
-"curl","3.2"
-"DatabaseConnectorJars","1.0.0"
-"DBI","1.0.0"
-"dplyr","0.7.4"
-"ffbase","0.12.7"
-"ggplot2","2.2.1"
-"iterators","1.0.10"
-"openssl","1.0.1"
-"processx","3.0.3"
-"shiny","1.0.5"
-"splines","3.5.0"
-"SqlRender","1.6.0"
-"tfruns","1.3"
-"urltools","1.7.0"
-"bitops","1.0-6"
-"crosstalk","1.0.0"
-"data.table","1.11.2"
-"DatabaseConnector","2.2.1"
-"foreach","1.4.4"
-"hexbin","1.27.2"
-"htmlwidgets","1.2"
-"httr","1.3.1"
-"parallel","3.5.0"
-"RcppParallel","4.4.0"
-"shape","1.4.4"
-"snow","0.4-2"
-"survival","2.41-3"
-"tensorflow","1.5"
-"tidyr","0.8.0"
-"uuid","0.1-2"
-"XML","3.98-1.13"
-"xml2","1.2.0"
-"zeallot","0.1.0"
-"zip","1.0.0"
-"AUC","0.3.0"
-"Cyclops","1.3.2"
-"diagram","1.6.4"
-"doParallel","1.0.11"
-"FeatureExtraction","2.2.1"
-"gridExtra","2.3"
-"keras","2.1.6"
-"officer","0.3.1"
-"ParallelLogger","1.1.0"
-"plotly","4.7.1"
-"PRROC","1.3.1"
-"RCurl","1.95-4.10"
-"RJSONIO","1.3-0"
-"slam","0.1-43"
-"survAUC","1.0-5"
-"xgboost","0.6.4.1"
-"PatientLevelPrediction","3.0.0"
diff --git a/inst/settings/resultsDataModelSpecification.csv b/inst/settings/resultsDataModelSpecification.csv
new file mode 100644
index 000000000..21f196bb2
--- /dev/null
+++ b/inst/settings/resultsDataModelSpecification.csv
@@ -0,0 +1,200 @@
+table_name,column_name,data_type,is_required,primary_key,empty_is_na,mincellcount,description
+cohorts,cohort_id,int,Yes,Yes,No,No,a unique identifier for the cohort in the plp results database
+cohorts,cohort_definition_id,bigint,Yes,No,No,No,the identifier in ATLAS for the cohort
+cohorts,cohort_name,varchar,Yes,No,No,No,the name of the cohort
+cohort_definition,cohort_definition_id,bigint,Yes,No,No,No,The ATLAS cohort definition id
+cohort_definition,cohort_name,varchar,Yes,No,No,No,The name of the cohort
+cohort_definition,description,text,No,No,Yes,No,A description of the cohort
+cohort_definition,json,text,Yes,No,No,No,The json spec for the cohort
+cohort_definition,sql_command,text,No,No,Yes,No,The SQL used to create the cohort
+database_meta_data,database_id,varchar,Yes,Yes,Yes,No,The shared databaseId
+database_meta_data,cdm_source_name,varchar,Yes,No,Yes,No,The name of the database
+database_meta_data,cdm_source_abbreviation,varchar,Yes,No,Yes,No,The abbreviated name of the database
+database_meta_data,cdm_holder,varchar,No,No,Yes,No,The owner of the CDM
+database_meta_data,source_description,text,No,No,Yes,No,The full description of the database
+database_meta_data,source_documentation_reference,varchar,No,No,Yes,No,The link to the documentation
+database_meta_data,cdm_etl_reference,varchar,No,No,Yes,No,The link to the ETL document
+database_meta_data,source_release_date,varchar,No,No,Yes,No,The release date for the data
+database_meta_data,cdm_release_date,varchar,No,No,Yes,No,The release date for the CDM data
+database_meta_data,cdm_version,varchar,No,No,Yes,No,The vocabulary version
+database_meta_data,vocabulary_version,varchar,No,No,Yes,No,The max date in the database
+database_meta_data,max_obs_period_end_date,varchar,No,No,Yes,No,
+database_details,database_id,int,Yes,Yes,No,No,a unique identifier for the database
+database_details,database_meta_data_id,varchar,Yes,No,No,No,The shared databaseId
+tars,tar_id,int,Yes,Yes,No,No,a unique identifier for the tar
+tars,tar_start_day,int,Yes,No,No,No,the number of days offset the tar_start_anchor for the time-at-risk to start
+tars,tar_start_anchor,varchar,Yes,No,No,No,whether to use the cohort start or cohort end
+tars,tar_end_day,int,Yes,No,No,No,the number of days offset the tar_end_anchor for the time-at-risk to end
+tars,tar_end_anchor,varchar,Yes,No,No,No,whether to use the cohort start or cohort end
+population_settings,population_setting_id,int,Yes,Yes,No,No,a unique identifier for the population settings
+population_settings,population_settings_json,text,Yes,No,No,No,the json with the settings
+covariate_settings,covariate_setting_id,int,Yes,Yes,No,No,a unique identifier for the covaraite settings
+covariate_settings,covariate_settings_json,text,Yes,No,No,No,the json with the settings
+model_settings,model_setting_id,int,Yes,Yes,No,No,a unique identifier for the model settings
+model_settings,model_type,varchar,No,No,No,No,the type of model 
+model_settings,model_settings_json,varchar,Yes,No,No,No,the json with the settings
+split_settings,split_setting_id,int,Yes,Yes,No,No,a unique identifier for the split settings
+split_settings,split_settings_json,text,Yes,No,No,No,the json with the settings
+plp_data_settings,plp_data_setting_id,int,Yes,Yes,No,No,a unique identifier for the plp data settings
+plp_data_settings,plp_data_settings_json,text,Yes,No,No,No,the json with the settings
+feature_engineering_settings,feature_engineering_setting_id,int,Yes,Yes,No,No,a unique identifier for the feature engineering settings
+feature_engineering_settings,feature_engineering_settings_json,text,Yes,No,No,No,the json with the settings
+tidy_covariates_settings,tidy_covariates_setting_id,int,Yes,Yes,No,No,a unique identifier for the tidy covariates settings
+tidy_covariates_settings,tidy_covariates_settings_json,text,Yes,No,No,No,the json with the settings
+sample_settings,sample_setting_id,int,Yes,Yes,No,No,a unique identifier for the sample settings
+sample_settings,sample_settings_json,text,Yes,No,No,No,the json with the settings
+model_designs,model_design_id,int,Yes,Yes,No,No,a unique identifier for the model design settings
+model_designs,target_id,int,Yes,No,No,No,the  identifier for the target cohort id
+model_designs,outcome_id,int,Yes,No,No,No,the  identifier for the outcome cohort id
+model_designs,tar_id,int,Yes,No,No,No,the  identifier for the time at risk setting
+model_designs,plp_data_setting_id,int,Yes,No,No,No,the  identifier for the plp data setting
+model_designs,population_setting_id,int,Yes,No,No,No,the  identifier for the population setting
+model_designs,model_setting_id,int,Yes,No,No,No,the  identifier for the model setting
+model_designs,covariate_setting_id,int,Yes,No,No,No,the  identifier for the covaraite setting
+model_designs,sample_setting_id,int,Yes,No,No,No,the  identifier for the sample setting
+model_designs,split_setting_id,int,Yes,No,No,No,the  identifier for the split setting
+model_designs,feature_engineering_setting_id,int,Yes,No,No,No,the  identifier for the feature engineering setting
+model_designs,tidy_covariates_setting_id,int,Yes,No,No,No,the  identifier for the tidy covariate setting
+diagnostics,diagnostic_id,int,Yes,Yes,No,No,the unique identifier for the diagnostic results
+diagnostics,model_design_id,int,Yes,No,No,No,the identifier for the model design
+diagnostics,database_id,int,Yes,No,No,No,the identifier for the database
+diagnostics,execution_date_time,varchar,No,No,No,No,the date/time the diagnostic was run
+diagnostic_summary,diagnostic_id,int,Yes,No,No,No,the identifier for the diagnostics
+diagnostic_summary,probast_id,varchar,Yes,No,No,No,the probast id being diagnosed
+diagnostic_summary,result_value,varchar,Yes,No,No,No,the diagnostic result
+diagnostic_predictors,diagnostic_id,int,Yes,No,No,No,the identifier for the diagnostics
+diagnostic_predictors,days_to_event,int,Yes,No,No,No,the time between index to the day of interest
+diagnostic_predictors,outcome_at_time,int,Yes,No,No,Yes,the number of outcomes on the day of interest
+diagnostic_predictors,observed_at_start_of_day,bigint,Yes,No,No,Yes,the number of people observed up to the day of interest
+diagnostic_predictors,input_type,varchar,Yes,No,No,No,the setting id the results are for
+diagnostic_participants,diagnostic_id,int,Yes,No,No,No,the identifier for the diagnostics
+diagnostic_participants,design,varchar,Yes,No,No,No,the inclusion criteria of interest
+diagnostic_participants,metric,varchar,Yes,No,No,No,the metric calculated
+diagnostic_participants,value,float,Yes,No,No,No,the value calculated
+diagnostic_participants,probast_id,varchar,Yes,No,No,No,the corresponding probast id
+diagnostic_outcomes,diagnostic_id,int,Yes,No,No,No,the identifier for the diagnostics
+diagnostic_outcomes,xvalue,int,Yes,No,No,No,the value for the x-axis
+diagnostic_outcomes,outcome_percent,float,Yes,No,No,No,the percentage of people with the outcome
+diagnostic_outcomes,aggregation,varchar,Yes,No,No,No,"the type of aggregation (age,sex, year)"
+diagnostic_outcomes,probast_id,varchar,Yes,No,No,No,the corresponding probast id
+diagnostic_outcomes,input_type,varchar,Yes,No,No,No,the inclusion criteria of interest
+diagnostic_designs,diagnostic_id,int,Yes,Yes,No,No,the identifier for the diagnostics
+diagnostic_designs,probast_id,varchar,Yes,No,No,No,not used
+diagnostic_designs,value,varchar,Yes,No,No,No,not used
+models,model_id,int,Yes,Yes,No,No,A unique identifier for the model
+models,analysis_id,varchar,No,No,No,No,The analysis id from the model
+models,model_design_id,int,Yes,No,No,No,The corresponding model design id
+models,database_id,int,Yes,No,No,No,The corresponding database id
+models,model_type,varchar,No,No,No,No,The type of model 
+models,plp_model_file,text,Yes,No,No,No,A directory where the model is saved
+models,train_details,text,Yes,No,No,No,json containing the training details
+models,preprocessing,text,No,No,No,No,json containing the preprocessing details
+models,execution_date_time,varchar,No,No,No,No,the date/time the model was trained
+models,training_time,varchar,No,No,No,No,the time it took to develop the model
+models,intercept,float,No,No,No,No,the intercept (if the model is a GLM)
+recalibrations,recalibration_id,int,Yes,Yes,No,No,A unique identifier for the recalibration
+recalibrations,original_model_id,int,Yes,No,No,No,The corresponding uncalibrated model id
+recalibrations,recalibrated_model_id,int,Yes,No,No,No,The model id for the recalibrated model
+recalibrations,recalibration_type,varchar,Yes,No,No,No,The type of recalibration
+recalibrations,recalibration_json,varchar,Yes,No,No,No,The recalibration details
+performances,performance_id,int,Yes,Yes,No,No,A unique identifier for the performance
+performances,model_design_id,int,Yes,No,No,No,The corresponding model design id for development
+performances,development_database_id,int,Yes,No,No,No,The corresponding development database is
+performances,validation_database_id,int,Yes,No,No,No,The corresponding validation database is
+performances,target_id,int,Yes,No,No,No,The corresponding validation target cohort id
+performances,outcome_id,int,Yes,No,No,No,The corresponding validation outcome cohort id
+performances,tar_id,int,Yes,No,No,No,The corresponding validation time at risk id
+performances,plp_data_setting_id,int,Yes,No,No,No,The corresponding validation data settings id
+performances,population_setting_id,int,Yes,No,No,No,The corresponding validation population settings id
+performances,model_development,int,No,No,No,No,flag whether the performage is development or validation
+performances,execution_date_time,varchar,Yes,No,No,No,The date/time the validation was executed
+performances,plp_version,varchar,Yes,No,No,No,The PLP version for the validation execution
+attrition,performance_id,int,Yes,No,No,No,The corresponding performance id
+attrition,outcome_id,int,Yes,No,No,No,The corresponding outcome id
+attrition,description,varchar,Yes,No,No,No,A description of the inclusions/exclusion 
+attrition,target_count,int,Yes,No,No,No,The number of target patients remaining
+attrition,unique_people,int,Yes,No,No,No,The number of distinct target patients remaining
+attrition,outcomes,int,Yes,No,No,No,The number of target patients with the outcome remaining
+prediction_distribution,performance_id,int,Yes,No,No,No,The corresponding performance id
+prediction_distribution,evaluation,varchar,Yes,No,No,No,The type of evalaution (test/train/CV)
+prediction_distribution,class_label,int,Yes,No,No,No,whether the group is the with outcome or without outcome
+prediction_distribution,person_count,int,Yes,No,No,No,the number of patients 
+prediction_distribution,average_predicted_probability,float,Yes,No,No,No,the mean predicted risk 
+prediction_distribution,st_dev_predicted_probability,float,Yes,No,No,No,the standard deviation of the predicted risk
+prediction_distribution,min_predicted_probability,float,Yes,No,No,No,the min predicted risk
+prediction_distribution,p_05_predicted_probability,float,Yes,No,No,No,the 5% quantile of predicted risk
+prediction_distribution,p_25_predicted_probability,float,Yes,No,No,No,the 25% quantile of predicted risk
+prediction_distribution,median_predicted_probability,float,Yes,No,No,No,The median predicted risk
+prediction_distribution,p_75_predicted_probability,float,Yes,No,No,No,the 75% quantile of predicted risk
+prediction_distribution,p_95_predicted_probability,float,Yes,No,No,No,the 95% quantile of predicted risk
+prediction_distribution,max_predicted_probability,float,Yes,No,No,No,the max predicted risk
+covariate_summary,performance_id,int,Yes,No,No,No,The corresponding performance id
+covariate_summary,covariate_id,bigint,Yes,No,No,No,The id for the covariate
+covariate_summary,covariate_name,varchar,Yes,No,No,No,the name for the covariate
+covariate_summary,concept_id,float,Yes,No,No,No,the concept id used to construct the covariate
+covariate_summary,covariate_value,float,Yes,No,No,No,the coefficient or covariate importance
+covariate_summary,covariate_count,int,Yes,No,No,Yes,the number of people with the covariate
+covariate_summary,covariate_mean,float,Yes,No,No,No,the mean value 
+covariate_summary,covariate_st_dev,float,Yes,No,No,No,the standard deviation of the values
+covariate_summary,with_no_outcome_covariate_count,int,Yes,No,No,Yes,the number of people without the outcome with the covariate
+covariate_summary,with_no_outcome_covariate_mean,float,Yes,No,No,No,the mean value for people without the outcome
+covariate_summary,with_no_outcome_covariate_st_dev,float,Yes,No,No,No,the standard deviation of the values for people without the outcome
+covariate_summary,with_outcome_covariate_count,int,Yes,No,No,Yes,the number of people with the outcome with the covariate
+covariate_summary,with_outcome_covariate_mean,float,Yes,No,No,No,the mean value for people with the outcome
+covariate_summary,with_outcome_covariate_st_dev,float,Yes,No,No,No,the standard deviation of the values for people with the outcome
+covariate_summary,standardized_mean_diff,float,Yes,No,No,No,The standardized mean difference for those with and without the outcome
+threshold_summary,performance_id,int,Yes,No,No,No,The corresponding performance id
+threshold_summary,evaluation,varchar,Yes,No,No,No,The type of evalaution (test/train/CV)
+threshold_summary,prediction_threshold,float,Yes,No,No,No,The cut-off value being summarised
+threshold_summary,preference_threshold,float,Yes,No,No,No,the preference score of the cut-off value
+threshold_summary,positive_count,int,Yes,No,No,No,the number of patients predicted to have the outcome at the cut-off
+threshold_summary,negative_count,int,Yes,No,No,No,the number of patients predicted to not have the outcome at the cut-off
+threshold_summary,true_count,int,Yes,No,No,No,the number of patients with the outcome
+threshold_summary,false_count,int,Yes,No,No,No,the number of patients without the outcome
+threshold_summary,true_positive_count,int,Yes,No,No,No,the number of patients correctly predicted to have the outcome at the cut-off
+threshold_summary,true_negative_count,int,Yes,No,No,No,the number of patients correctly predicted to not have the outcome at the cut-off
+threshold_summary,false_positive_count,int,Yes,No,No,No,the number of patients incorrectly predicted to have the outcome at the cut-off
+threshold_summary,false_negative_count,int,Yes,No,No,No,the number of patients incorrectly predicted to not have the outcome at the cut-off
+threshold_summary,f_1_score,float,No,No,Yes,No,the named metric at the cut-off
+threshold_summary,accuracy,float,No,No,Yes,No,the named metric at the cut-off
+threshold_summary,sensitivity,float,No,No,Yes,No,the named metric at the cut-off
+threshold_summary,false_negative_rate,float,No,No,Yes,No,the named metric at the cut-off
+threshold_summary,false_positive_rate,float,No,No,Yes,No,the named metric at the cut-off
+threshold_summary,specificity ,float,No,No,Yes,No,the named metric at the cut-off
+threshold_summary,positive_predictive_value,float,No,No,Yes,No,the named metric at the cut-off
+threshold_summary,false_discovery_rate,float,No,No,Yes,No,the named metric at the cut-off
+threshold_summary,negative_predictive_value,float,No,No,Yes,No,the named metric at the cut-off
+threshold_summary,false_omission_rate,float,No,No,Yes,No,the named metric at the cut-off
+threshold_summary,positive_likelihood_ratio,float,No,No,Yes,No,the named metric at the cut-off
+threshold_summary,negative_likelihood_ratio,float,No,No,Yes,No,the named metric at the cut-off
+threshold_summary,diagnostic_odds_ratio,float,No,No,Yes,No,the named metric at the cut-off
+calibration_summary,performance_id,int,Yes,No,No,No,The corresponding performance id
+calibration_summary,evaluation,varchar,Yes,No,No,No,The type of evalaution (test/train/CV)
+calibration_summary,prediction_threshold,float,Yes,No,No,No,The cut-off value being summarised
+calibration_summary,person_count_at_risk,int,Yes,No,No,Yes,The number of people in the target population
+calibration_summary,person_count_with_outcome,int,Yes,No,No,Yes,The number of target patients with the outcome during TAR
+calibration_summary,average_predicted_probability,float,Yes,No,No,No,the mean predicted risk 
+calibration_summary,st_dev_predicted_probability,float,Yes,No,No,No,the standard deviation of the predicted risk
+calibration_summary,min_predicted_probability,float,Yes,No,No,No,the min predicted risk
+calibration_summary,p_25_predicted_probability,float,Yes,No,No,No,the 25% quantile of predicted risk
+calibration_summary,median_predicted_probability,float,Yes,No,No,No,The median predicted risk
+calibration_summary,p_75_predicted_probability,float,Yes,No,No,No,the 75% quantile of predicted risk
+calibration_summary,max_predicted_probability,float,Yes,No,No,No,the max predicted risk
+calibration_summary,observed_incidence,float,Yes,No,No,No,The true incidence (outcome %)
+evaluation_statistics,performance_id,int,Yes,No,No,No,The corresponding performance id
+evaluation_statistics,evaluation,varchar,Yes,No,No,No,The type of evalaution (test/train/CV)
+evaluation_statistics,metric,varchar,Yes,No,No,No,The metric of interest
+evaluation_statistics,value,float,Yes,No,No,No,The value for the metric of interest
+demographic_summary,performance_id,int,Yes,No,No,No,The corresponding performance id
+demographic_summary,evaluation,varchar,Yes,No,No,No,The type of evalaution (test/train/CV)
+demographic_summary,age_group,varchar,Yes,No,No,No,The age group of interest
+demographic_summary,gen_group,varchar,Yes,No,No,No,The gender of interest
+demographic_summary,person_count_at_risk,int,Yes,No,No,Yes,The number of target patients with the age/gender of interest
+demographic_summary,person_count_with_outcome,int,Yes,No,No,Yes,The number of target patients with the age/gender of interest who also have the outcome during TAR
+demographic_summary,average_predicted_probability,float,Yes,No,No,No,the mean predicted risk 
+demographic_summary,st_dev_predicted_probability,float,Yes,No,No,No,the standard deviation of the predicted risk
+demographic_summary,min_predicted_probability,float,Yes,No,No,No,the min predicted risk
+demographic_summary,p_25_predicted_probability,float,Yes,No,No,No,the 25% quantile of predicted risk
+demographic_summary,p_50_predicted_probability,float,Yes,No,No,No,The median predicted risk
+demographic_summary,p_75_predicted_probability,float,Yes,No,No,No,the 75% quantile of predicted risk
+demographic_summary,max_predicted_probability,float,Yes,No,No,No,the max predicted risk
\ No newline at end of file
diff --git a/inst/shiny/DiagnosticsExplorer/DiagnosticsExplorer.Rproj b/inst/shiny/DiagnosticsExplorer/DiagnosticsExplorer.Rproj
deleted file mode 100644
index 8e3c2ebc9..000000000
--- a/inst/shiny/DiagnosticsExplorer/DiagnosticsExplorer.Rproj
+++ /dev/null
@@ -1,13 +0,0 @@
-Version: 1.0
-
-RestoreWorkspace: Default
-SaveWorkspace: Default
-AlwaysSaveHistory: Default
-
-EnableCodeIndexing: Yes
-UseSpacesForTab: Yes
-NumSpacesForTab: 2
-Encoding: UTF-8
-
-RnwWeave: Sweave
-LaTeX: pdfLaTeX
diff --git a/inst/shiny/DiagnosticsExplorer/PlotsAndTables.R b/inst/shiny/DiagnosticsExplorer/PlotsAndTables.R
deleted file mode 100644
index 007038fc4..000000000
--- a/inst/shiny/DiagnosticsExplorer/PlotsAndTables.R
+++ /dev/null
@@ -1,66 +0,0 @@
-getPercentileTable <- function(table,mytargetid,myoutcomeid,mydatabasename,columnName){
-  myVar <- dplyr::sym(columnName) # giving error on my setup
-  
-  result <- table %>% 
-  filter(targetid==mytargetid & 
-         outcomeid==myoutcomeid)
-  result<-  result %>% filter(type %in% c('N','0%','1%','5%','25%','50%','75%','95%','99%','100%')) 
-  
-  if (!is.null(mydatabasename)){
-    result <- result %>% filter(databasename==mydatabasename) %>%
-    select(!!c(year = 'year',myVar = columnName, type ='type')) %>%
-    arrange(year) %>%
-    spread(key=type,value=myVar,fill='NA') %>%
-    select(year,N,'0%','1%','5%','25%','50%','75%','95%','99%','100%')
-  } else {
-    result <- result %>% select(!!c(databasename = 'databasename', year = 'year', myVar = columnName, type ='type')) %>%
-      arrange(year) %>%
-      spread(key=type,value=myVar,fill='NA') %>%
-      select(databasename,year,N,'0%','1%','5%','25%','50%','75%','95%','99%','100%')
-  }
-
-  return(result)
-}
-
-
-# box plots 
-myBoxplot <- function(myDatabasename,targetId,outcomeId,variable){
-  
-  plotdata <- getPercentileTable(distribution,
-                                 targetId,
-                                 outcomeId,
-                                 NULL,
-                                 tolower(variable)) %>%
-    filter(databasename == myDatabasename)
-  
-  if(nrow(plotdata)==0){
-    return(ggplot2::ggplot()) +
-      ggplot2::labs(title=paste('Database:', myDatabasename))
-  } else {
-    
-    plotdata[plotdata=='NA'] <- 0
-    colnames(plotdata) <- c('databasename','year','N','p0','p1','p5','p25','p50','p75','p95','p99','p100')
-    
-    plotResult <- ggplot2::ggplot(plotdata, ggplot2::aes(x=as.factor(year),
-                                                         ymin= ifelse(is.na(p0), 0, as.double(as.character(p0))),
-                                                         lower= ifelse(is.na(p25), 0, as.double(as.character(p25))),
-                                                         middle= ifelse(is.na(p50), 0, as.double(as.character(p50))),
-                                                         upper= ifelse(is.na(p75), 0, as.double(as.character(p75))), 
-                                                         ymax= ifelse(is.na(p100), 0, as.double(as.character(p100))), 
-                                                         color= as.factor(databasename))) + 
-      ggplot2::geom_hline(yintercept=365, linetype="dashed", color = "black") +
-      ggplot2::geom_hline(yintercept=365*2, linetype="dashed", color = "black") +
-      ggplot2::geom_hline(yintercept=365*3, linetype="dashed", color = "black") +
-      ggplot2::geom_hline(yintercept=365*4, linetype="dashed", color = "black") +
-      ggplot2::geom_hline(yintercept=365*5, linetype="dashed", color = "black") +
-      #ggplot2::coord_flip() +
-      ggplot2::geom_boxplot(stat="identity")  +
-      ggplot2::scale_x_discrete("Year") + 
-      ggplot2::scale_y_continuous("Time in Days") + 
-      ggplot2::theme(legend.position="none") +
-      ggplot2::labs(title=paste('Database:', myDatabasename))
-    
-    return(plotResult)
-  }
-}
-
diff --git a/inst/shiny/DiagnosticsExplorer/global.R b/inst/shiny/DiagnosticsExplorer/global.R
deleted file mode 100644
index 4cebc84a5..000000000
--- a/inst/shiny/DiagnosticsExplorer/global.R
+++ /dev/null
@@ -1,120 +0,0 @@
-# check package dependancies - will move to launcher after study-a-thon
-stop <- F
-for(pkg in c('shiny', 'shinydashboard', 'DT', 'plotly', 'dplyr', 'tidyr','ggplot2', 'shinycssloaders')){
-  if(is.na(tryCatch({utils::packageVersion(pkg)}, error = function(e) NA))){
-    warning(paste0('Package ', pkg, ' not installed - please install'))
-    stop <- T
-  }
-}
-if(stop){stop('Need to install dependancies')}
-
-library(shiny)
-library(shinydashboard)
-library(DT)
-library(plotly)
-library(dplyr)
-library(tidyr)
-#library(scales)
-#library(ggiraph)
-
-
-rm(list=ls()[ls()%in%c('settings', 'namesdetails', 'characterization','distribution','proportion')])
-source("PlotsAndTables.R")
-
-
-if (!exists("shinySettings")) {
-  if (file.exists("diagnostics")) {
-    shinySettings <- list(dataFolder = "diagnostics")
-  } else {
-    shinySettings <- list(dataFolder = "./diagnostics")
-  }
-  
-}
-
-dataFolder <- shinySettings$dataFolder
-
-if (file.exists(file.path(dataFolder, "PreMerged.RData"))) {
-  writeLines("Using merged data detected in data folder")
-  load(file.path(dataFolder, "PreMerged.RData"))
-} else {
-  zipFiles <- list.files(dataFolder, pattern = ".zip", full.names = TRUE)
-  
-  loadFile <- function(file, folder, overwrite) {
-    print(file)
-    tableName <- gsub(".csv$", "", file)
-    camelCaseName <- SqlRender::snakeCaseToCamelCase(tableName)
-    #data <- readr::read_csv(file.path(folder, file), col_types = readr::cols(), guess_max = 1e7, locale = readr::locale(encoding = "UTF-8"))
-    data <- utils::read.csv(file.path(folder, file))
-    
-    colnames(data) <- SqlRender::snakeCaseToCamelCase(colnames(data))
-    
-    if (!overwrite && exists(camelCaseName, envir = .GlobalEnv)) {
-      existingData <- get(camelCaseName, envir = .GlobalEnv)
-      if (nrow(existingData) > 0) {
-        if (nrow(data) > 0 &&
-            all(colnames(existingData) %in% colnames(data)) &&
-            all(colnames(data) %in% colnames(existingData))) {
-          data <- data[, colnames(existingData)]
-        }
-        
-        if (!isTRUE(all.equal(colnames(data), colnames(existingData), check.attributes = FALSE))) {
-          stop("Table columns do no match previously seen columns. Columns in ", 
-               file, 
-               ":\n", 
-               paste(colnames(data), collapse = ", "), 
-               "\nPrevious columns:\n",
-               paste(colnames(existingData), collapse = ", "))
-        }
-      }
-      data <- rbind(existingData, data)
-    }
-    assign(camelCaseName, data, envir = .GlobalEnv)
-    
-    invisible(NULL)
-  }
-  
-  for (i in 1:length(zipFiles)) {
-    writeLines(paste("Processing", zipFiles[i]))
-    tempFolder <- tempfile()
-    dir.create(tempFolder)
-    unzip(zipFiles[i], exdir = tempFolder)
-    
-    csvFiles <- list.files(tempFolder, pattern = ".csv")
-    lapply(csvFiles, loadFile, folder = tempFolder, overwrite = (i == 1))
-    
-    unlink(tempFolder, recursive = TRUE)
-  }
-}
-
-namesdetails$names <- as.character(namesdetails$names)
-# Fixing the labels (more to add)
-getNames <- function(cohortid){
-  res <- namesdetails %>% filter(ids == cohortid) %>% select(names)
-  res$names[1]
-}
-getId <- function(cohortname){
-  res <- namesdetails %>% filter(names == cohortname) %>% select(ids)
-  res$ids[1]
-}
-
-# Sort selectors
-databases <- sort(as.list(unique(settings %>% select(cdmdatabasename)))$cdmdatabasename)
-
-outcomeCohorts <- lapply(as.list(unique(settings %>% select(outcomeid)))$outcomeid, getNames)
-targetCohorts <- lapply(as.list(unique(settings %>% select(cohortid)))$cohortid, getNames)
-
-settings$tar <- unlist(
-  lapply(1:nrow(settings), function(x) paste0(settings$startanchor[x], ' + ', settings$riskwindowstart[x], 
-                                          ' days - ', settings$endanchor[x], ' + ', settings$riskwindowend[x], ' days'))
-  )
-tars <- as.list(unique(settings %>% select(tar)))$tar
-#tars <- unique(settings %>% select(riskwindowstart,	startanchor,	riskwindowend,	endanchor))
-#tars <- lapply(1:nrow(tars), function(x) paste0(tars$startanchor[x], ' + ', tars$riskwindowstart[x], 
-#                                  ' days - ', tars$endanchor[x], ' + ', tars$riskwindowend[x], ' days'))
-
-# Variable Selector
-distributionVars <- c('daysFromObsStart','daysToObsEnd','daysToOutcomeAfterMin','daysToOutcomeBeforeMin')
-
-
-writeLines("Data Loaded")
-
diff --git a/inst/shiny/DiagnosticsExplorer/html/about.html b/inst/shiny/DiagnosticsExplorer/html/about.html
deleted file mode 100644
index 3f52b2bb3..000000000
--- a/inst/shiny/DiagnosticsExplorer/html/about.html
+++ /dev/null
@@ -1,4 +0,0 @@
-<h3>Description</h3>
-<p>General informatiom about the tool. Also includes links to the protocol and code to execute the study</p>
-
-
diff --git a/inst/shiny/DiagnosticsExplorer/html/characterizationInfo.html b/inst/shiny/DiagnosticsExplorer/html/characterizationInfo.html
deleted file mode 100644
index a1563840b..000000000
--- a/inst/shiny/DiagnosticsExplorer/html/characterizationInfo.html
+++ /dev/null
@@ -1,3 +0,0 @@
-<h3>Characterisation</h3>
-
-Text to be added
diff --git a/inst/shiny/DiagnosticsExplorer/html/databases.html b/inst/shiny/DiagnosticsExplorer/html/databases.html
deleted file mode 100644
index 841fafa85..000000000
--- a/inst/shiny/DiagnosticsExplorer/html/databases.html
+++ /dev/null
@@ -1,8 +0,0 @@
-<h3>Description</h3>
-<p>A table showing the meta data of the included databases in the Drug Utilization Study.</p>
-
-<h3>Options</h3>
-<p>To be added</p>
-
-
-
diff --git a/inst/shiny/DiagnosticsExplorer/html/distributionInfo.html b/inst/shiny/DiagnosticsExplorer/html/distributionInfo.html
deleted file mode 100644
index 8eda392bb..000000000
--- a/inst/shiny/DiagnosticsExplorer/html/distributionInfo.html
+++ /dev/null
@@ -1,4 +0,0 @@
-<h3>Distributions/h3>
-
-Text to be added
-
diff --git a/inst/shiny/DiagnosticsExplorer/html/proportionInfo.html b/inst/shiny/DiagnosticsExplorer/html/proportionInfo.html
deleted file mode 100644
index 3e07d9245..000000000
--- a/inst/shiny/DiagnosticsExplorer/html/proportionInfo.html
+++ /dev/null
@@ -1,3 +0,0 @@
-<h2>Proportion</h2>
-
-Text to be added
diff --git a/inst/shiny/DiagnosticsExplorer/html/results.html b/inst/shiny/DiagnosticsExplorer/html/results.html
deleted file mode 100644
index 748f8cf39..000000000
--- a/inst/shiny/DiagnosticsExplorer/html/results.html
+++ /dev/null
@@ -1,10 +0,0 @@
-<h3>Description</h3>
-<p>Multiple tables and figures showing the results of the study.</p>
-
-<h3>Options</h3>
-<p>On the left you first have to select the analysis for which you want to view the results. Other options become available, such as selectors for the ingredient, formulations etc. if these are applicable.</p>
-
-<p>A "Figures" tab becomes available for certain analyses on the top in the right part of the screen.</p>
-
-
-
diff --git a/inst/shiny/DiagnosticsExplorer/server.R b/inst/shiny/DiagnosticsExplorer/server.R
deleted file mode 100644
index fe303f9e0..000000000
--- a/inst/shiny/DiagnosticsExplorer/server.R
+++ /dev/null
@@ -1,549 +0,0 @@
-prettyHr <- function(x) {
-  result <- sprintf("%.2f", x)
-  result[is.na(x)] <- "NA"
-  result <- suppressWarnings(format(as.numeric(result), big.mark=",")) # add thousands separator
-  return(result)
-}
-
-addThousandsSeparator<-function(table){
-  if(is.data.frame(table)){
-    is.num <- sapply(table, is.numeric)
-    table[is.num] <- lapply(table[is.num], function(x) format(as.numeric(x), big.mark=","))
-    return(table)
-  } else {
-    is.not.na<- !sapply(suppressWarnings(as.numeric(table)), is.na)
-    table[is.not.na] <- format(as.numeric(table[is.not.na]), big.mark=",")
-    return(table)
-  }
-  
-}
-
-getHoveroverStyle <- function(left_px, top_px) {
-  style <- paste0("position:absolute; z-index:100; background-color: rgba(245, 245, 245, 0.85); ",
-                   "left:",
-                   left_px - 200,
-                   "px; top:",
-                   top_px - 130,
-                   "px; width:400px;")
-}
-
-
-
-shinyServer(function(input, output, session) {
-  
-  
-  # Tables
-  
-  output$proportionTable <- renderDT({
-    
-    analysisId <- settings %>% filter(cdmdatabasename == input$pdatabase & 
-                                        tar == input$ptar &
-                                        outcomeid == getId(input$poutcomeName) &
-                                        cohortid ==  getId(input$ptargetName)
-    ) %>% select(analysisid)
-    analysisId <-analysisId$analysisid[1]
-    
-    if(input$pxyear){
-     proportionAll <- proportion %>% dplyr::filter(agegroup == 'all')
-    }else{
-      proportionAll <- proportion %>% dplyr::filter(year == 'all') 
-    }
-    
-    table <- proportionAll %>% filter(analysisid == analysisId) %>%
-      select(year,agegroup,gender,n,o,opercent)
-    
-    if(input$pgender){
-      table <- table %>% filter(gender != -1)
-    }else{
-      table <- table %>% filter(opercent != -1) %>% 
-        group_by(year,agegroup) %>%
-        summarise(n = sum(as.double(as.character(n)), na.rm = T),
-                  o = sum(as.double(as.character(o)), na.rm = T),
-                  opercent = sum(o, na.rm = T)*100/sum(n, na.rm = T))
-    }
-    
-    # get the correct columns
-    if(input$pxyear & input$pgender){
-      table <-table %>% ungroup() %>%
-        select(year,gender,n,o,opercent)
-    } else if(!input$pxyear & input$pgender){
-      table <-table %>% ungroup() %>%
-        select(agegroup,gender,n,o,opercent)
-    } else if(!input$pxyear & !input$pgender){
-      table <-table %>% ungroup() %>%
-        select(agegroup,n,o,opercent)
-    } else if(input$pxyear & !input$pgender){
-      table <- table %>% ungroup() %>%
-        select(year,n,o,opercent)
-    }
-    
-    table$opercent = as.numeric(prettyHr(table$opercent))
-    result<-datatable(table,
-      filter="top",
-      options = list(pageLenth=25,
-                     scrollX = TRUE,
-                     dom='Blfrtip', 
-                     buttons=c('colvis','csv','excel')),
-      extensions = 'Buttons',
-      rownames = FALSE,
-      escape = FALSE,
-      class = "stripe nowrap compact")
-    return(result)}
-  )
-  
-  output$proportionPlot <- renderPlotly({
-    mySubplot <- function(myanalysisId,myDatabasename, myTar){
-      
-      if(input$pxyear){
-        proportionAll <- proportion %>% dplyr::filter(agegroup == 'all')
-      }else{
-        proportionAll <- proportion %>% dplyr::filter(year == 'all') 
-      }
-      
-      plotdata <- proportionAll %>% filter(analysisid == myanalysisId)
-      plotdataM <- plotdata %>% filter(gender == 8507)
-      plotdataF <- plotdata %>% filter(gender == 8532)
-      plotdataA <- plotdata %>% filter(opercent != -1) %>% 
-        group_by(year,agegroup) %>%
-        summarise(n = sum(as.double(as.character(n)), na.rm = T),
-                  o = sum(as.double(as.character(o)), na.rm = T),
-                  opercent = sum(o, na.rm = T)*100/sum(n, na.rm = T)) %>%
-        ungroup() %>% data.frame()
-      
-      fig1 <- plot_ly()
-      if(input$pxyear){
-        levels <- c(2005:2020)
-      } else{
-        levels <- c('0 - 4','5 - 9','10 - 14','15 - 19', '20 - 24',
-                    '25 - 29','30 - 34','35 - 39','40 - 44','45 - 49',
-                    '50 - 54','55 - 59','60 - 64','65 - 69','70 - 74',
-                    '75 - 79','80 - 84','85 - 89','90 - 94','95 - 99')
-      }
-      plotdataF$var <- factor(as.character(plotdataF[,ifelse(input$pxyear,'year','agegroup')]), 
-                              levels = levels,ordered = TRUE)
-      plotdataM$var <- factor(as.character(plotdataM[,ifelse(input$pxyear,'year','agegroup')]), 
-                                           levels = levels, ordered = TRUE)
-      plotdataA$var <- factor(as.character(plotdataA[,ifelse(input$pxyear,'year','agegroup')]), 
-                                           levels = levels, ordered = TRUE)
-      plotdataF <- plotdataF[order(plotdataF$var),]
-      plotdataM <- plotdataM[order(plotdataM$var),]
-      plotdataA <- plotdataA[order(plotdataA$var),]
-      
-      if(input$pgender){
-      fig1 <- fig1 %>%
-        add_trace(
-          x = plotdataF$var,
-          y = plotdataF$opercent,
-          type = 'scatter',
-          name = '',
-          mode = 'lines+markers',
-          showlegend = FALSE,
-          hovertemplate = paste('<b>',ifelse(input$pxyear,'year','agegroup'),'</b>: ',plotdataF$var,
-                                '<br><i>T size </i>',plotdataF$n,'<br>',
-                                '<i>O percent</i>: %{y:.2f}',
-                                '<br>')
-        )
-      
-      fig1 <- fig1 %>%
-        add_trace(
-          x = plotdataM$var,
-          y = plotdataM$opercent,
-          type = 'scatter',
-          name = '',
-          mode = 'lines+markers',
-          showlegend = FALSE, 
-          hovertemplate = paste('<b>',ifelse(input$pxyear,'year','agegroup'),'</b>: ',plotdataM$var,
-                                '<br><i>T size </i>',plotdataM$n,'<br>',
-                                '<i>O percent</i>: %{y:.2f}',
-                                '<br>')
-          #markers=list(color='red',dash='dashed')
-        )
-      } else{
-        fig1 <- fig1 %>%
-          add_trace(
-            x = plotdataA$var,
-            y = plotdataA$opercent,
-            type = 'scatter',
-            name = '',
-            mode = 'lines+markers',
-            showlegend = FALSE, 
-            hovertemplate = paste('<b>',ifelse(input$pxyear,'year','agegroup'),'</b>: ',plotdataA$var,
-                                  '<br><i>T size </i>',plotdataA$n,'<br>',
-                                  '<i>O percent</i>: %{y:.2f}',
-                                  '<br>')
-          )
-      }
-      
-      fig1
-    }
-    
-    subplots_list <- list()
-    ind <- 1
-    for(i in 1:length(input$databases)){
-      
-      analysisId <- settings %>% filter(cdmdatabasename == input$pdatabases[i] & 
-                                          tar == input$ptar &
-                                          outcomeid == getId(input$poutcomeName) &
-                                          cohortid ==  getId(input$ptargetName)
-      ) %>% select(analysisid)
-      analysisId <-analysisId$analysisid[1]
-      
-      # if there is a result add it 
-      if(!is.na(analysisId)){
-        tempPlot <- mySubplot(analysisId, input$databases[i], input$ptar)
-        subplots_list[[ind]] <- tempPlot
-        ind <- ind+1
-      }
-    }
-    
-    
-    
-    fig <- subplot(subplots_list, nrows=length(input$databases),shareX = F, shareY =F)
-    
-    return(fig)
-  })
-  
-  
-  output$survivalPlot <- renderPlot({
-    
-    data <- survival %>% 
-      dplyr::filter(outcomeid == getId(input$soutcomeName)) %>%
-      dplyr::filter(cohortid == getId(input$stargetName)) %>%
-      dplyr::filter(cdmdatabasename == input$sdatabase)
-    
-    data <- data %>% dplyr::mutate(decreaseP = events/(events+natrisk))
-    
-    yaxis <- lapply(unique(data$censoredtime), function(i) 1-sum(data %>% dplyr::filter(censoredtime <= i) %>% dplyr::select(decreaseP)))
-    
-    extra <- data.frame(censoredtime = unique(data$censoredtime),
-                        yaxis = unlist(yaxis))
-    
-    data <- data %>% inner_join(extra, by = 'censoredtime')
-    
-    library(ggplot2)
-    ggplot() +
-      geom_step(data=data, mapping=aes(x=censoredtime, y=yaxis)) +
-      #geom_step(data=d, mapping=aes(x=x, y=y), direction="vh", linetype=3) +
-      geom_point(data=data, mapping=aes(x=censoredtime, y=yaxis), color="red") +
-      geom_vline(xintercept = 365, linetype="dotted", 
-                 color = "black", size=1) +
-      geom_vline(xintercept = 2*365, linetype="dotted", 
-                 color = "black", size=1) +
-      geom_vline(xintercept = 3*365, linetype="dotted", 
-                 color = "black", size=1) + 
-      geom_vline(xintercept = 4*365, linetype="dotted",
-                 color = "black", size=1) +
-      geom_vline(xintercept = 5*365, linetype="dotted", 
-                 color = "black", size=1) +
-      geom_vline(xintercept = 10*365, linetype="dotted", 
-                 color = "black", size=1) +
-      ylim(0, 1) + labs(x = "Time from index (days)",
-                        y = "Outcome free")
-    
-  })
-
-  
-  output$characterizationTable <- renderDT({
-    
-    analysisId <- settings %>% filter(cdmdatabasename == input$cdatabase & 
-                                        tar == input$ctar &
-                                        outcomeid == getId(input$coutcomeName) &
-                                        cohortid ==  getId(input$ctargetName)
-    ) %>% select(analysisid)
-    analysisId <-analysisId$analysisid[1]
-    
-    table <- characterization %>% filter(analysisid == analysisId)
-    table$covariatemeanwithoutcome = as.numeric(prettyHr(table$covariatemeanwithoutcome))
-    table$covariatemeanwithnooutcome = as.numeric(prettyHr(table$covariatemeanwithnooutcome))
-    result<-datatable(table,
-                      filter="top",
-                      options = list(pageLenth=25,
-                                     scrollX = TRUE,
-                                     dom ='Blfrtip', 
-                                     buttons=c('colvis','csv','excel')),
-                      extensions = 'Buttons',
-                      rownames = FALSE,
-                      escape = FALSE,
-                      class = "stripe nowrap compact")
-    return(result)}
-  )
-  
-  
-  output$characterizationPlot <- renderPlotly({
-    mySubplot <- function(myanalysisId,myDatabasename, myTar){
-      
-      plotdata <- characterization %>% filter(analysisid == myanalysisId)
-      
-      plotdataMeas <- plotdata %>% filter(covariatemeanwithnooutcome > 1 | covariatemeanwithoutcome > 1)
-      plotdata <- plotdata %>% filter(covariatemeanwithnooutcome < 1 & covariatemeanwithoutcome < 1)
-      
-
-      fig1 <- plot_ly()
-      fig2 <- plot_ly()
-      
-      fig1 <- fig1 %>%
-        add_trace(
-          type = 'scatter',
-          name = '',
-          mode = 'markers',
-          x = plotdata$covariatemeanwithnooutcome,
-          y = plotdata$covariatemeanwithoutcome,
-          showlegend = FALSE,
-          hovertemplate = paste(plotdata$covariatename,'<br>',
-                                '<i>Mean with outcome</i>: %{y:.2f}',
-                                '<br><i>Mean with no outcome</i>: %{x}<br>')
-        ) %>%
-        add_trace(x=c(0, 1), y=c(0, 1), 
-                  type="scatter", mode="lines", name='x = y') %>%
-      
-        layout(annotations = list(x = 0.5, y = 1.05, 
-                                  text=sprintf("<b>Database: %s - TAR: %s<b>",myDatabasename,myTar ),
-                                  xref='paper', yref='paper', showarrow=F),
-               xaxis = list(title = "Non-outcomes"), 
-               yaxis = list(title = "Outcomes"),
-               showlegend = FALSE)
-      
-      if(nrow(plotdataMeas)>0){
-        fig2 <- fig2 %>%
-          add_trace(
-            type = 'scatter',
-            name = '',
-            mode = 'markers',
-            x = plotdataMeas$covariatemeanwithnooutcome,
-            y = plotdataMeas$covariatemeanwithoutcome,
-            showlegend = FALSE,
-            hovertemplate = paste(plotdataMeas$covariatename,'<br>',
-                                  '<i>Mean with outcome</i>: %{y:.2f}',
-                                  '<br><i>Mean with no outcome</i>: %{x}<br>')
-          ) %>%
-          add_trace(x=c(0, max(c(plotdataMeas$covariatemeanwithnooutcome,plotdataMeas$covariatemeanwithoutcome))), 
-                    y=c(0, max(c(plotdataMeas$covariatemeanwithnooutcome,plotdataMeas$covariatemeanwithoutcome))), 
-                    type="scatter", mode="lines", name='x = y')  %>% 
-          layout(#xaxis = list(title = "Varible mean of non-outcomes"), 
-                 #yaxis = list(title = "Varible mean of outcomes"),
-                 showlegend = FALSE)
-      }
-      
-      list(fig1,fig2)
-      
-    }
-    
-    subplots_list <- list()
-    ind <- 1
-    for(i in 1:length(input$databases)){
-      
-      analysisId <- settings %>% filter(cdmdatabasename == input$cdatabases[i] & 
-                                          tar == input$ctar &
-                                          outcomeid == getId(input$coutcomeName) &
-                                          cohortid ==  getId(input$ctargetName)
-      ) %>% select(analysisid)
-      analysisId <-analysisId$analysisid[1]
-      
-      # if there is a result add it 
-      if(!is.na(analysisId)){
-        tempPlot <- mySubplot(analysisId, input$databases[i], input$ctar)
-        subplots_list[[ind]] <- tempPlot[[1]]
-        ind <- ind+1
-        subplots_list[[ind]] <- tempPlot[[2]]
-        ind <- ind+1
-      }
-    }
-    
-
-    
-    fig <- subplot(subplots_list, nrows=length(input$databases),shareX = F, shareY =F)
-    
-    return(fig)
-  })
-  
-  
-  
-  
-  
-  
-  output$distributionTable <- renderDataTable({
-    table <- getPercentileTable(distribution,getId(input$targetName),getId(input$outcomeName),input$database,tolower(input$variable))
-    options(digits = 2)
-      
-      selection = list(mode = "single", target = "row")
-      table <- datatable(
-        table,
-        extensions = c('Buttons','FixedColumns'),
-        options = list(
-          aoColumnDefs = list(list(className= 'dt-left', targets = "_all")),
-          pageLength = 50,
-          ordering = FALSE,
-          dom ='Blfrtip',
-          scrollX = TRUE,
-          fixedColumns = TRUE,
-          buttons =
-            list(
-              'copy',
-              'print',
-              list(
-                extend = 'collection',
-                buttons = c('colvis','csv', 'excel'),
-                text = 'Download'
-              )
-            )
-        ),
-        #options = options,
-        selection = selection,
-        rownames = FALSE,
-        escape = FALSE,
-        class = "stripe nowrap compact"
-      )
-      return(table)
-
-  })
-  
-  # Plot titles
-  output$distributionTimePlotTitle <- renderText(
-   paste0(input$variable, " over the years in ",  paste(input$databases, collapse=', '))
-  )
-  
-  # Plots
-  output$distributionTimePlot <- renderPlotly({
-    mySubplot <- function(myDatabasename){
-      fig <- plot_ly()
-      plotdata <-
-        getPercentileTable(distribution,
-                           getId(input$targetName),
-                           getId(input$outcomeName),
-                           NULL,
-                           tolower(input$variable)) %>%
-        filter(databasename == myDatabasename)
-      date <- as.Date(paste(plotdata$year, "01", "01", sep = ""), "%Y%m%d")
-      fig <- fig %>%
-        add_trace(
-          type = 'scatter',
-          name = '',
-          mode = 'lines+markers',
-          x = date,
-          y = as.numeric(plotdata$`50%`),
-          showlegend = FALSE,
-          hovertemplate = paste(myDatabasename,'<br>',
-                                '<i>Median</i>: %{y:.2f}',
-                                '<br><b>Year</b>: %{x}<br>')
-        ) %>%
-        
-        add_trace(
-          type = 'scatter',
-          name = '',
-          mode = 'lines+markers',
-          x = date,
-          y = as.numeric(plotdata$`95%`),
-          showlegend = FALSE,
-          hovertemplate = paste(myDatabasename,'<br>',
-                                '<i>P95</i>: %{y:.2f}',
-                                '<br><b>Year</b>: %{x}<br>')
-        ) %>%
-        layout(annotations = list(x = 0.5, y = 1.05, text=sprintf("<b>%s<b>",myDatabasename), xref='paper', yref='paper', showarrow=F))
-    }
-    
-    subplots_list <- list()
-    for(i in 1:length(input$databases)){
-      subplots_list[[i]] <- mySubplot(input$databases[i])
-    }
-    
-    fig <- subplot(subplots_list, nrows=length(input$databases),shareX = T, shareY =T)
-    
-    return(fig)
-  })
-  
-  output$distributionBoxPlot <- renderPlot({
-    
-      subplots_list <- list()
-      for(i in 1:length(input$databases)){
-        subplots_list[[i]] <- myBoxplot(input$databases[i],getId(input$targetName),getId(input$outcomeName),input$variable)
-      }
-      
-      require(gridExtra)
-      fig <- do.call("grid.arrange", c(subplots_list, ncol=length(input$databases)))
-      
-    return(fig)
-  })
-  
-  getdistributionTooltip <- function(
-                                   top_px,
-                                   point) {
-    text <- 'Test hoover'
-    #text <- gsub("-", "<", sprintf("<b>%s proportion: </b> %0.3f per 1000 persons", proportionType, point$proportion))
-    # if (!is.na(point$ageGroup)) {
-    #   text <- paste(text, sprintf("<b>Age group: </b> %s years", point$ageGroup), sep = "<br/>")
-    #   top_px <- top_px - 15
-    # }
-    # if (!is.na(point$gender)) {
-    #   text <- paste(text, sprintf("<b>Gender: </b> %s", point$gender), sep = "<br/>")
-    #   top_px <- top_px - 15
-    # }
-    # if (!is.na(point$calendarYear)) {
-    #   text <- paste(text, sprintf("<b>Calendar year: </b> %s", point$calendarYear), sep = "<br/>")
-    #   top_px <- top_px - 15
-    # }
-    # if (!is.na(point$cohortCount)) {
-    #   text <- paste(text, sprintf("<b>%s patients: </b> %s", proportionType, scales::comma(point$cohortCount)), sep = "<br/>")
-    #   top_px <- top_px - 15
-    # }
-    # if (!is.na(point$numPersons)) {
-    #   text <- paste(text, sprintf("<b>Denominator: </b> %s", scales::comma(point$numPersons)), sep = "<br/>")
-    #   top_px <- top_px - 15
-    # }
-    # text <- paste(text, sprintf("<b>Database: </b> %s", point$databaseId), sep = "<br/>")
-    return(list(top_px = top_px, text = text))
-  }
-  
-  output$hoverInfoDistribution <- renderUI({
-    data <- getPercentileTable(distribution,getId(input$targetName),getId(input$outcomeName),NULL,tolower(input$variable)) %>%
-      filter(databasename %in% input$databases)
-    if (is.null(data)) {
-      return(NULL)
-    }else {
-      hover <- input$plotHoverDistribution
-      point <- nearPoints(data, hover, threshold = 5, maxpoints = 1, addDist = TRUE)
-      if (nrow(point) == 0) {
-        return(NULL)
-      }
-      left_px <- hover$coords_css$x
-      top_px <- hover$coords_css$y
-      
-      tooltip <- getProportionTooltip(top_px, point)
-      style <- getHoveroverStyle(left_px = left_px, top_px = tooltip$top_px)
-      div(
-        style = "position: relative; width: 0; height: 0",
-        wellPanel(
-          style = style,
-          p(HTML(tooltip$text))
-        )
-      )
-    }
-  }) 
-  
-  # Functionality for help messages
-  showInfoBox <- function(title, htmlFileName) {
-    showModal(modalDialog(
-      title = title,
-      easyClose = TRUE,
-      footer = NULL,
-      size = "l",
-      HTML(readChar(htmlFileName, file.info(htmlFileName)$size) )
-    ))
-  }
-  
-  observeEvent(input$aboutInfo, {
-    showInfoBox("About", "html/about.html")
-  })
-  
-  observeEvent(input$incidenceInfo, {
-    showInfoBox("Study Results", "html/incidenceInfo.html")
-  })
-  
-  observeEvent(input$characterizationInfo, {
-    showInfoBox("Study Results", "html/characterizationInfo.html")
-  })
-  
-  observeEvent(input$distributionInfo, {
-    showInfoBox("Study Results", "html/distributionInfo.html")
-  })
-  
-})
diff --git a/inst/shiny/DiagnosticsExplorer/ui.R b/inst/shiny/DiagnosticsExplorer/ui.R
deleted file mode 100644
index 7e6226464..000000000
--- a/inst/shiny/DiagnosticsExplorer/ui.R
+++ /dev/null
@@ -1,214 +0,0 @@
-library(shinydashboard)
-library(shiny)
-library(DT)
-library(plotly)
-library(shinyWidgets)
-
-addInfo <- function(item, infoId) {
-  infoTag <- tags$small(
-    class = "badge pull-right action-button",
-    style = "padding: 1px 6px 2px 6px; background-color: steelblue;",
-    type = "button",
-    id = infoId,
-    "i"
-  )
-  item$children[[1]]$children <-
-    append(item$children[[1]]$children, list(infoTag))
-  return(item)
-}
-
-dashboardPage(
-  dashboardHeader(title = "Diagnostics Explorer"),
-  dashboardSidebar(
-    sidebarMenu(
-      id = "tabs",
-      ## Tabs
-      addInfo(menuItem("About", tabName = "about"), "aboutInfo"),
-      addInfo(menuItem("Proportion", tabName = "proportion"), "proportionInfo"),
-      addInfo(menuItem("Survival", tabName = "survival"), "survivalInfo"),
-      addInfo(menuItem("Characterization", tabName = "characterization"), "characterizationInfo"),
-      addInfo(menuItem("Distribution", tabName = "distribution"), "distributionInfo"),
-      
-      ## Option panel
-      
-      # propoortion options
-      # characterization options
-      conditionalPanel(
-        condition = "input.tabs=='proportion'",
-        selectInput("ptargetName", "Target", targetCohorts),
-        selectInput("poutcomeName", "Outcome", outcomeCohorts),
-        selectInput("ptar","Time-at-risk",tars),
-        shinyWidgets::switchInput("pgender", "Gender Split",value = TRUE, width = '80%'),
-        shinyWidgets::switchInput("pxyear",value = TRUE, "By year")
-      ),
-      conditionalPanel(
-        condition = "input.tabs=='proportion' && input.proportionTabsetPanel == 'Tables'",
-        selectInput("pdatabase", "Database", databases)),
-      
-      conditionalPanel(condition = "input.tabs == 'proportion' && (input.proportionTabsetPanel == 'Figure' )",
-                       hr(),
-                       checkboxGroupInput("pdatabases", "Database", databases, selected = databases[1])
-      ),
-      
-      
-      # survival
-      conditionalPanel(
-        condition = "input.tabs=='survival'",
-        selectInput("stargetName", "Target", targetCohorts),
-        selectInput("soutcomeName", "Outcome", outcomeCohorts),
-        selectInput("sdatabase", "Database", databases)
-      ),
-      
-      
-      # distribution options
-      conditionalPanel(
-        condition = "input.tabs=='distribution' && input.distributionTabsetPanel == 'Tables'",
-        selectInput("database", "Database", databases)),
-      conditionalPanel(
-         condition = "input.tabs=='distribution'",
-
-         selectInput("targetName", "Target", targetCohorts),
-         selectInput("outcomeName", "Outcome", outcomeCohorts),
-         selectInput("variable","distributionVar",distributionVars)
-      ),
-      conditionalPanel(condition = "input.tabs == 'distribution' && (input.distributionTabsetPanel == 'Time Trend' | input.distributionTabsetPanel == 'Box Plot' )",
-                       hr(),
-                       checkboxGroupInput("databases", "Database", databases, selected = databases[1])
-      ),
-      # characterization options
-      conditionalPanel(
-        condition = "input.tabs=='characterization'",
-        selectInput("ctargetName", "Target", targetCohorts),
-        selectInput("coutcomeName", "Outcome", outcomeCohorts),
-        selectInput("ctar","Time-at-risk",tars)
-      ),
-      conditionalPanel(
-        condition = "input.tabs=='characterization' && input.characterizationTabsetPanel == 'Tables'",
-        selectInput("cdatabase", "Database", databases)),
-      
-    conditionalPanel(condition = "input.tabs == 'characterization' && (input.characterizationTabsetPanel == 'Figure' )",
-                     hr(),
-                     checkboxGroupInput("cdatabases", "Database", databases, selected = databases[1])
-    )
-    )
-    
-  ),
-  dashboardBody(
-    
-    tags$body(tags$div(id="ppitest", style="width:1in;visible:hidden;padding:0px")),
-    tags$script('$(document).on("shiny:connected", function(e) {
-                                    var w = window.innerWidth;
-                                    var h = window.innerHeight;
-                                    var d =  document.getElementById("ppitest").offsetWidth;
-                                    var obj = {width: w, height: h, dpi: d};
-                                    Shiny.onInputChange("pltChange", obj);
-                                });
-                                $(window).resize(function(e) {
-                                    var w = $(this).width();
-                                    var h = $(this).height();
-                                    var d =  document.getElementById("ppitest").offsetWidth;
-                                    var obj = {width: w, height: h, dpi: d};
-                                    Shiny.onInputChange("pltChange", obj);
-                                });
-                            '),
-    
-    tabItems(
-    tabItem(
-      tabName = "about",
-      br(),
-      p(
-        "This interactive web-based application provides Diagnostics for a Patient-Level Prediction study."
-      ),
-      h3("Rationale and background"),
-      p(
-        " The idea is to first generate Diagnostics for the prediction problem before the execution. [MORE TO ADD]"
-      ),
-      h3("External links"),
-      HTML("<p>Below are links for study-related artifacts that have been made available as part of this study:</p>"),
-      HTML("<ul>"),
-      HTML("<li>The study is registered here (to add)"),
-      HTML("<li>The full source code for the study will be made available once the study is finalized"),
-      HTML("</ul>"),
-      h3("Development Status"),
-      p(
-        " The results in this application are currently under review and should be treated as preliminary at this moment."
-      )
-    )
-    ,
-    tabItem(tabName = "proportion",
-            tabsetPanel(
-              id = "proportionTabsetPanel",
-              tabPanel(
-                "Figure",
-                box(
-                  width = 12,
-                  br(),
-                  shinycssloaders::withSpinner(plotlyOutput("proportionPlot"))
-                )
-              ),
-              tabPanel(
-                "Tables",
-                shinycssloaders::withSpinner(dataTableOutput("proportionTable"))
-              )
-            )
-    )
-    ,
-    
-    tabItem(tabName = "survival",
-                box(
-                  width = 12,
-                  br(),
-                  shinycssloaders::withSpinner(plotOutput("survivalPlot"))
-                )
-              
-            
-    ),
-    
-    tabItem(tabName = "characterization",
-            tabsetPanel(
-              id = "characterizationTabsetPanel",
-              tabPanel(
-                "Tables",
-                shinycssloaders::withSpinner(dataTableOutput("characterizationTable"))
-              ),
-              tabPanel(
-                "Figure",
-                box(
-                  width = 12,
-                  br(),
-                  shinycssloaders::withSpinner(plotlyOutput("characterizationPlot"))
-                )
-              )
-            )
-    ),
-    tabItem(tabName = "distribution",
-            tabsetPanel(
-              id = "distributionTabsetPanel",
-              tabPanel(
-                "Tables",
-                 dataTableOutput("distributionTable")
-              ),
-              tabPanel(
-                "Time Trend",
-                box(
-                  title = textOutput("distributionTimePlotTitle"),
-                  width = 12,
-                  br(),
-                  shinycssloaders::withSpinner(plotlyOutput("distributionTimePlot"))
-                )
-              ),
-              tabPanel(
-                "Box Plot",
-                box(
-                  width = 12,
-                  br(),
-                  shinycssloaders::withSpinner(plotOutput("distributionBoxPlot"))
-                )
-              )
-              
-            )
-    )
-
-  )
-  )
-)
diff --git a/inst/shiny/DiagnosticsExplorer/www/favicon.ico b/inst/shiny/DiagnosticsExplorer/www/favicon.ico
deleted file mode 100644
index 849a1fa44..000000000
Binary files a/inst/shiny/DiagnosticsExplorer/www/favicon.ico and /dev/null differ
diff --git a/inst/shiny/DiagnosticsExplorer/www/logo.png b/inst/shiny/DiagnosticsExplorer/www/logo.png
deleted file mode 100644
index c6307af6b..000000000
Binary files a/inst/shiny/DiagnosticsExplorer/www/logo.png and /dev/null differ
diff --git a/inst/shiny/PLPViewer/PLPViewer.Rproj b/inst/shiny/PLPViewer/PLPViewer.Rproj
deleted file mode 100644
index 8e3c2ebc9..000000000
--- a/inst/shiny/PLPViewer/PLPViewer.Rproj
+++ /dev/null
@@ -1,13 +0,0 @@
-Version: 1.0
-
-RestoreWorkspace: Default
-SaveWorkspace: Default
-AlwaysSaveHistory: Default
-
-EnableCodeIndexing: Yes
-UseSpacesForTab: Yes
-NumSpacesForTab: 2
-Encoding: UTF-8
-
-RnwWeave: Sweave
-LaTeX: pdfLaTeX
diff --git a/inst/shiny/PLPViewer/databaseExtras.R b/inst/shiny/PLPViewer/databaseExtras.R
deleted file mode 100644
index 07aeab2ed..000000000
--- a/inst/shiny/PLPViewer/databaseExtras.R
+++ /dev/null
@@ -1,407 +0,0 @@
-# repo files
-
-editColnames <- function(cnames, edits){
-  lwcnames <- tolower(cnames)
-  
-  for(edit in edits){
-    if(tolower(edit)%in%lwcnames){
-      cnames[tolower(edit)==lwcnames] <- edit
-    }
-  }
-  return(cnames)
-  
-}
-
-editTar <- function(summaryTable){
-  
-  summaryTable <- summaryTable %>% dplyr::mutate(TAR = paste0('(',trimws(.data$tarStartAnchor),' + ',.data$tarStartDay, ') - (',trimws(.data$tarEndAnchor),' + ',.data$tarEndDay, ')' )) %>%
-    dplyr::select(-c(.data$tarStartAnchor, .data$tarStartDay, .data$tarEndAnchor, .data$tarEndDay))
-  
-  return(summaryTable)
-}
-
-
-getDbSummary <- function(con, mySchema, targetDialect, myTableAppend = '' ){
-    ParallelLogger::logInfo("gettingDb summary")
-
-    sql <- "SELECT distinct s.study_id, 
-     results.result_id, 
-     results.model_id as analysis_id, 
-     results.researcher_id, 
-     d.database_acronym AS Dev, 
-     d.database_acronym AS Val,
-     targets.cohort_name AS T, outcomes.cohort_name AS O,
-       model_settings.model_type AS model, 
-       model_designs.covariate_setting_id, 
-       tars.tar_start_day, tars.tar_start_anchor, tars.tar_end_day, tars.tar_end_anchor,
-       ROUND(aucResult.auc, 3) as auc,
-       ROUND(auprcResult.auprc,4) as auprc,
-       nResult.population_size, 
-       oResult.outcome_count,
-       ROUND(nTest.test_size*100.0/nResult.population_size, 1) as eval_percent,
-       ROUND(oResult.outcome_count*100.0/nResult.population_size,4) as outcome_percent
-       
-       FROM @my_schema.@my_table_appendresults AS results INNER JOIN @my_schema.@my_table_appendmodels AS models 
-          ON results.model_id = models.model_id and
-             results.database_id = models.database_id
-             
-    inner join @my_schema.@my_table_appendmodel_designs as model_designs
-    on model_designs.model_design_id = models.model_design_id and
-    results.target_id = model_designs.target_id and 
-             results.outcome_id = model_designs.outcome_id and 
-             results.tar_id = model_designs.tar_id and
-             results.population_setting_id = model_designs.population_setting_id
-         
-    inner join @my_schema.@my_table_appendmodel_settings as model_settings
-    on model_settings.model_setting_id = model_designs.model_setting_id
-    
-             
-        INNER JOIN @my_schema.@my_table_appendstudy_models AS s on models.model_id = s.model_id
-       
-             
-        LEFT JOIN (SELECT cohort_id, cohort_name FROM @my_schema.@my_table_appendcohorts) AS targets ON results.target_id = targets.cohort_id
-        LEFT JOIN (SELECT cohort_id, cohort_name FROM @my_schema.@my_table_appendcohorts) AS outcomes ON results.outcome_id = outcomes.cohort_id
-        LEFT JOIN @my_schema.@my_table_appenddatabase_details AS d ON results.database_id = d.database_id 
-        LEFT JOIN @my_schema.@my_table_appendtars AS tars ON results.tar_id = tars.tar_id
-        LEFT JOIN (SELECT result_id, value AS auc FROM @my_schema.@my_table_appendevaluation_statistics where metric = 'AUROC' and evaluation in ('Test','Validation') ) AS aucResult ON results.result_id = aucResult.result_id
-        LEFT JOIN (SELECT result_id, value AS auprc FROM @my_schema.@my_table_appendevaluation_statistics where metric = 'AUPRC' and evaluation in ('Test','Validation') ) AS auprcResult ON results.result_id = auprcResult.result_id
-        LEFT JOIN (SELECT result_id, sum(value) AS population_size FROM @my_schema.@my_table_appendevaluation_statistics where metric = 'populationSize' and evaluation in ('Test','Train') group by result_id) AS nResult ON results.result_id = nResult.result_id
-        LEFT JOIN (SELECT result_id, sum(value) AS outcome_count FROM @my_schema.@my_table_appendevaluation_statistics where metric = 'outcomeCount' and evaluation in ('Test','Train') group by result_id) AS oResult ON results.result_id = oResult.result_id
-        LEFT JOIN (SELECT result_id, value AS test_size FROM @my_schema.@my_table_appendevaluation_statistics where metric = 'populationSize' and evaluation = 'Test') AS nTest ON results.result_id = nTest.result_id;"
-    
-    sql <- SqlRender::render(sql = sql, 
-                             my_schema = mySchema,
-                             my_table_append = myTableAppend)
-    
-    sql <- SqlRender::translate(sql = sql, targetDialect =  targetDialect)
-    
-    summaryTable <- DatabaseConnector::dbGetQuery(conn =  con, statement = sql) 
-    colnames(summaryTable) <- SqlRender::snakeCaseToCamelCase(colnames(summaryTable))
-    
-    summaryTable$t <- trimws(summaryTable$t)
-    summaryTable$o <- trimws(summaryTable$o)
-    
-    summaryTable <- summaryTable %>% 
-      dplyr::rename(`Covariate setting` = covariateSettingId) %>%
-      dplyr::rename(`T Size` = populationSize) %>% 
-      dplyr::rename(`O Count` = outcomeCount) %>%
-      dplyr::rename(`Val (%)` = evalPercent) %>%
-      dplyr::rename(`O Incidence (%)` = outcomePercent)
-    
-    summaryTable <- editTar(summaryTable)
-    
-    colnames(summaryTable) <- editColnames(cnames = colnames(summaryTable), 
-                                           edits = c('AUC','AUPRC', 'T', 'O', 'Dev','Val', 'TAR', 'Model'))
-    
-    summaryTable$timeStamp <- 0
-    summaryTable$Analysis <- summaryTable$analysisId
-    ParallelLogger::logInfo("Got db summary")
-    return(summaryTable[,c('Dev', 'Val', 'T','O', 'Model','Covariate setting',
-                           'TAR', 'AUC', 'AUPRC', 
-                           'T Size', 'O Count','Val (%)', 'O Incidence (%)', 'timeStamp', 'analysisId', 'researcherId', 'resultId', 'Analysis', 'studyId')])
-
-}
-
-
-getValSummary <- function(con, mySchema, modelId, targetDialect, myTableAppend = '' ){
-  ParallelLogger::logInfo("getting Val summary")
-  
-  sql <- "SELECT results.result_id, results.model_id as analysis_id, 
-                results.researcher_id, 
-                                --databases.database_acronym AS Dev, 
-                                d.database_acronym AS Val,
-                                targets.cohort_name AS T, outcomes.cohort_name AS O,
-   model_settings.model_type AS model, 
-   model_designs.covariate_setting_id, 
-   tars.tar_start_day, tars.tar_start_anchor, tars.tar_end_day, tars.tar_end_anchor,
-   ROUND(aucResult.auc, 3) as auc,
-   ROUND(auprcResult.auprc,4) as auprc,
-   nResult.population_size, 
-   oResult.outcome_count,
-   ROUND(nTest.test_size*100.0/nResult.population_size, 1) as eval_percent,
-   ROUND(oResult.outcome_count*100.0/nResult.population_size,4) as outcome_percent,
-   ROUND(calibration_in_large, 3) as calibration_in_large
-   
-   FROM @my_schema.@my_table_appendresults AS results INNER JOIN @my_schema.@my_table_appendmodels AS models
-    ON 
-         results.model_id = models.model_id AND
-         results.model_id = @model_id
-         
-    inner join @my_schema.@my_table_appendmodel_designs as model_designs
-    on model_designs.model_design_id = models.model_design_id and
-             results.tar_id = model_designs.tar_id and
-         results.population_setting_id = model_designs.population_setting_id
-         
-    inner join @my_schema.@my_table_appendmodel_settings as model_settings
-    on model_settings.model_setting_id = model_designs.model_setting_id
-    
-    LEFT JOIN (SELECT cohort_id, cohort_name FROM @my_schema.@my_table_appendcohorts) AS targets ON results.target_id = targets.cohort_id
-    LEFT JOIN (SELECT cohort_id, cohort_name FROM @my_schema.@my_table_appendcohorts) AS outcomes ON results.outcome_id = outcomes.cohort_id
-    LEFT JOIN @my_schema.@my_table_appenddatabase_details AS d ON results.database_id = d.database_id 
-    LEFT JOIN @my_schema.@my_table_appendtars AS tars ON results.tar_id = tars.tar_id
-    LEFT JOIN (SELECT result_id, value AS auc FROM @my_schema.@my_table_appendevaluation_statistics where metric = 'AUROC' and evaluation in ('Test','Validation') ) AS aucResult ON results.result_id = aucResult.result_id
-    LEFT JOIN (SELECT result_id, value AS auprc FROM @my_schema.@my_table_appendevaluation_statistics where metric = 'AUPRC' and evaluation in ('Test','Validation') ) AS auprcResult ON results.result_id = auprcResult.result_id
-    
-    LEFT JOIN (SELECT result_id, value AS calibration_in_large FROM @my_schema.@my_table_appendevaluation_statistics where metric = 'calibrationInLarge intercept' and evaluation in ('Test','Validation') ) AS CalibrationInLargeResult ON results.result_id = CalibrationInLargeResult.result_id
-
-    LEFT JOIN (SELECT result_id, sum(value) AS population_size FROM @my_schema.@my_table_appendevaluation_statistics where metric = 'populationSize' and evaluation in ('Test','Train','Validation') group by result_id) AS nResult ON results.result_id = nResult.result_id
-    LEFT JOIN (SELECT result_id, sum(value) AS outcome_count FROM @my_schema.@my_table_appendevaluation_statistics where metric = 'outcomeCount' and evaluation in ('Test','Train','Validation') group by result_id) AS oResult ON results.result_id = oResult.result_id
-    LEFT JOIN (SELECT result_id, value AS test_size FROM @my_schema.@my_table_appendevaluation_statistics where metric = 'populationSize' and evaluation in ('Test','Validation')) AS nTest ON results.result_id = nTest.result_id;"
-  
-  sql <- SqlRender::render(sql = sql, 
-                           my_schema = mySchema, 
-                           model_id = modelId,
-                           my_table_append = myTableAppend)
-
-  sql <- SqlRender::translate(sql = sql, targetDialect =  targetDialect)
-  
-  valTable <- DatabaseConnector::dbGetQuery(conn =  con, statement = sql) 
-  colnames(valTable) <- SqlRender::snakeCaseToCamelCase(colnames(valTable))
-  
-  valTable$t <- trimws(valTable$t)
-  valTable$o <- trimws(valTable$o)
-  
-  valTable <- valTable %>% 
-    dplyr::rename(`Covariate setting` = covariateSettingId) %>%
-    dplyr::rename(`T Size` = populationSize) %>% 
-    dplyr::rename(`O Count` = outcomeCount) %>%
-    dplyr::rename(`Val (%)` = evalPercent) %>%
-    dplyr::rename(`O Incidence (%)` = outcomePercent)
-  
-  valTable <- editTar(valTable)
-  
-  #colnames(valTable) <- editColnames(cnames = colnames(valTable), 
-  #                                   edits = c('AUC','AUPRC', 'T', 'O', 'Dev','Val', 'TAR', 'Model'))
-  colnames(valTable) <- editColnames(cnames = colnames(valTable), 
-                                     edits = c('AUC','AUPRC', 'T', 'O','Val', 'TAR', 'Model'))
-  
-  valTable$timeStamp <- 0
-  valTable$Analysis <- valTable$analysisId
-  ParallelLogger::logInfo("got db summary")
-  #return(valTable[,c('Dev', 'Val', 'T','O', 'Model','Covariate setting',
-  return(valTable[,c('Val', 'T','O', 'Model','Covariate setting',
-                     'TAR', 'AUC', 'AUPRC', 'calibrationInLarge',
-                     'T Size', 'O Count','Val (%)', 'O Incidence (%)', 'timeStamp', 'analysisId', 'researcherId', 'resultId', 'Analysis')])
-  
-}
-
-getResult <- function(con, tableName, resultId, mySchema, targetDialect){
-  sql <- "SELECT * FROM @my_schema.@table_name WHERE result_id = @result_id"
-  sql <- SqlRender::render(sql = sql, 
-                           my_schema = mySchema,
-                           table_name = tableName,
-                           result_id = resultId)
-  sql <- SqlRender::translate(sql = sql, targetDialect =  targetDialect)
-  
-  result <- DatabaseConnector::dbGetQuery(conn =  con, statement = sql) 
-  colnames(result) <- SqlRender::snakeCaseToCamelCase(colnames(result))
-  return(result)
-}
-
-
-loadPlpFromDb <- function(chosenRow, mySchema, con, val = F, targetDialect, myTableAppend = ''){
-  resultId <- chosenRow$resultId
-  modelId <- chosenRow$analysisId
-  researcherId <- chosenRow$researcherId
-  result <- list()
-  result$performanceEvaluation <- list()
-
-  if (!val){
-    print(paste0('model: ', modelId))
-    ## get hyper_param_search plpResult$model$hyperParamSearch <- ...
-    #old sql <- "SELECT population_setting_id, model_setting_id, covariate_setting_id, hyper_param_search FROM @my_schema.@my_table_appendmodels AS models WHERE model_id = @model_id;"
-    sql <- "SELECT population_setting_id, model_setting_id, covariate_setting_id, hyper_param_search FROM 
-    @my_schema.@my_table_appendmodel_designs md inner join 
-    @my_schema.@my_table_appendmodels AS models 
-    on models.model_design_id = md.model_design_id
-    WHERE models.model_id = @model_id;"
-    
-    sql <- SqlRender::render(sql = sql, 
-                             my_schema = mySchema,
-                             model_id = modelId,
-                             my_table_append = myTableAppend)
-    sql <- SqlRender::translate(sql = sql, targetDialect =  targetDialect)
-    ParallelLogger::logInfo("starting population, model setting and covariate setting")
-
-    ids <- DatabaseConnector::dbGetQuery(conn =  con, statement = sql) 
-    colnames(ids) <- SqlRender::snakeCaseToCamelCase(colnames(ids))
-    
-    ParallelLogger::logInfo("finishing population, model setting and covariate setting")
-    
-    popSetId <- ids$populationSettingId
-    modSetId <- ids$modelSettingId
-    covSetId <- ids$covariateSettingId
-    
-    hyperParamSearch <- jsonlite::unserializeJSON(ids$hyperParamSearch)
-   
-    #covariateSummary 
-    #made this null to speed up programme
-    result$covariateSummary <- NULL
-   
-    #inputSetting
-    result$model <- list(settings = list())
-    
-    sql <- "SELECT * FROM @my_schema.@my_table_appendmodel_settings AS model_setting WHERE model_setting_id = @model_setting_id"
-    sql <- SqlRender::render(sql = sql, 
-                             my_schema = mySchema,
-                             model_setting_id = modSetId,
-                             my_table_append = myTableAppend)
-    sql <- SqlRender::translate(sql = sql, targetDialect =  targetDialect)
-    ParallelLogger::logInfo("start modeSet")
-
-    tempModSettings <- DatabaseConnector::dbGetQuery(conn =  con, statement = sql) 
-    colnames(tempModSettings) <- SqlRender::snakeCaseToCamelCase(colnames(tempModSettings))
-    ParallelLogger::logInfo("end modeSet")
-    
-    if(length(tempModSettings$modelSettingsJson)>0){
-      result$model$settings$modelSettings <- jsonlite::unserializeJSON(tempModSettings$modelSettingsJson)
-    } else{
-      result$model$settings$modelSettings <- list('missing', list(param = 'na'))
-    }
-    
-    sql <- "SELECT * FROM @my_schema.@my_table_appendcovariate_settings AS covariate_setting WHERE covariate_setting_id = @covariate_setting_id"
-    sql <- SqlRender::render(sql = sql, 
-                             my_schema = mySchema,
-                             covariate_setting_id = covSetId,
-                             my_table_append = myTableAppend
-                             )
-    sql <- SqlRender::translate(sql = sql, targetDialect =  targetDialect)
-    ParallelLogger::logInfo("start covSet")
-    tempCovSettings  <- DatabaseConnector::dbGetQuery(conn =  con, statement = sql) 
-    colnames(tempCovSettings) <- SqlRender::snakeCaseToCamelCase(colnames(tempCovSettings))
-    ParallelLogger::logInfo("end covSet")
-    
-    if(length(tempCovSettings$covariateSettingsJson)>0){
-      # old result$inputSetting$dataExtrractionSettings$covariateSettings <- jsonlite::unserializeJSON(tempCovSettings$covariateSettingsJson)
-      result$model$settings$covariateSettings <- jsonlite::fromJSON(tempCovSettings$covariateSettingsJson, simplifyVector = T, simplifyDataFrame = F, simplifyMatrix = T)
-      
-      extractAttributes <- function(x){
-        
-        ind <- grep('attr_', names(x))
-        
-        if(length(ind)>0){
-          attributeValues <- x[ind]
-          x <- x[-ind]
-          names(attributeValues) <- gsub(pattern = 'attr_',replacement = '',x = names(attributeValues))
-          attributeValues$names <- names(x)
-          attributes(x) <- attributeValues
-        }
-        
-        return(x)
-      }
-      
-      result$model$settings$covariateSettings <- lapply(result$model$settings$covariateSettings, function(x) extractAttributes(x))
-  } else{
-    result$model$settings$covariateSettings <- list()
-  }
-    
-    
-    sql <- "SELECT * FROM @my_schema.@my_table_appendpopulation_settings AS population_settings WHERE population_setting_id = @population_setting_id"
-    ParallelLogger::logInfo("start popSet")
-    sql <- SqlRender::render(sql = sql, 
-                             my_schema = mySchema,
-                             population_setting_id = popSetId,
-                             my_table_append = myTableAppend)
-    sql <- SqlRender::translate(sql = sql, targetDialect =  targetDialect)
-    
-    tempPopSettings  <- DatabaseConnector::dbGetQuery(conn =  con, statement = sql) 
-    colnames(tempPopSettings) <- SqlRender::snakeCaseToCamelCase(colnames(tempPopSettings))
-    ParallelLogger::logInfo("end popSet")
-    
-    if(length(tempPopSettings$populationSettingsJson)>0){
-      result$model$settings$populationSettings <- jsonlite::unserializeJSON(tempPopSettings$populationSettingsJson)
-    } else{
-      result$model$settings$populationSettings <- NULL
-    }
-    
-    # attrition
-    sql <- "SELECT * FROM @my_schema.@my_table_appendattrition WHERE result_id = @result_id"
-    ParallelLogger::logInfo("start attrition")
-    sql <- SqlRender::render(sql = sql, 
-      my_schema = mySchema,
-      result_id = resultId,
-      my_table_append = myTableAppend)
-    sql <- SqlRender::translate(sql = sql, targetDialect =  targetDialect)
-    
-    tempAttrition  <- DatabaseConnector::dbGetQuery(conn =  con, statement = sql) 
-    colnames(tempAttrition) <- SqlRender::snakeCaseToCamelCase(colnames(tempAttrition))
-    ParallelLogger::logInfo("end popSet")
-    
-    result$model$trainDetails$attrition <- as.data.frame(tempAttrition)
-    
-    result$performanceEvaluation$demographicSummary <- getResult(con, paste0(myTableAppend,'demographic_summary'), resultId, mySchema, targetDialect = targetDialect)
-    result$performanceEvaluation$demographicSummary$evaluation <- trimws(result$performanceEvaluation$demographicSummary$evaluation)
-    result$performanceEvaluation$demographicSummary$ageGroup <- trimws(result$performanceEvaluation$demographicSummary$ageGroup)
-    result$performanceEvaluation$demographicSummary$genGroup <- trimws(result$performanceEvaluation$demographicSummary$genGroup)
-    colnames(result$performanceEvaluation$demographicSummary) <- editColnames(colnames(result$performanceEvaluation$demographicSummary), c('evaluation',"PersonCountAtRisk","PersonCountWithOutcome", "StDevPredictedProbability",
-                                                                                                                                           "MinPredictedProbability", "P25PredictedProbability", "P50PredictedProbability",
-                                                                                                                                           "P75PredictedProbability", "MaxPredictedProbability"))
-    result$performanceEvaluation$predictionDistribution <- getResult(con, paste0(myTableAppend,'prediction_distribution'), resultId,mySchema, targetDialect = targetDialect ) 
-    result$performanceEvaluation$predictionDistribution$evaluation <- trimws(result$performanceEvaluation$predictionDistribution$evaluation)
-    result$performanceEvaluation$predictionDistribution$class <- result$performanceEvaluation$predictionDistribution$classLabel
-    colnames(result$performanceEvaluation$predictionDistribution) <- editColnames(colnames(result$performanceEvaluation$predictionDistribution), c('evaluation', "PersonCount", "StDevPredictedProbability", 
-                                                                                                                                                   "MinPredictedProbability", "P05PredictedProbability" , 
-                                                                                                                                                   "P25PredictedProbability", "MedianPredictedProbability", 
-                                                                                                                                                   "P75PredictedProbability" , "P95PredictedProbability","MaxPredictedProbability"))
-    
-    
-    result$model$trainDetails$hyperParamSearch <- hyperParamSearch
-  
-  }
-  
-  #performanceEvaluation
-  result$performanceEvaluation$evaluationStatistics <- getResult(con, paste0(myTableAppend,'evaluation_statistics'), resultId, mySchema, targetDialect = targetDialect )
-  
-  result$performanceEvaluation$thresholdSummary <- getResult(con, paste0(myTableAppend,'threshold_summary'), resultId,mySchema, targetDialect = targetDialect)
-  result$performanceEvaluation$thresholdSummary$evaluation <- trimws(result$performanceEvaluation$thresholdSummary$evaluation)
-  
-  result$performanceEvaluation$calibrationSummary <- getResult(con, paste0(myTableAppend,'calibration_summary'), resultId,mySchema, targetDialect = targetDialect)
-  result$performanceEvaluation$calibrationSummary$evaluation <- trimws(result$performanceEvaluation$calibrationSummary$evaluation)
-  colnames(result$performanceEvaluation$calibrationSummary) <- editColnames(colnames(result$performanceEvaluation$calibrationSummary), c('evaluation',"PersonCountAtRisk","PersonCountWithOutcome","StDevPredictedProbability",
-                                                                                                                                         "MinPredictedProbability", "P25PredictedProbability", "MedianPredictedProbability",
-                                                                                                                                         "P75PredictedProbability", "MaxPredictedProbability"))
-  
-
-  sql <- "SELECT researcher_name, researcher_email FROM @my_schema.@my_table_appendresearchers AS researchers WHERE researcher_id = @researcher_id"
-  ParallelLogger::logInfo("start researchers")
-  sql <- SqlRender::render(sql = sql, 
-                           my_schema = mySchema,
-                           researcher_id = researcherId,
-                           my_table_append = myTableAppend) 
-  sql <- SqlRender::translate(sql = sql, targetDialect =  targetDialect)
-  
-  result$researcherInfo <- DatabaseConnector::dbGetQuery(conn =  con, statement = sql) 
-  colnames(result$researcherInfo) <- SqlRender::snakeCaseToCamelCase(colnames(result$researcherInfo))
-  
-  ParallelLogger::logInfo("end researchers")
-  result$model_id <- modelId
-  
-  # add intercept
-  ParallelLogger::logInfo("start intercept")
-  sql <- "SELECT intercept FROM @my_schema.@my_table_appendmodels AS models WHERE model_id = @model_id"
-  sql <- SqlRender::render(sql = sql, 
-                           my_schema = mySchema,
-                           model_id = modelId,
-                           my_table_append = myTableAppend)
-  sql <- SqlRender::translate(sql = sql, targetDialect =  targetDialect)
-  
-  coefficient <- DatabaseConnector::dbGetQuery(conn =  con, statement = sql) 
-  colnames(coefficient) <- SqlRender::snakeCaseToCamelCase(colnames(coefficient))
-  
-  result$model$model <- list(coefficient = coefficient$intercept[1])
-  
-  #hack so the check in plot,utlipl.. doesnt break it
-  result$analysisRef <- ""
-  result$executionSummary <- ""
-  class(result) <- "runPlp"
-  ParallelLogger::logInfo("end")
-  return(result)
-}
-
-
-
-
-
-
-
-
diff --git a/inst/shiny/PLPViewer/emptyPlot.R b/inst/shiny/PLPViewer/emptyPlot.R
deleted file mode 100644
index cac00fec5..000000000
--- a/inst/shiny/PLPViewer/emptyPlot.R
+++ /dev/null
@@ -1,14 +0,0 @@
-emptyPlot <- function(title = NULL){
-  p <- plotly::plotly_empty(type = "scatter", mode = "markers") %>%
-    plotly::config(
-      displayModeBar = FALSE
-    ) %>%
-    plotly::layout(
-      title = list(
-        text = title,
-        yref = "paper",
-        y = 0.5
-      )
-    )
-  return(p)
-} 
diff --git a/inst/shiny/PLPViewer/global.R b/inst/shiny/PLPViewer/global.R
deleted file mode 100644
index 7fd64720e..000000000
--- a/inst/shiny/PLPViewer/global.R
+++ /dev/null
@@ -1,118 +0,0 @@
-# EDIT FOR REPO OR DATABASE
-useDatabase <- .GlobalEnv$shinySettings$useDatabase
-usePlpObject <- .GlobalEnv$shinySettings$usePlpObject
-useFileSystem <- .GlobalEnv$shinySettings$useFileSystem
-
-pathToMd <- "./www/shinyDescription.md"
-
-
-# set default database values
-connectionDetails <- NULL
-mySchema <- NULL
-targetDialect <- NULL
-myTableAppend <- ''
-
-# extract data if database 
-if(useDatabase){
-  
-  ParallelLogger::logInfo('Extracting results from database')
-  
-  result <- 'database'
-  inputType <- 'database'
-  validation <- NULL
-  
-  source("databaseExtras.R")
-  mySchema <- Sys.getenv("shinydbSchema")
-  myServer <- Sys.getenv("shinydbServer")
-  myUser <- Sys.getenv("shinydbUser")
-  myPassword <- Sys.getenv("shinydbPw")
-  targetDialect <- Sys.getenv("shinydbDbms")
-  myPort <- Sys.getenv("shinydbPort")
-
-  myTableAppend <- Sys.getenv("shinydbTableAppend")
-  
-  if(myPort != ""){
-    ParallelLogger::logInfo('Port')
-    ParallelLogger::logInfo(paste(myPort))
-    con <- pool::dbPool(drv = DatabaseConnector::DatabaseConnectorDriver(),
-                        dbms = targetDialect,
-                        server = myServer,
-                        user = myUser,
-                        password = myPassword,
-                        port = myPort)
-    
-  } else{
-    ParallelLogger::logInfo('No Port')
-    con <- pool::dbPool(drv = DatabaseConnector::DatabaseConnectorDriver(),
-                        dbms = targetDialect,
-                        server = myServer,
-                        user = myUser,
-                        password = myPassword)
-    
-  }
-  
-  onStop(function() {
-    if (DBI::dbIsValid(con)) {
-      ParallelLogger::logInfo("Closing connection pool")
-      pool::poolClose(con)
-    }
-  })
-
-  summaryTable <- getDbSummary(con = con, 
-                               mySchema = mySchema, 
-                               targetDialect = targetDialect,
-                               myTableAppend = myTableAppend)
-  
-} 
-
-
-# if plpObect
-if(usePlpObject){
-  source("processing.R")
-  ParallelLogger::logInfo('Loading results from plpObject')
-  
-  if(!is.null(.GlobalEnv$shinySettings$result)){
-    
-    result <- .GlobalEnv$shinySettings$result
-    
-    if(class(result)=='runPlp'){
-      inputType <-  'plpResult'
-    } else if(sum(names(result)%in%c("prediction","performanceEvaluation","inputSetting","executionSummary","model","analysisRef","covariateSummary"))==7){
-      inputType <- 'plpNoClass'
-    } else {
-      stop('Incorrect class for input result')
-    } 
- 
-    validation <- .GlobalEnv$shinySettings$validation
-    
-    summaryTable <- getSummary(inputType = inputType, 
-                               result = result,
-                               validation = validation)
-  }
-  
-}
-
-# if fileSystem
-if(useFileSystem){
-  source("processing.R")
-  ParallelLogger::logInfo('Loading results from file system')
-  
-  if(!is.null(.GlobalEnv$shinySettings$result)){
-    
-    valid <- ifelse(class(.GlobalEnv$shinySettings$result)=='character', dir.exists(.GlobalEnv$shinySettings$result),F)
-    
-    if(valid){
-  
-    result <- 'file'
-    validation <- NULL
-    inputType <- 'file'
-    summaryTable <- getSummary(inputType = 'file', 
-                               result = .GlobalEnv$shinySettings$result)
-    
-    }else{
-      print(paste0('invalid directory: ', .GlobalEnv$shinySettings$result))
-    }
-  }
-}
-
-
diff --git a/inst/shiny/PLPViewer/helpers.R b/inst/shiny/PLPViewer/helpers.R
deleted file mode 100644
index 250ef7f7c..000000000
--- a/inst/shiny/PLPViewer/helpers.R
+++ /dev/null
@@ -1,75 +0,0 @@
-# need to add mySchema and connectionDetails to input
-getPlpResult <- function(result, 
-                         validation,
-                         summaryTable, 
-                         inputType,
-                         resultRow, 
-                         val = F, 
-                         mySchema = NULL, 
-                         connectionDetails = NULL,
-                         targetDialect = NULL, 
-                         myTableAppend = NULL){
-  
-##ind <- resultRow()
-
-  if(!is.null(resultRow())){
-    print('Loading data')
-    print(paste0('input: ', inputType))
-    
-  if(inputType == 'database'){
-    tempResult <- loadPlpFromDb(summaryTable[resultRow(),], mySchema, con, val = val, targetDialect, myTableAppend)
-    return(tempResult)
-  } else if(inputType == 'plpResult'){
-    i <- resultRow()
-    if(i == 1){
-      tempResult <- result
-      tempResult$type <- 'test'
-    }else{
-      tempResult <- validation$validation[[i-1]]
-      tempResult$type <- 'validation'
-    }
-  }else if(inputType == 'plpNoClass'){
-    tempResult <- result
-    tempResult$type <- 'validation'
-  }else if( inputType == 'file') {
-    
-    # support rds, csv and runPlp objects
-    tempResult <- NULL
-    loc <- summaryTable[resultRow(),]$plpResultLocation
-    locLoaderFunc <- summaryTable[resultRow(),]$plpResultLoad
-
-    if(dir.exists(as.character(loc))){
-      tempResult <- do.call(as.character(locLoaderFunc), list(as.character(loc)))
-      tempResult$type <- ifelse(length(grep('/Validation',loc))>0,'validation','test')
-    }
-  }else {
-    stop('Incorrect class')
-  }
-  return(tempResult)
-  } else{
-    return(NULL)
-  }
-}
-
-
-
-
-addInfo <- function(item, infoId) {
-  infoTag <- tags$small(class = "badge pull-right action-button",
-                        style = "padding: 1px 6px 2px 6px; background-color: steelblue;",
-                        type = "button", 
-                        id = infoId,
-                        "i")
-  item$children[[1]]$children <- append(item$children[[1]]$children, list(infoTag))
-  return(item)
-}
-    
-showInfoBox <- function(title, htmlFileName) {
-  shiny::showModal(shiny::modalDialog(
-    title = title,
-    easyClose = TRUE,
-    footer = NULL,
-    size = "l",
-    shiny::HTML(readChar(htmlFileName, file.info(htmlFileName)$size) )
-  ))
-}
diff --git a/inst/shiny/PLPViewer/html/DataInfo.html b/inst/shiny/PLPViewer/html/DataInfo.html
deleted file mode 100644
index 2b8e163f8..000000000
--- a/inst/shiny/PLPViewer/html/DataInfo.html
+++ /dev/null
@@ -1,4 +0,0 @@
-<h3>Description</h3>
-  <p> This button provides information about the data used in the app </p>
-
-  
\ No newline at end of file
diff --git a/inst/shiny/PLPViewer/html/Description.html b/inst/shiny/PLPViewer/html/Description.html
deleted file mode 100644
index 72e455ca2..000000000
--- a/inst/shiny/PLPViewer/html/Description.html
+++ /dev/null
@@ -1,3 +0,0 @@
-<h3>Description</h3>
-  <p> Information about the study and links to the code used to run the study </p>
-  
\ No newline at end of file
diff --git a/inst/shiny/PLPViewer/html/Help.html b/inst/shiny/PLPViewer/html/Help.html
deleted file mode 100644
index 68c76d851..000000000
--- a/inst/shiny/PLPViewer/html/Help.html
+++ /dev/null
@@ -1,5 +0,0 @@
-<h3>Description</h3>
-  <p> This button provides a link to a YouTube video with a demonstration of the shiny app </p>
-
-
-  
\ No newline at end of file
diff --git a/inst/shiny/PLPViewer/html/Log.html b/inst/shiny/PLPViewer/html/Log.html
deleted file mode 100644
index e47baf294..000000000
--- a/inst/shiny/PLPViewer/html/Log.html
+++ /dev/null
@@ -1,20 +0,0 @@
-<h3>Description</h3>
-  
-
-<p> This button shows the log when the model was developed or validated </p>
-
-  
-
-<h3>Selecting Model</h3>
-  
-
-<p>To select a model to inspect either:</p>
-  
-<ul>
-  <li> i) Select it from the dropdowm menu on the left hand side</li>
-  
-      <li> or ii) Click on a row in the summary view to select it - this will show as the row will be highlighted.</li>
-  
-</ul>
-
-  
\ No newline at end of file
diff --git a/inst/shiny/PLPViewer/html/Model.html b/inst/shiny/PLPViewer/html/Model.html
deleted file mode 100644
index 41fbfc0b6..000000000
--- a/inst/shiny/PLPViewer/html/Model.html
+++ /dev/null
@@ -1,27 +0,0 @@
-<h3>Description</h3>
-  
-<p> The model button shows a plot and table with the characteristics of the patients with and without the outcome during the time-at-risk. </p>
-  
-  
-
-<h3>Interpretation</h3>
-  <ul>
-  <li>The plots show each covariate as a dot (binary covariates on the left side plot and measurements on the right side plot).  The x-axis is the fraction of patients (or mean value) with the covariate in the patients without the outcome and the y-axis is the fraction of patients (or mean value) with the covariate in the patients with the outcome.  Dots above the x=y line are more common in patients with the outcome and dots below the line are more common in patients without the outcome.</li>
-  <li>The table shows the covariate name, the variable importance or coefficient value, the mean value in those with and without the outcome and the standardized mean difference.</li>
-  </ul>  
-
-  
-
-<h3>Selecting Model</h3>
-  
-
-<p>To select a model to inspect either:</p>
-  
-<ul>
-  <li> i) Select it from the dropdowm menu on the left hand side</li>
-  
-      <li> or ii) Click on a row in the summary view to select it - this will show as the row will be highlighted.</li>
-  
-</ul>
-
-  
\ No newline at end of file
diff --git a/inst/shiny/PLPViewer/html/Performance.html b/inst/shiny/PLPViewer/html/Performance.html
deleted file mode 100644
index e01c313fc..000000000
--- a/inst/shiny/PLPViewer/html/Performance.html
+++ /dev/null
@@ -1,31 +0,0 @@
-<h3>Description</h3>
-  
-
-<p> The performance of the model including the operating characteristics at different risk cutoffs, the overall discrimination and the overall calibration. </p>
-  
-  
-
-<h3>Tabs</h3>
-  
-
-<p>The three tabs are:</p>
-  <ul>
-  <li>The 'Threshold dependant' tab shows the prediction question being explores and various operating characteristics for a range of risk cutoffs (the threshold bar is interactive and enables you to explore different values by moving the bar left or right) </li>
-  <li>The 'Discrimination' tab shows the AUROC, AUPRC, predicted risk distributions and F1 score</li>
-  <li>The 'Calibration' tab shows the generic calibration plot and the calibration per age group and gender.</li>
-  </ul>
-
-
-<h3>Selecting Model</h3>
-  
-
-<p>To select a model to inspect either:</p>
-  
-<ul>
-  <li> i) Select it from the dropdowm menu on the left hand side</li>
-  
-      <li> or ii) Click on a row in the summary view to select it - this will show as the row will be highlighted.</li>
-  
-</ul>
-
-  
diff --git a/inst/shiny/PLPViewer/html/Settings.html b/inst/shiny/PLPViewer/html/Settings.html
deleted file mode 100644
index 9b7b625e5..000000000
--- a/inst/shiny/PLPViewer/html/Settings.html
+++ /dev/null
@@ -1,20 +0,0 @@
-<h3>Description</h3>
-  
-
-<p> Tabs containing the settings (model/population/covariate) selected when developing the model </p>
-  
-  
-
-<h3>Selecting Model</h3>
-  
-
-<p>To select a model to inspect either:</p>
-  
-<ul>
-  <li> i) Select it from the dropdowm menu on the left hand side</li>
-  
-      <li> or ii) Click on a row in the summary view to select it - this will show as the row will be highlighted.</li>
-  
-</ul>
-
-  
diff --git a/inst/shiny/PLPViewer/html/Summary.html b/inst/shiny/PLPViewer/html/Summary.html
deleted file mode 100644
index 8df81cacb..000000000
--- a/inst/shiny/PLPViewer/html/Summary.html
+++ /dev/null
@@ -1,51 +0,0 @@
-<h3>Description</h3>
-  
-<p>A table showing summary information for each validation result. Each row corresponds to a model applied to a specific target population, outcome and time-at-risk triple for a specific database.  Summary details include the validation data size (target population and outcome counts) and discriminative performance. </p>
-  
-  
-
-<h3>Options</h3>
-  
-<p>Click on a row to select it - this will show as the row will be highlighted.  This will populate the following parts of the app for further exploration:</p>
-  
-<ul>
-  
-<li>The complete performance of the result for the selected row can be viewed by clicking on the 'Performance' button in the left menu</li>
-  
-<li>The model corresponding to the result for the selected row can be viewed by clicking on the 'Model' button in the left menu</li>
-  
-<li>The log file corresponding to the result for the selected row can be viewed by clicking on the 'Log' button in the left menu (this is not always available)</li>
-    
-<li>The model development settings for the selected row can be viewed by clicking on the 'Settings' button and then 'Model Settings' tab (this is not always available)</li>
-    
-<li>The population settings (information about time-at-risk and exclusions) for the selected row can be viewed by clicking on the 'Settings' button and then 'Population Settings' tab (this is not always available)</li>
-    
-<li>The covariate settings (information about the model features) for the selected row can be viewed by clicking on the 'Settings' button and then 'Covariate Settings' tab (this is not always available)</li>
-  
-</ul>
-  
-  
-
-<h3>Using the Filter</h3>
-  
-
-<p> Select a specific: </p>
-  
-
-<ul>
-  
-<li>development database - database used to develop the model being validated</li>
-  
-<li>validation database - database used to evaluate the model being validated</li>
-  
-<li>time-at-risk - time period relative to index where the outcome is being predicted</li> 
-  
-<li>target population - the patient population we are interested in predicting the outcome risk for </li> 
-  
-<li>outcome - the event being predicted</li>
-  <li>model - the type of model (e.g., logistic regression, decision tree)</li>
-  
-</ul>
-  
-
-<p>to filter the table rows of interest. </p> 
diff --git a/inst/shiny/PLPViewer/html/boxHelp.html b/inst/shiny/PLPViewer/html/boxHelp.html
deleted file mode 100644
index d9ff1b38b..000000000
--- a/inst/shiny/PLPViewer/html/boxHelp.html
+++ /dev/null
@@ -1,6 +0,0 @@
-<h3>Description</h3>
-  <p> These plots show the box plots displaying the risk distributions for those with the outcome during the time-at-risk (class 1) and those without the outcome during the time-at-risk (class 0) </p>
-  
-<h3>Interpretation</h3>
-  <p> If a model is able to discriminate between those with and without the outcome then it should be assigning a higher risk to those with the outcome, so the box plot for class 1 should be shifted to the right relative to the box plot for class 0. If the model is not able to discriminate then the box plots will look similar. </p>
-
diff --git a/inst/shiny/PLPViewer/html/calHelp.html b/inst/shiny/PLPViewer/html/calHelp.html
deleted file mode 100644
index 9013fc297..000000000
--- a/inst/shiny/PLPViewer/html/calHelp.html
+++ /dev/null
@@ -1,6 +0,0 @@
-<h3>Description</h3>
-  <p> The calibration plots show how closely the predicted risk matched the true observed risk.  The calibration plot is calculated (using labelled data) by partitioning the patients into deciles based on predicted risk and then within each decile the mean predicted risk is calculated and the fraction of patients with the outcome (the observed risk) is calculated.  The calibration plot is then generated by plotting the observed risk against the mean predicted risk for each decile. </p>
-  
-<h3>Interpretation</h3>
-  <p> If a model is well calibrated the mean predicted risk should be approximately the same as the observed risk.  Therefor all 10 dots should fall on the x=y line.  If the dots fall above the x=y line then there is a higher oberved risk than predicted, so our model is assigning lower than the true risk to patients (underestimated risk). If the dots fall below the x=y line then there is a lower observed risk than predicted, so our model is assigning higher than the true risk to patients (overestimated risk).</p>  
-
diff --git a/inst/shiny/PLPViewer/html/demoHelp.html b/inst/shiny/PLPViewer/html/demoHelp.html
deleted file mode 100644
index 2b6eaac56..000000000
--- a/inst/shiny/PLPViewer/html/demoHelp.html
+++ /dev/null
@@ -1,8 +0,0 @@
-<h3>Description</h3>
-  <p> The demographic calibration plots show how closely the predicted risk matched the true observed risk for each age/gender strata.  We partition the patients into age and gender groups, then calculate the mean predicted risk within each age/gender group and the fraction of patients within the group that actually had the outcome during the time-at-risk (observed risk).  We then plot the observed and predicted risk for each age group split by gender.  </p>
-  
-<h3>Interpretation</h3>
-  <p> If a model is well calibrated the mean predicted risk should be approximately the same as the observed risk for each age/gender.  Therefore, the observed risk and predicted risk plots should overlap.  If there is deviation between the predicted risk and observed risk for a certain age group, then this tells us the model is not well calibrated for that age group.  This may indicate the need to fit a model specifically for that age group if there is sufficient data.</p>
-  
-<p> In addition, this plot shows us the age trend of risk  (e.g., you can see whether the risk increases as patients age) and it shows us how males and females differ in terms of risk of the outcome during the time-at-risk.</p> 
-
diff --git a/inst/shiny/PLPViewer/html/f1Help.html b/inst/shiny/PLPViewer/html/f1Help.html
deleted file mode 100644
index e4c59cc6c..000000000
--- a/inst/shiny/PLPViewer/html/f1Help.html
+++ /dev/null
@@ -1,13 +0,0 @@
-<h3>Description</h3>
-  <p> The F1 score plot shows the F1 score for each risk threshold. Click <a href="https://en.wikipedia.org/wiki/F1_score">here</a> for more information about the F1 score.  </p>
-  
-<h3>Interpretation</h3>
-  <p> The F1-score combines the sensitivity and precision of the model into a single measure of accuracy.</p>
-
-<h3>Definitions</h3>
-  <ul> 
-  <li> Sensitivity - probability that somebody with the outcome will be identified as having the outcome by the model at a specified cutoff (e.g., their predicted risk >= specified cutoff)  
-  </li>
-   <li> Precision (positive predictive value) - probability that somebody identified by the model as having the outcome at a specified cutoff truly has the outcome 
-   </li>
-  </ul>
diff --git a/inst/shiny/PLPViewer/html/prcHelp.html b/inst/shiny/PLPViewer/html/prcHelp.html
deleted file mode 100644
index 3416a13ab..000000000
--- a/inst/shiny/PLPViewer/html/prcHelp.html
+++ /dev/null
@@ -1,19 +0,0 @@
-<h3>Description</h3>
-  <p> The precision recall (PR) curve shows the trade-off between precision (positive predictive value) and recall (sensitivity) for all possible risk cutoffs. The area below the curve is a measure of overall discriminative performance. Click <a href="https://acutecaretesting.org/en/articles/precision-recall-curves-what-are-they-and-how-are-they-used">here</a> for more information.  </p>
-  
-<h3>Interpretation</h3>
-  <p> The red dashed line shows the fraction of the target population who have the outcome (the average risk). The main line shows the relationship between the precision and recall.  If the main line is above the red dashed line, then this means the model is able to identify a group of patients who have a higher risk than the average risk, the higher the line is above the red dashed line, the higher the relative risk we can identify for some subset of patients. 
-</p>
-
-<h3>Notes</h3>
-  <p> If the outcome is rare (so the data are imbalanced) a precision recall curve (PRC) gives an insight into the clinical utility of the model as it tells you about the precision of the model </p>  
-  
-<h3>Definitions</h3>
-  <ul> 
-  <li> Sensitivity (recall) - probability that somebody with the outcome will be identified as having the outcome by the model at a specified cutoff (e.g., their predicted risk >= specified cutoff)  
-  </li>
-<li> 
-   Specificity - probability that somebody without the outcome will be identified as a non-outcome by the model at a specified cutoff (e.g., their predicted risk < specified cutoff) </li> 
-   <li> Precision (positive predictive value) - probability that somebody identified by the model as having the outcome at a specified cutoff truly has the outcome 
-   </li>
-  </ul> 
diff --git a/inst/shiny/PLPViewer/html/predDistHelp.html b/inst/shiny/PLPViewer/html/predDistHelp.html
deleted file mode 100644
index bbccd8585..000000000
--- a/inst/shiny/PLPViewer/html/predDistHelp.html
+++ /dev/null
@@ -1,6 +0,0 @@
-<h3>Description</h3>
-  <p> These plots show the probability density function for those with the outcome (red) and those without the outcome (green) </p>
-  
-<h3>Interpretation</h3>
-  <p> If a prediction model is able to discriminate between those who and without the outcome during the time-at-risk then these distributions should be disjoint.  The more overlap between the distributions, the worse the discrimination. </p>
-
diff --git a/inst/shiny/PLPViewer/html/prefDistHelp.html b/inst/shiny/PLPViewer/html/prefDistHelp.html
deleted file mode 100644
index ba0339fab..000000000
--- a/inst/shiny/PLPViewer/html/prefDistHelp.html
+++ /dev/null
@@ -1,6 +0,0 @@
-<h3>Description</h3>
-  <p> These plots show the preference score density function for those with the outcome (red) and those without the outcome (green) </p>
-  
-<h3>Interpretation</h3>
-  <p> If a prediction model is able to discriminate between those who and without the outcome during the time-at-risk then these distributions should be disjoint.  The more overlap between the distributions, the worse the discrimination. </p>
-
diff --git a/inst/shiny/PLPViewer/html/rocHelp.html b/inst/shiny/PLPViewer/html/rocHelp.html
deleted file mode 100644
index f11c74f8d..000000000
--- a/inst/shiny/PLPViewer/html/rocHelp.html
+++ /dev/null
@@ -1,18 +0,0 @@
-<h3>Description</h3>
-  <p> The receiver operating characteristic (ROC) curve shows the trade-off between sensitivity and specificity for all possible risk cutoffs. The area below the curve is a measure of overall discriminative performance. Click <a href="https://en.wikipedia.org/wiki/Receiver_operating_characteristic">here</a> for more information.  </p>
-  
-<h3>Interpretation</h3>
-  <p> If a model is not able to discriminate then the curve will be approximately the x=y line.  A perfectly discriminative model will go up vertically and then across.</p>
-
-<h3>Notes</h3>
-  <p> If the outcome is rare then the ROC curve doesn't provide insight into the precision of the model and a precision recall curve (PRC) should also be inspected. </p>  
-  
-<h3>Definitions</h3>
-  <ul> 
-  <li> Sensitivity - probability that somebody with the outcome will be identified as having the outcome by the model at a specified cutoff (e.g., their predicted risk >= specified cutoff)  
-  </li>
-<li> 
-   Specificity - probability that somebody without the outcome will be identified as a non-outcome by the model at a specified cutoff (e.g., their predicted risk < specified cutoff) </li> 
-   <li> Precision (positive predictive value) - probability that somebody identified by the model as having the outcome at a specified cutoff truly has the outcome 
-   </li>
-  </ul> 
diff --git a/inst/shiny/PLPViewer/modules/calibration.R b/inst/shiny/PLPViewer/modules/calibration.R
deleted file mode 100644
index fa5d52186..000000000
--- a/inst/shiny/PLPViewer/modules/calibration.R
+++ /dev/null
@@ -1,318 +0,0 @@
-calibrationViewer <- function(id) {
-  ns <- shiny::NS(id)
-  shiny::div(
-    
-    shiny::fluidRow(
-      shinydashboard::box(
-        status = 'info', width = 12,
-        title = 'Summary',
-        solidHeader = TRUE,
-        shiny::p('Click on one of these rows to view corresponding plots:'),
-        DT::dataTableOutput(ns('calTable')
-        )
-      )
-    ),
-    
-    shiny::fluidRow(
-      shinydashboard::box(
-        status = 'info',
-        title = shiny::actionLink(
-          ns("calHelp"),
-          "Calibration Plot", 
-          icon = shiny::icon("info")
-        ),
-        solidHeader = TRUE,
-        shinycssloaders::withSpinner(shiny::plotOutput(ns('cal')))
-      ),
-      shinydashboard::box(
-        status = 'info',
-        title = shiny::actionLink(
-          ns("demoHelp"),
-          "Demographic Plot", 
-          icon = shiny::icon("info")
-        ),
-        solidHeader = TRUE,
-        side = "right",
-        shinycssloaders::withSpinner(shiny::plotOutput(ns('demo')))
-      )
-    )
-  )
-}
-
-calibrationServer <- function(id, plpResult) {
-  shiny::moduleServer(
-      id,
-    function(input, output, session) {
-      
-      sumTable <- shiny::reactive({
-        data <- plpResult()$performanceEvaluation$evaluationStatistics
-        
-        for(i in 1:ncol(data)){
-          data[,i] <- unlist(data[,i])
-        }
-
-        data$value <- as.double(as.character(data$value))
-        data$value <- format(data$value, digits = 4, scientific = F)
-        ind <- data$metric %in% c(
-          'calibrationInLarge intercept', 
-          'weak calibration intercept',
-          'weak calibration gradient',
-          'calibrationInLarge mean prediction',
-          'calibrationInLarge observed risk',
-          'ici',
-          'Emean',
-          'E90',
-          'Emax', 
-          'correctionFactor',
-          'adjustGradient',
-          'adjustIntercept'
-        )
-        
-        tidyr::pivot_wider(
-          data[ind,],
-          names_from = 'metric', 
-          values_from = 'value'
-          )
-        #reshape2::dcast(data[ind,], evaluation ~ metric, value.var = 'value')
-        
-      })
-      
-      output$calTable <- DT::renderDataTable({
-        if(is.null(plpResult()$performanceEvaluation)){
-          DT::datatable(NULL)
-        } else{
-          result <- sumTable()
-          row.names(result) <- NULL
-          DT::datatable(result, selection = 'single')
-        }
-      })
-      
-      output$cal <- shiny::renderPlot({
-        type <- trimws(sumTable()$evaluation[input$calTable_rows_selected])
-        print(type)
-        tryCatch(
-          {plotSparseCalibration2(
-            evaluation = plpResult()$performanceEvaluation, 
-            type =  type)
-          },
-          error = function(err){emptyPlot(title = err)}
-          )
-      })
-      
-      output$demo <- shiny::renderPlot({
-        type <- trimws(sumTable()$evaluation[input$calTable_rows_selected])
-        tryCatch(
-          plotDemographicSummary(
-            evaluation = plpResult()$performanceEvaluation, 
-            type = type
-          ),
-          error= function(cond){return(NULL)}
-        )
-      })
-      
-      
-      shiny::observeEvent(input$calHelp, {
-        shiny::showInfoBox("Calibration Help", "html/calHelp.html")
-      })
-      shiny::observeEvent(input$demoHelp, {
-        shiny::showInfoBox("Demographic Help", "html/demoHelp.html")
-      })
-      
-      
-    }
-  )
-}
-
-
-
-plotDemographicSummary <- function(evaluation, type = NULL,  fileName=NULL){
-  if (!all(is.na(evaluation$demographicSummary$averagePredictedProbability))){
-    
-    ind <- 1:nrow(evaluation$demographicSummary)
-    if(is.null(type)){
-      if(!is.null(evaluation$demographicSummary$evaluation)){
-        ind <- evaluation$demographicSummary$evaluation%in%c('Test','validation')
-      }
-    } else{
-      ind <- evaluation$demographicSummary$evaluation==type
-    }
-    
-    x<- evaluation$demographicSummary[ind,colnames(evaluation$demographicSummary)%in%c('ageGroup','genGroup','averagePredictedProbability',
-                                                                                       'PersonCountAtRisk', 'PersonCountWithOutcome')]
-    
-    
-    # remove -1 values:
-    x$averagePredictedProbability[is.na(x$averagePredictedProbability)] <- 0
-    x <- x[x$PersonCountWithOutcome != -1,]
-    if(nrow(x)==0){
-      return(NULL)
-    }
-    
-    x$observed <- x$PersonCountWithOutcome/x$PersonCountAtRisk
-    
-    
-    x <- x[,colnames(x)%in%c('ageGroup','genGroup','averagePredictedProbability','observed')]
-    
-    # if age or gender missing add 
-    if(sum(colnames(x)=='ageGroup')==1 && sum(colnames(x)=='genGroup')==0  ){
-      x$genGroup = rep('Non', nrow(x))
-      evaluation$demographicSummary$genGroup = rep('Non', nrow(evaluation$demographicSummary))
-    } 
-    if(sum(colnames(x)=='ageGroup')==0 && sum(colnames(x)=='genGroup')==1  ){
-      x$ageGroup = rep('-1', nrow(x))
-      evaluation$demographicSummary$ageGroup = rep('-1', nrow(evaluation$demographicSummary))
-      
-    }
-    
-    x <- tidyr::pivot_longer(
-      data = x, 
-      cols = !colnames(x)[colnames(x) %in% c('ageGroup','genGroup')], 
-      names_to = 'variable', 
-      values_to = 'value'
-      )
-    #x <- reshape2::melt(x, id.vars=c('ageGroup','genGroup'))
-    
-    # 1.96*StDevPredictedProbability
-    ci <- evaluation$demographicSummary[ind,colnames(evaluation$demographicSummary)%in%c('ageGroup','genGroup','averagePredictedProbability','StDevPredictedProbability')]
-    ci$StDevPredictedProbability[is.na(ci$StDevPredictedProbability)] <- 1
-    ci$lower <- ci$averagePredictedProbability-1.96*ci$StDevPredictedProbability
-    ci$lower[ci$lower <0] <- 0
-    ci$upper <- ci$averagePredictedProbability+1.96*ci$StDevPredictedProbability
-    ci$upper[ci$upper >1] <- max(ci$upper[ci$upper <1])
-    
-    x$age <- gsub('Age group:','', x$ageGroup)
-    x$age <- factor(x$age,levels=c(" 0-4"," 5-9"," 10-14",
-                                   " 15-19"," 20-24"," 25-29"," 30-34"," 35-39"," 40-44",
-                                   " 45-49"," 50-54"," 55-59"," 60-64"," 65-69"," 70-74",
-                                   " 75-79"," 80-84"," 85-89"," 90-94"," 95-99","-1"),ordered=TRUE)
-    
-    
-    
-    x <- merge(x, ci[,c('ageGroup','genGroup','lower','upper')], by=c('ageGroup','genGroup'))
-    x <- x[!is.na(x$value),]
-    
-    plot <- ggplot2::ggplot(data=x, 
-                            ggplot2::aes(x=age, 
-                                         group=interaction(variable,genGroup))) +
-      
-      ggplot2::geom_line(ggplot2::aes(y=value, group=variable,
-                                      color=variable,
-                                      linetype = variable))+
-      ggplot2::geom_ribbon(data=x[x$variable!='observed',],
-                           ggplot2::aes(ymin=lower, ymax=upper
-                                        , group=genGroup), 
-                           fill="blue", alpha=0.2) +
-      ggplot2::facet_grid(.~ genGroup, scales = "free") +
-      ggplot2::theme(axis.text.x = ggplot2::element_text(angle = 90, hjust = 1)) +
-      #ggplot2::coord_flip() +
-      ggplot2::scale_y_continuous("Fraction") +
-      ggplot2::scale_x_discrete("Age") +
-      ggplot2::scale_color_manual(values = c("royalblue4","red"),
-                                  guide = ggplot2::guide_legend(title = NULL),
-                                  labels = c("Expected", "Observed")) +
-      
-      ggplot2::guides(linetype=FALSE)
-    
-    if (!is.null(fileName))
-      ggplot2::ggsave(fileName, plot, width = 7, height = 4.5, dpi = 400)
-    return(plot)
-  }
-}
-
-plotSparseCalibration2 <- function(evaluation,
-                                   smooth = "loess",
-                                   span = 1,
-                                   nKnots = 5,
-                                   scatter = T,
-                                   bins = 20,
-                                   zoom =  "data",
-                                   sample = T,
-                                   fileName = NULL,
-                                   type = NULL) {
-  
-  ind <- 1:nrow(evaluation$calibrationSummary)
-  
-  if(is.null(type)){
-    if(!is.null(evaluation$calibrationSummary$evaluation)){
-      ind <- evaluation$calibrationSummary$evaluation%in%c('Test','validation')
-    }
-  } else{
-    ind <- evaluation$calibrationSummary$evaluation == type
-  }
-  # use calibrationSummary
-  sparsePred <- evaluation$calibrationSummary[ind,]
-  
-  limVal <- max(max(sparsePred$averagePredictedProbability),max(sparsePred$observedIncidence))
-  
-  smooth_plot <- ggplot2::ggplot(data = sparsePred, ggplot2::aes(x = averagePredictedProbability, 
-                                                                 y = observedIncidence)) +
-    ggplot2::stat_smooth(ggplot2::aes(color = "Loess", linetype = "Loess"),
-                         method = "loess",
-                         se = TRUE,
-                         #span = span,
-                         size = 1,
-                         show.legend = F) +
-    ggplot2::geom_segment(ggplot2::aes(x = 0,
-                                       xend = 1,
-                                       y = 0,
-                                       yend = 1,
-                                       color = "Ideal",
-                                       linetype = "Ideal")) +
-    ggplot2::coord_cartesian(xlim = c(0,limVal),
-                             ylim = c(0,limVal)) + 
-    ggplot2::scale_linetype_manual(name = "Models",
-                                   values = c(Loess = "solid",
-                                              Ideal = "dashed")) + 
-    ggplot2::scale_color_manual(name = "Models", values = c(Loess = "blue", Ideal = "red")) + 
-    ggplot2::labs(x = "Predicted Probability", y = "Observed Probability")
-  
-  # construct the plot grid
-  if (scatter) {
-    smooth_plot <- smooth_plot + ggplot2::geom_point(data = sparsePred,
-                                                     ggplot2::aes(x = averagePredictedProbability,
-                                                                  y = observedIncidence),
-                                                     color = "black",
-                                                     size = 2)
-  }
-  
-  # Histogram object detailing the distibution of event/noevent for each probability interval
-  
-  popData1 <- sparsePred[,c('averagePredictedProbability', 'PersonCountWithOutcome')]
-  popData1$Label <- "Outcome"
-  colnames(popData1) <- c('averagePredictedProbability','PersonCount',"Label")
-  popData2 <- sparsePred[,c('averagePredictedProbability', 'PersonCountAtRisk')]
-  popData2$Label <- "No Outcome"
-  popData2$PersonCountAtRisk <- -1*(popData2$PersonCountAtRisk -popData1$PersonCount)
-  colnames(popData2) <- c('averagePredictedProbability','PersonCount',"Label")
-  popData <- rbind(popData1, popData2)
-  popData$averagePredictedProbability <- factor(popData$averagePredictedProbability)
-  hist_plot <- ggplot2::ggplot(popData, ggplot2::aes(y = averagePredictedProbability, x = PersonCount, 
-                                                     fill = Label)) + 
-    ggplot2::geom_bar(data = subset(popData,Label == "Outcome"), stat = "identity") + 
-    ggplot2::geom_bar(data = subset(popData,Label == "No Outcome"), stat = "identity") + 
-    ggplot2::geom_bar(stat = "identity") + 
-    ggplot2::scale_x_continuous(labels = abs) + 
-    #ggplot2::scale_fill_brewer(palette = "Set1") + 
-    ggplot2::coord_flip( ) +
-    ggplot2::theme_bw() + 
-    ggplot2::theme(axis.title.x=ggplot2::element_blank(),
-                   axis.text.x=ggplot2::element_blank(),
-                   axis.ticks.x=ggplot2::element_blank())
-  
-  # testting whether this is installed in shinydeploy
-  plot <- gridExtra::grid.arrange(smooth_plot,
-                                  hist_plot,
-                                  ncol = 1,
-                                  heights=c(2,1))
-  
-  #plot <- cowplot::plot_grid(smooth_plot,
-  #                           hist_plot,
-  #                           ncol = 1,
-  #                           axis = "lr",
-  #                           align = "v",
-  #                           rel_heights = c(1, 0.6))
-  
-  if (!is.null(fileName))
-    ggplot2::ggsave(fileName, plot, width = 5, height = 4.5, dpi = 400)
-  return(plot)
-}
diff --git a/inst/shiny/PLPViewer/modules/covariateSummary.R b/inst/shiny/PLPViewer/modules/covariateSummary.R
deleted file mode 100644
index 34fe1dac4..000000000
--- a/inst/shiny/PLPViewer/modules/covariateSummary.R
+++ /dev/null
@@ -1,283 +0,0 @@
-covariateSummaryViewer <- function(id) {
-  ns <- shiny::NS(id)
-  
-  shiny::div(
-  shiny::fluidRow(
-    shinydashboard::box( 
-      status = 'info',
-      title = "Binary", 
-      solidHeader = TRUE,
-      shinycssloaders::withSpinner(
-        plotly::plotlyOutput(
-          ns('covariateSummaryBinary')
-        )
-      )
-    ),
-    shinydashboard::box(
-      status = 'info',
-      title = "Measurements", 
-      solidHeader = TRUE,
-      side = "right",
-      shinycssloaders::withSpinner(
-        plotly::plotlyOutput(
-          ns('covariateSummaryMeasure')
-        )
-      )
-    )
-  ),
-    
-    shiny::fluidRow(
-      width=12,
-      shinydashboard::box(
-        status = 'info', width = 12,
-        title = "Covariates", solidHeader = TRUE,
-        DT::dataTableOutput(ns('modelCovariateInfo'))
-      )
-    ),
-    shiny::fluidRow(
-      width=12,
-      shinydashboard::box(status = 'info', 
-        width = 12,
-        title = "Model Table", 
-        solidHeader = TRUE,
-        shiny::downloadButton("downloadData", "Download Model"),
-        DT::dataTableOutput(ns('modelView'))
-      )
-    )
-  )
-
-}
-
-covariateSummaryServer <- function(id, plpResult, summaryTable, resultRow, mySchema, con,
-                                   inputSingleView,
-                                   myTableAppend = '', 
-                                   targetDialect = NULL) {
-  shiny::moduleServer(
-    id,
-    function(input, output, session) {
-      
-        covariateSummary <- shiny::reactive({
-          if(inputSingleView == "Model"){
-          if(is.null(plpResult()$covariateSummary)){
-            covariateSummary <- tryCatch(
-              {loadCovSumFromDb(summaryTable[resultRow(),], mySchema, con, myTableAppend, targetDialect)},
-              error = function(e){return(NULL)}
-              )
-            print(colnames(covariateSummary)) # temp for debugging
-            return(covariateSummary)
-          } else{
-            return(plpResult()$covariateSummary)
-          }
-          }
-        })
-        
-      # covariate table
-        output$modelView <- DT::renderDataTable(
-          editCovariates(covariateSummary())$table,
-          colnames = editCovariates(covariateSummary())$colnames
-        )
-      
-
-      output$modelCovariateInfo <- DT::renderDataTable(
-        data.frame(
-          covariates = nrow(covariateSummary()),
-          nonZeroCount = sum(covariateSummary()$covariateValue!=0, na.rm = T),
-          intercept = getIntercept(plpResult())
-          )
-        )
-      
-      # covariate model plots
-      covs <- shiny::reactive({
-        if(is.null(covariateSummary()))
-          return(NULL)
-        plotCovariateSummary(formatCovariateTable(covariateSummary()))
-      })
-      
-      output$covariateSummaryBinary <- plotly::renderPlotly({ covs()$binary })
-      output$covariateSummaryMeasure <- plotly::renderPlotly({ covs()$meas })
-      
-      # Downloadable csv of model ----
-      output$downloadData <- shiny::downloadHandler(
-        filename = function(){'model.csv'},
-        content = function(file) {
-          utils::write.csv( 
-            covariateSummary()[,colnames(covariateSummary()) %in% c('covariateName','covariateValue','CovariateCount','WithOutcomeCovariateMean','WithNoOutcomeCovariateMean','WithOutcome_CovariateMean','WithNoOutcome_CovariateMean' )],
-            file, 
-            row.names = FALSE
-            )
-        }
-      )
-      
-    }
-  )
-}
-
-
-# helpers
-
-# format covariate summary table
-formatCovariateTable <- function(covariateSummary){
-  covariateSummary <- as.data.frame(covariateSummary)
-  colnames(covariateSummary) <- gsub('_','', colnames(covariateSummary) )
-  for(coln in c('covariateValue','WithOutcomeCovariateMean','WithNoOutcomeCovariateMean','StandardizedMeanDiff')){
-    if(sum(colnames(covariateSummary)==coln)>0){
-      covariateSummary[,coln] <- format(round(covariateSummary[,coln], 4), nsmall = 4)
-      class(covariateSummary[,coln]) <- "numeric"
-    }
-  }
-  return(covariateSummary)
-}
-
-
-
-editCovariates <- function(covs){
-  colnames(covs) <- gsub('_','', colnames(covs) )
-  if(!is.null(covs$StandardizedMeanDiff)){
-    return(list(table = formatCovariateTable(covs[,c('covariateName','covariateValue','CovariateCount','WithOutcomeCovariateMean','WithNoOutcomeCovariateMean','StandardizedMeanDiff')]),
-                colnames = c('Covariate Name', 'Value','Count', 'Outcome Mean', 'Non-outcome Mean','Std Mean Diff')
-    ))
-  } else{
-    return(list(table = formatCovariateTable(covs[,c('covariateName','covariateValue','CovariateCount','WithOutcomeCovariateMean','WithNoOutcomeCovariateMean')]),
-                colnames = c('Covariate Name', 'Value','Count', 'Outcome Mean', 'Non-outcome Mean')
-    ))
-  }
-}
-
-
-
-plotCovariateSummary <- function(covariateSummary){
-  
-  colnames(covariateSummary) <- gsub('_','', colnames(covariateSummary) )
-  
-  #writeLines(paste(colnames(covariateSummary)))
-  #writeLines(paste(covariateSummary[1,]))
-  # remove na values 
-  covariateSummary$WithNoOutcomeCovariateMean[is.na(covariateSummary$WithNoOutcomeCovariateMean)] <- 0
-  covariateSummary$WithOutcomeCovariateMean[is.na(covariateSummary$WithOutcomeCovariateMean)] <- 0
-  if(!'covariateValue'%in%colnames(covariateSummary)){
-    covariateSummary$covariateValue <- 1
-  }
-  if(sum(is.na(covariateSummary$covariateValue))>0){
-    covariateSummary$covariateValue[is.na(covariateSummary$covariateValue)] <- 0
-  }
-  
-  # SPEED EDIT remove the none model variables
-  covariateSummary <- covariateSummary[covariateSummary$covariateValue!=0,]
-  
-  # save dots based on coef value 
-  covariateSummary$size <- abs(covariateSummary$covariateValue)
-  covariateSummary$size[is.na(covariateSummary$size)] <- 4
-  covariateSummary$size <- 4+4*covariateSummary$size/max(covariateSummary$size)
-  
-  # color based on analysis id
-  covariateSummary$color <- sapply(covariateSummary$covariateName, function(x) ifelse(is.na(x), '', strsplit(as.character(x), ' ')[[1]][1]))
-  
-  covariateSummary$times <- sapply(sapply(covariateSummary$covariateName, function(x) ifelse(is.na(x), '', strsplit(as.character(x), 'during day ')[[1]][2])),function(x) ifelse(is.na(x), '', strsplit(as.character(x), ': ')[[1]][1]))
-  covariateSummary$desc <- sapply(covariateSummary$covariateName, function(x) ifelse(is.na(x), '', strsplit(as.character(x), ': ')[[1]][2]))
-  
-  
-  l <- list(x = 0.01, y = 1,
-            font = list(
-              family = "sans-serif",
-              size = 10,
-              color = "#000"),
-            bgcolor = "#E2E2E2",
-            bordercolor = "#FFFFFF",
-            borderwidth = 1)
-  
-  ind <- covariateSummary$WithNoOutcomeCovariateMean <=1 & covariateSummary$WithOutcomeCovariateMean <= 1
-  # create two plots -1 or less or g1
-  binary <- plotly::plot_ly(x = covariateSummary$WithNoOutcomeCovariateMean[ind],
-                            #size = covariateSummary$size[ind],
-                            showlegend = F) %>%
-    plotly::add_markers(y = covariateSummary$WithOutcomeCovariateMean[ind],
-                        color=factor(covariateSummary$color[ind]),
-                        hoverinfo = 'text',
-                        text = ~paste('</br> Type: ', covariateSummary$color[ind],
-                                      '</br> Time: ', covariateSummary$times[ind],
-                                      '</br> Name: ', covariateSummary$desc[ind]),
-                        showlegend = T
-    ) %>%
-    plotly::add_trace(x= c(0,1), y = c(0,1),mode = 'lines',
-                      line = list(dash = "dash"), color = I('black'),
-                      type='scatter', showlegend = FALSE) %>%
-    plotly::layout(#title = 'Prevalance of baseline predictors in persons with and without outcome',
-      xaxis = list(title = "Prevalance in persons without outcome",
-                   range = c(0, 1)),
-      yaxis = list(title = "Prevalance in persons with outcome",
-                   range = c(0, 1)),
-      #legend = l, showlegend = T,
-      legend = list(orientation = 'h', y = -0.3), showlegend = T)
-  
-  if(sum(!ind)>0){
-    maxValue <- max(c(covariateSummary$WithNoOutcomeCovariateMean[!ind],
-                      covariateSummary$WithOutcomeCovariateMean[!ind]), na.rm = T)
-    meas <- plotly::plot_ly(x = covariateSummary$WithNoOutcomeCovariateMean[!ind] ) %>%
-      plotly::add_markers(y = covariateSummary$WithOutcomeCovariateMean[!ind],
-                          hoverinfo = 'text',
-                          text = ~paste('</br> Type: ', covariateSummary$color[!ind],
-                                        '</br> Time: ', covariateSummary$times[!ind],
-                                        '</br> Name: ', covariateSummary$desc[!ind])) %>%
-      plotly::add_trace(x= c(0,maxValue), y = c(0,maxValue),mode = 'lines',
-                        line = list(dash = "dash"), color = I('black'),
-                        type='scatter', showlegend = FALSE) %>%
-      plotly::layout(#title = 'Prevalance of baseline predictors in persons with and without outcome',
-        xaxis = list(title = "Mean in persons without outcome"),
-        yaxis = list(title = "Mean in persons with outcome"),
-        showlegend = FALSE)
-  } else {
-    meas <- NULL
-  }
-  
-  return(list(binary=binary,
-              meas = meas))
-}
-
-
-
-getIntercept <- function(plpResult){
-  
-  if('model'%in%names(plpResult)){
-    
-    if('model'%in%names(plpResult$model)){
-      
-      if('coefficients'%in%names(plpResult$model$model)){
-        
-        return(plpResult$model$model$coefficients[1])
-        
-      }
-      
-    }
-  }
-  return(0)
-}
-
-
-toFirstUpper <- function(vec){
-  res <- lapply(vec, function(x) paste0(toupper(substr(x,1,1)), substr(x,2,nchar(x))))
-  return(unlist(res))
-}
-
-# code for database covariate extract
-loadCovSumFromDb <- function(chosenRow, mySchema, con, myTableAppend = '', targetDialect = 'redshift'){
-  ParallelLogger::logInfo("starting covsum")
-  resultId <- chosenRow$resultId
-  sql <- "SELECT * FROM @my_schema.@my_table_appendcovariate_summary AS covariate_summary WHERE result_id = @result_id;" 
-  
-  sql <- SqlRender::render(sql = sql,
-    my_schema = mySchema,
-    result_id = resultId,
-    my_table_append = myTableAppend)
-  sql <- SqlRender::translate(sql = sql, targetDialect =  targetDialect)
-  
-  covariateSummary <- DatabaseConnector::dbGetQuery(conn =  con, statement = sql) 
-  colnames(covariateSummary) <- SqlRender::snakeCaseToCamelCase(colnames(covariateSummary))
-  
-  # capitalize the first letter of column names 
-  # except covariateId covariateName analysisId conceptId
-  ind <- !colnames(covariateSummary) %in% c('covariateValue','covariateId', 'covariateName', 'analysisId', 'conceptId')
-  colnames(covariateSummary)[ind] <- toFirstUpper(colnames(covariateSummary)[ind])
-  
-  ParallelLogger::logInfo("finishing covsum")
-  return(covariateSummary)
-}
diff --git a/inst/shiny/PLPViewer/modules/cutoff.R b/inst/shiny/PLPViewer/modules/cutoff.R
deleted file mode 100644
index 2f1587829..000000000
--- a/inst/shiny/PLPViewer/modules/cutoff.R
+++ /dev/null
@@ -1,280 +0,0 @@
-cutoffViewer <- function(id) {
-  
-  ns <- shiny::NS(id)
-  
-  shiny::fluidRow(
-    
-    shiny::column(width = 12,
-      
-      shinydashboard::box(
-        width = 12,
-        title = "Probability threshold plot: ",
-        status = "info", 
-        solidHeader = TRUE,
-        plotly::plotlyOutput(ns("ptp"))                 
-      ),
-      
-      shinydashboard::box(
-        width = 12,
-        title = "Cutoff Slider: ",
-        status = "info", 
-        solidHeader = TRUE,
-        shiny::sliderInput(
-          ns("slider1"), 
-          shiny::span(
-            "Pick Threshold ", 
-            shiny::textOutput('threshold'), 
-            style="font-family: Arial;font-size:14px;"
-            ), 
-          min = 1, 
-          max = 100, 
-          value = 50, 
-          ticks = F
-        )                  
-      ),
-      
-      shinydashboard::box(
-        width = 12,
-        title = "Dashboard",
-        status = "warning", solidHeader = TRUE,
-        shinydashboard::infoBoxOutput(ns("performanceBoxThreshold")),
-        shinydashboard::infoBoxOutput(ns("performanceBoxIncidence")),
-        shinydashboard::infoBoxOutput(ns("performanceBoxPPV")),
-        shinydashboard::infoBoxOutput(ns("performanceBoxSpecificity")),
-        shinydashboard::infoBoxOutput(ns("performanceBoxSensitivity")),
-        shinydashboard::infoBoxOutput(ns("performanceBoxNPV")
-        )
-      ),
-      
-      shinydashboard::box(
-        width = 12,
-        title = "Cutoff Performance",
-        status = "warning", solidHeader = TRUE,
-        shiny::tableOutput(ns('twobytwo'))
-      )
-    )
-  )
-}
-
-cutoffServer <- function(id, plpResult) {
-  shiny::moduleServer(
-    id,
-    function(input, output, session) {
-      
-      performance <- shiny::reactive({
-        eval <- plpResult()$performanceEvaluation
-        if(is.null(eval)){
-          return(NULL)
-        } else {
-          intPlot <- getORC(eval, input$slider1)
-          threshold <- intPlot$threshold
-          prefthreshold <- intPlot$prefthreshold
-          TP <- intPlot$TP
-          FP <- intPlot$FP
-          TN <- intPlot$TN
-          FN <- intPlot$FN
-        }
-        
-        twobytwo <- as.data.frame(matrix(c(FP,TP,TN,FN), byrow=T, ncol=2))
-        colnames(twobytwo) <- c('Ground Truth Negative','Ground Truth Positive')
-        rownames(twobytwo) <- c('Predicted Positive','Predicted Negative')
-        
-        list(threshold = threshold, 
-             prefthreshold = prefthreshold,
-             twobytwo = twobytwo,
-             Incidence = (TP+FN)/(TP+TN+FP+FN),
-             Threshold = threshold,
-             Sensitivity = TP/(TP+FN),
-             Specificity = TN/(TN+FP),
-             PPV = TP/(TP+FP),
-             NPV = TN/(TN+FN) )
-      })
-      
-      # add probability threshold plot
-      ptp <- shiny::reactive({
-        eval <- plpResult()$performanceEvaluation
-        if(is.null(eval)){
-          return(NULL)
-        } else {
-          probThresPlot(eval = eval, pointOfInterest = input$slider1)
-        }
-          
-        })
-      
-      output$ptp <- plotly::renderPlotly(ptp())
-          
-          
-      
-      
-      # update threshold slider based on results size
-      shiny::observe({ 
-        if(!is.null(plpResult()$performanceEvaluation)){
-          n <- nrow(plpResult()$performanceEvaluation$thresholdSummary[plpResult()$performanceEvaluation$thresholdSummary$evaluation%in%c('Test','Validation'),])
-        }else{
-          n <- 100
-        }
-        
-        shiny::updateSliderInput(session, inputId = "slider1",  
-                                 min = 1, max = n, value = round(n/2))
-      })
-      
-      # Do the tables and plots:
-      
-      output$performance <- shiny::renderTable(performance()$performance, 
-                                               rownames = F, digits = 3)
-      output$twobytwo <- shiny::renderTable(performance()$twobytwo, 
-                                            rownames = T, digits = 0)
-      
-      
-      output$threshold <- shiny::renderText(format(performance()$threshold,digits=5))
-      
-      
-      # dashboard
-      
-      output$performanceBoxIncidence <- shinydashboard::renderInfoBox({
-        shinydashboard::infoBox(
-          "Incidence", paste0(round(performance()$Incidence*100, digits=3),'%'), icon = shiny::icon("ambulance"),
-          color = "green"
-        )
-      })
-      
-      output$performanceBoxThreshold <- shinydashboard::renderInfoBox({
-        shinydashboard::infoBox(
-          "Threshold", format((performance()$Threshold), scientific = F, digits=3), icon = shiny::icon("edit"),
-          color = "yellow"
-        )
-      })
-      
-      output$performanceBoxPPV <- shinydashboard::renderInfoBox({
-        shinydashboard::infoBox(
-          "PPV", paste0(round(performance()$PPV*1000)/10, "%"), icon = shiny::icon("thumbs-up"),
-          color = "orange"
-        )
-      })
-      
-      output$performanceBoxSpecificity <- shinydashboard::renderInfoBox({
-        shinydashboard::infoBox(
-          "Specificity", paste0(round(performance()$Specificity*1000)/10, "%"), icon = shiny::icon("bullseye"),
-          color = "purple"
-        )
-      })
-      
-      output$performanceBoxSensitivity <- shinydashboard::renderInfoBox({
-        shinydashboard::infoBox(
-          "Sensitivity", paste0(round(performance()$Sensitivity*1000)/10, "%"), icon = shiny::icon("low-vision"),
-          color = "blue"
-        )
-      })
-      
-      output$performanceBoxNPV <- shinydashboard::renderInfoBox({
-        shinydashboard::infoBox(
-          "NPV", paste0(round(performance()$NPV*1000)/10, "%"), icon = shiny::icon("minus-square"),
-          color = "black"
-        )
-      })
-    
-      
-    }
-  )
-}
-
-
-
-getORC <- function(eval, pointOfInterest){
-  
-  data <- eval$thresholdSummary[eval$thresholdSummary$evaluation%in%c('Test','Validation'),]
-  data <- data[order(data$predictionThreshold),]
-  pointOfInterest <- data[pointOfInterest,]
-  
-  threshold <- pointOfInterest$predictionThreshold
-  TP <- pointOfInterest$truePositiveCount
-  TN <- pointOfInterest$trueNegativeCount
-  FP <- pointOfInterest$falsePositiveCount
-  FN <- pointOfInterest$falseNegativeCount
-  preferenceThreshold <- pointOfInterest$preferenceThreshold
-  return(list(threshold = threshold, prefthreshold=preferenceThreshold,
-              TP = TP, TN=TN,
-              FP= FP, FN=FN))
-}
-
-probThresPlot <- function(eval, pointOfInterest){
-  
-  eval <- eval$thresholdSummary[eval$thresholdSummary$evaluation%in%c('Test','Validation'),]
-  eval <- eval[order(eval$predictionThreshold),]
-  
-  ay <- list(
-    tickfont = list(color = "red"),
-    overlaying = "y",
-    side = "right",
-    title = "positivePredictiveValue y-axis"
-  )
-  vline <- function(x = 0, color = "green") {
-    list(
-      type = "line",
-      y0 = 0,
-      y1 = 1,
-      yref = "paper",
-      x0 = x,
-      x1 = x,
-      line = list(color = color, dash="dot")
-    )
-  }
-  
-  eval$popfrac <- eval$positiveCount/(eval$positiveCount+eval$negativeCount)
-  
-  fig <- plotly::plot_ly(
-    data = eval, 
-    x = ~ predictionThreshold, 
-    y = ~ sensitivity, 
-    name = 'sensitivity', 
-    color = 'blue', 
-    type = 'scatter', 
-    mode = 'lines'
-  ) %>% 
-    plotly::add_trace(
-      yaxis = "y2",
-      y = ~ positivePredictiveValue, 
-      name = 'positivePredictiveValue', 
-      color = 'red', 
-      mode = 'lines'
-    ) %>% 
-    plotly::add_trace(
-      y = ~ negativePredictiveValue, 
-      name = 'negativePredictiveValue', 
-      color = 'green', 
-      mode = 'lines'
-    ) %>% 
-    plotly::add_trace(
-      y = ~ popfrac, 
-      name = 'Fraction flagged',
-      color = 'black', 
-      mode = 'lines'
-    ) %>% 
-    plotly::layout(
-      title = "Probability Threshold Plot", 
-      yaxis2 = ay,
-      #xaxis = list(title="Prediction Threshold"),
-      #yaxis = list(title="Metric yaxis")
-    #) %>%
-    #plotly::layout(
-      plot_bgcolor='#e5ecf6',
-      xaxis = list(
-        title = "Prediction Threshold",
-        zerolinecolor = '#ffff',
-        zerolinewidth = 2,
-        gridcolor = 'ffff'
-      ),
-      yaxis = list(
-        title = "Metric yaxis",
-        zerolinecolor = '#ffff',
-        zerolinewidth = 2,
-        gridcolor = 'ffff'
-      ),
-      shapes = list(vline(eval$predictionThreshold[pointOfInterest]))
-    )
-  
-  return(fig)
-  
-}
-
-
diff --git a/inst/shiny/PLPViewer/modules/discrimination.R b/inst/shiny/PLPViewer/modules/discrimination.R
deleted file mode 100644
index ce854ff0e..000000000
--- a/inst/shiny/PLPViewer/modules/discrimination.R
+++ /dev/null
@@ -1,510 +0,0 @@
-discriminationViewer <- function(id) {
-  
-  ns <- shiny::NS(id)
-  
-  shiny::div(
-    
-    # summary table
-    shiny::fluidRow(
-      shinydashboard::box(
-        status = 'info', 
-        width = 12,
-        title = 'Summary',
-        solidHeader = TRUE,
-        shiny::p('Click on one of these rows to view corresponding plots:'),
-        DT::dataTableOutput(ns('summaryTable'))
-      )
-    ),
-    
-    
-    shiny::fluidRow(
-      shinydashboard::box( 
-        status = 'info',
-        title = shiny::actionLink(
-          ns("rocHelp"), 
-          "ROC Plot", 
-          icon = icon("info")
-        ),
-        solidHeader = TRUE,
-        shinycssloaders::withSpinner(
-          plotly::plotlyOutput(ns('roc'))
-        )
-      ),
-      shinydashboard::box(
-        status = 'info',
-        title = shiny::actionLink(
-          ns("prcHelp"), 
-          "Precision recall plot", 
-          icon = icon("info")
-        ),
-        solidHeader = TRUE,
-        side = "right",
-        shinycssloaders::withSpinner(
-          plotly::plotlyOutput(ns('pr'))
-        )
-      )
-    ),
-    
-    shiny::fluidRow(
-      shinydashboard::box(
-        status = 'info',
-        title = shiny::actionLink(
-          ns("f1Help"), 
-          "F1 Score Plot", 
-          icon = icon("info")
-        ),
-        solidHeader = TRUE,
-        shinycssloaders::withSpinner(
-          plotly::plotlyOutput(ns('f1'))
-        )
-      ),
-      shinydashboard::box(
-        status = 'info',
-        title = shiny::actionLink(
-          ns("boxHelp"),
-          "Box Plot", 
-          icon = icon("info")
-        ),
-        solidHeader = TRUE,
-        side = "right",
-        shinycssloaders::withSpinner(
-          shiny::plotOutput(ns('box'))
-        )
-      )
-    ),
-  
-    shiny::fluidRow(
-      shinydashboard::box(
-        status = 'info',
-        title = shiny::actionLink(
-          ns("predDistHelp"),
-          "Prediction Score Distribution", 
-          icon = icon("info")
-        ),
-        solidHeader = TRUE,
-        shinycssloaders::withSpinner(
-          shiny::plotOutput(ns('preddist'))
-        )
-      ),
-      shinydashboard::box(
-        status = 'info',
-        title = shiny::actionLink(
-          ns("prefDistHelp"),
-          "Preference Score Distribution", 
-          icon = icon("info")
-        ),
-        solidHeader = TRUE,
-        side = "right",
-        shinycssloaders::withSpinner(
-          shiny::plotOutput(ns('prefdist'))
-        )
-      )
-    )
-  )
-}
-
-discriminationServer <- function(id, plpResult) {
-  shiny::moduleServer(
-    id,
-    function(input, output, session) {
-      
-      sumTable <- shiny::reactive({
-        data <- plpResult()$performanceEvaluation$evaluationStatistics
-        for(i in 1:ncol(data)){
-          data[,i] <- unlist(data[,i])
-        }
-        data$value <- as.double(as.character(data$value))
-        data$value <- format(data$value, digits = 4, scientific = F)
-        ind <- data$metric %in% c('AUROC', 
-                                  '95% lower AUROC',
-                                  '95% upper AUROC',
-                                  'AUPRC'
-                                  
-        )
-        
-        tidyr::pivot_wider(
-          data = data[ind,], 
-          names_from = 'metric', 
-          values_from = 'value'
-          )
-        #reshape2::dcast(data[ind,], evaluation ~ metric, value.var = 'value')
-        
-      })
-      
- 
-      output$summaryTable <- DT::renderDataTable({
-        if(is.null(plpResult()$performanceEvaluation)){
-          DT::datatable(NULL)
-        } else{
-          result <- sumTable()
-          row.names(result) <- NULL
-          DT::datatable(result,selection = 'single')
-        }
-      })
-      
-      
-      plots <-  shiny::reactive({
-        
-          result <- list(roc = tryCatch({rocPlot(eval = plpResult()$performanceEvaluation)},
-                                        error = function(cond){
-                                          list(train = emptyPlot(title = 'No performanceEvaluation'))
-                                          }),
-                         pr = tryCatch({prPlot(eval = plpResult()$performanceEvaluation)},
-                                       error = function(cond){
-                           list(train = emptyPlot(title = 'No performanceEvaluation'))
-                         }),
-                         f1 = tryCatch({f1Plot(eval = plpResult()$performanceEvaluation)},
-                                       error = function(cond){
-                                         list(train = emptyPlot(title = 'No performanceEvaluation'))
-                                       }),
-                         prefpdf = tryCatch({plotPreferencePDF(plpResult()$performanceEvaluation)},
-                                            error = function(cond){
-                                              NULL
-                                            }),
-                         predpdf = tryCatch({plotPredictedPDF(plpResult()$performanceEvaluation)},
-                                            error = function(cond){
-                                              NULL
-                                            }),
-                         box = tryCatch({plotPredictionDistribution(plpResult()$performanceEvaluation)},
-                                        error = function(cond){
-                                          NULL
-                                        })
-          )
-          
-        return(result)
-      }
-      )
-      
-     output$roc <- plotly::renderPlotly({
-       type <- trimws(sumTable()$evaluation[input$summaryTable_rows_selected])
-       tryCatch({plots()$roc[[type]]}, error = function(err){emptyPlot(title = err)})
-     })
-      
-      output$pr <- plotly::renderPlotly({
-        type <- trimws(sumTable()$evaluation[input$summaryTable_rows_selected])
-        tryCatch({plots()$pr[[type]]}, error = function(err){emptyPlot(title = err)})
-     })
-      
-      output$f1 <- plotly::renderPlotly({
-        type <- trimws(sumTable()$evaluation[input$summaryTable_rows_selected])
-        tryCatch({plots()$f1[[type]]}, error = function(err){emptyPlot(title = err)})
-      })
-      
-      # preference plot
-      output$prefdist <- shiny::renderPlot({
-        type <- trimws(sumTable()$evaluation[input$summaryTable_rows_selected])
-        tryCatch({plots()$prefpdf[[type]]}, error = function(err){emptyPlot(title = err)})
-      })
-      
-      output$preddist <- shiny::renderPlot({
-        type <- trimws(sumTable()$evaluation[input$summaryTable_rows_selected])
-        tryCatch({plots()$predpdf[[type]]}, error = function(err){emptyPlot(title = err)})
-      })
-      
-      output$box <- shiny::renderPlot({
-        type <- trimws(sumTable()$evaluation[input$summaryTable_rows_selected])
-        tryCatch({plots()$box[[type]]}, error = function(err){emptyPlot(title = err)})
-      })
-
-      shiny::observeEvent(input$rocHelp, {
-        showInfoBox("ROC Help", "html/rocHelp.html")
-      })
-      shiny::observeEvent(input$prcHelp, {
-        showInfoBox("PRC Help", "html/prcHelp.html")
-      })
-      shiny::observeEvent(input$f1Help, {
-        showInfoBox("F1 Score Plot Help", "html/f1Help.html")
-      })
-      shiny::observeEvent(input$boxHelp, {
-        showInfoBox("Box Plot Help", "html/boxHelp.html")
-      })
-      shiny::observeEvent(input$predDistHelp, {
-        showInfoBox("Predicted Risk Distribution Help", "html/predDistHelp.html")
-      })
-      shiny::observeEvent(input$prefDistHelp, {
-        showInfoBox("Preference Score Distribution Help", "html/prefDistHelp.html")
-      })
-      
-    
-      
-    }
-  )
-}
-
-
-
-# pltting
-rocPlot <- function(eval, type){
-  
-  types <- unique(eval$thresholdSummary$evaluation)
-  rocobject <- list()
-  length(rocobject) <- length(types)
-  names(rocobject) <- types
-  
-  for(type in types){
-    data <- eval$thresholdSummary[eval$thresholdSummary$evaluation%in%type,]
-
-    rocobject[[type]] <- plotly::plot_ly(x = 1-c(0,data$specificity,1)) %>%
-      plotly::add_lines(y = c(1,data$sensitivity,0),name = "hv", 
-                        text = paste('Risk Threshold:',c(0,data$predictionThreshold,1)),
-                        line = list(shape = "hv",
-                                    color = 'rgb(22, 96, 167)'),
-                        fill = 'tozeroy') %>%
-      plotly::add_trace(x= c(0,1), y = c(0,1),mode = 'lines',
-                        line = list(dash = "dash"), color = I('black'),
-                        type='scatter') %>%
-      plotly::layout(title = "ROC Plot",
-                     xaxis = list(title = "1-specificity"),
-                     yaxis = list (title = "Sensitivity"),
-                     showlegend = FALSE)
-  }
-  return(rocobject)
-}
-
-prPlot <- function(eval, type){
-  types <- unique(eval$thresholdSummary$evaluation)
-  probject <- list()
-  length(probject) <- length(types)
-  names(probject) <- types
-  
-  for(type in types){
-    data <- eval$thresholdSummary[eval$thresholdSummary$evaluation%in%type,]
-    
-    popAv <- data$trueCount[1]/(data$trueCount[1] + data$falseCount[1])
-    probject[[type]]  <- plotly::plot_ly(x = data$sensitivity) %>%
-      plotly::add_lines(y = data$positivePredictiveValue, name = "hv", 
-                        text = paste('Risk Threshold:',data$predictionThreshold),
-                        line = list(shape = "hv",
-                                    color = 'rgb(22, 96, 167)'),
-                        fill = 'tozeroy') %>%
-      plotly::add_trace(x= c(0,1), y = c(popAv,popAv),mode = 'lines',
-                        line = list(dash = "dash"), color = I('red'),
-                        type='scatter') %>%
-      plotly::layout(title = "PR Plot",
-                     xaxis = list(title = "Recall"),
-                     yaxis = list (title = "Precision"),
-                     showlegend = FALSE)
- 
-  }
-  return(probject)
-}
-
-f1Plot <- function(eval, type){
-  types <- unique(eval$thresholdSummary$evaluation)
-  f1object <- list()
-  length(f1object) <- length(types)
-  names(f1object) <- types
-  
-  for(type in types){
-    data <- eval$thresholdSummary[eval$thresholdSummary$evaluation%in%type,]
-    
-    f1object[[type]]  <- plotly::plot_ly(x = data$predictionThreshold) %>%
-      plotly::add_lines(y = data$f1Score, name = "hv", 
-                        text = paste('Risk Threshold:',data$predictionThreshold),
-                        line = list(shape = "hv",
-                                    color = 'rgb(22, 96, 167)'),
-                        fill = 'tozeroy') %>%
-      plotly::layout(title = "F1-Score Plot",
-                     xaxis = list(title = "Prediction Threshold"),
-                     yaxis = list (title = "F1-Score"),
-                     showlegend = FALSE)
-    
-  }
-  return(f1object)
-}
-
-
-
-
-
-
-# adding plots from PLP temporarily as shiny deploy doesnt have PatientLevelPrediction
-
-plotPredictedPDF <- function(evaluation, fileName=NULL){
-  
-  if(!is.null(evaluation$thresholdSummary$evaluation)){
-    types <- unique(evaluation$thresholdSummary$evaluation)
-  } else{
-    evaluation$thresholdSummary$evaluation <- 'na'
-    types <- 'na'
-  }
-  
-  plotResult <- list()
-  length(plotResult) <- length(types)
-  names(plotResult) <- types
-  
-  for(type in types){
-    
-  ind <- 1:nrow(evaluation$thresholdSummary)
-  if(!is.null(evaluation$thresholdSummary$evaluation)){
-    ind <- evaluation$thresholdSummary$evaluation == type
-  }
-  
-  
-  x<- evaluation$thresholdSummary[ind,c('predictionThreshold','truePositiveCount','trueNegativeCount',
-                                        'falsePositiveCount','falseNegativeCount')]
-  x<- x[order(x$predictionThreshold,-x$truePositiveCount, -x$falsePositiveCount),]
-  x$out <- c(x$truePositiveCount[-length(x$truePositiveCount)]-x$truePositiveCount[-1], x$truePositiveCount[length(x$truePositiveCount)])
-  x$nout <- c(x$falsePositiveCount[-length(x$falsePositiveCount)]-x$falsePositiveCount[-1], x$falsePositiveCount[length(x$falsePositiveCount)])
-  
-  vals <- c()
-  for(i in 1:length(x$predictionThreshold)){
-    if(i!=length(x$predictionThreshold)){
-      upper <- x$predictionThreshold[i+1]} else {upper <- min(x$predictionThreshold[i]+0.01,1)}
-    val <- x$predictionThreshold[i]+runif(x$out[i])*(upper-x$predictionThreshold[i])
-    vals <- c(val, vals)
-  }
-  vals[!is.na(vals)]
-  
-  vals2 <- c()
-  for(i in 1:length(x$predictionThreshold)){
-    if(i!=length(x$predictionThreshold)){
-      upper <- x$predictionThreshold[i+1]} else {upper <- min(x$predictionThreshold[i]+0.01,1)}
-    val2 <- x$predictionThreshold[i]+runif(x$nout[i])*(upper-x$predictionThreshold[i])
-    vals2 <- c(val2, vals2)
-  }
-  vals2[!is.na(vals2)]
-  
-  x <- rbind(data.frame(variable=rep('outcome',length(vals)), value=vals),
-             data.frame(variable=rep('No outcome',length(vals2)), value=vals2)
-  )
-  
-  plot <- ggplot2::ggplot(x, ggplot2::aes(x=value,
-                                          group=variable,
-                                          fill=variable)) +
-    ggplot2::geom_density(ggplot2::aes(x=value, fill=variable), alpha=.3) +
-    ggplot2::scale_x_continuous("Prediction Threshold")+#, limits=c(0,1)) +
-    ggplot2::scale_y_continuous("Density") + 
-    ggplot2::guides(fill=ggplot2::guide_legend(title="Class"))
-  
-  plotResult[[type]] <- plot
-}
-return(plotResult)
-}
-
-
-
-
-plotPreferencePDF <- function(evaluation, fileName=NULL){
-  
-  if(!is.null(evaluation$thresholdSummary$evaluation)){
-    types <- unique(evaluation$thresholdSummary$evaluation)
-  } else{
-    evaluation$thresholdSummary$evaluation <- 'na'
-    types <- 'na'
-  }
-  
-  plotResult <- list()
-  length(plotResult) <- length(types)
-  names(plotResult) <- types
-  
-  for(type in types){
-  
-  ind <- 1:nrow(evaluation$thresholdSummary)
-  if(!is.null(evaluation$thresholdSummary$evaluation)){
-    ind <- evaluation$thresholdSummary$evaluation == type
-  }
-  
-  x <- evaluation$thresholdSummary[ind,c('preferenceThreshold','truePositiveCount','trueNegativeCount',
-                                        'falsePositiveCount','falseNegativeCount')]
-  x <- x[order(x$preferenceThreshold,-x$truePositiveCount, x$trueNegativeCount),]
-  x$out <- c(x$truePositiveCount[-length(x$truePositiveCount)]-x$truePositiveCount[-1], x$truePositiveCount[length(x$truePositiveCount)])
-  x$nout <- c(x$falsePositiveCount[-length(x$falsePositiveCount)]-x$falsePositiveCount[-1], x$falsePositiveCount[length(x$falsePositiveCount)])
-  
-  vals <- c()
-  for(i in 1:length(x$preferenceThreshold)){
-    if(i!=length(x$preferenceThreshold)){
-      upper <- x$preferenceThreshold[i+1]} else {upper <- 1}
-    val <- x$preferenceThreshold[i]+runif(x$out[i])*(upper-x$preferenceThreshold[i])
-    vals <- c(val, vals)
-  }
-  vals[!is.na(vals)]
-  
-  vals2 <- c()
-  for(i in 1:length(x$preferenceThreshold)){
-    if(i!=length(x$preferenceThreshold)){
-      upper <- x$preferenceThreshold[i+1]} else {upper <- 1}
-    val2 <- x$preferenceThreshold[i]+runif(x$nout[i])*(upper-x$preferenceThreshold[i])
-    vals2 <- c(val2, vals2)
-  }
-  vals2[!is.na(vals2)]
-  
-  x <- rbind(data.frame(variable=rep('outcome',length(vals)), value=vals),
-             data.frame(variable=rep('No outcome',length(vals2)), value=vals2)
-  )
-  
-  plot <- ggplot2::ggplot(x, ggplot2::aes(x=value,
-                                          group=variable,
-                                          fill=variable)) +
-    ggplot2::geom_density(ggplot2::aes(x=value, fill=variable), alpha=.3) +
-    ggplot2::scale_x_continuous("Preference Threshold")+#, limits=c(0,1)) +
-    ggplot2::scale_y_continuous("Density") + 
-    ggplot2::guides(fill=ggplot2::guide_legend(title="Class"))
-  
-  plotResult[[type]] <- plot
-  
-}
-
-return(plotResult)
-}
-
-plotPredictionDistribution <- function(evaluation){
-  
-  if(!is.null(evaluation$predictionDistribution$evaluation)){
-    types <- unique(evaluation$predictionDistribution$evaluation)
-  } else{
-    evaluation$predictionDistribution$evaluation <- 'na'
-    types <- 'na'
-  }
-  
-  plotResult <- list()
-  length(plotResult) <- length(types)
-  names(plotResult) <- types
-  
-  for(type in types){
-  
-  ind <- 1:nrow(evaluation$predictionDistribution)
-  if(!is.null(evaluation$predictionDistribution$evaluation)){
-    ind <- evaluation$predictionDistribution$evaluation == type
-  }
-  x<- evaluation$predictionDistribution[ind,]
-  
-  #(x=Class, y=predictedProbabllity sequence:  min->P05->P25->Median->P75->P95->max)
-  
-  non05 <- x$P05PredictedProbability[x$class==0]
-  non95 <- x$P95PredictedProbability[x$class==0]
-  one05 <- x$P05PredictedProbability[x$class==1]
-  one95 <- x$P95PredictedProbability[x$class==1]
-  
-  plot <-   ggplot2::ggplot(x, 
-    ggplot2::aes(
-      x=as.factor(class),
-      ymin=MinPredictedProbability,
-      lower=P25PredictedProbability,
-      middle=MedianPredictedProbability,
-      upper=P75PredictedProbability, 
-      ymax=MaxPredictedProbability, 
-      color=as.factor(class)
-    )
-  ) + 
-    ggplot2::coord_flip() +
-    ggplot2::geom_boxplot(stat="identity")  +
-    ggplot2::scale_x_discrete("Class") + 
-    ggplot2::scale_y_continuous("Predicted Probability") + 
-    ggplot2::theme(legend.position="none") +
-    ggplot2::geom_segment(ggplot2::aes(x = 0.9, y = non05, 
-                                       xend = 1.1, yend = non05), color='red') +
-    ggplot2::geom_segment(ggplot2::aes(x = 0.9, y = non95, 
-                                       xend = 1.1, yend = non95), color='red') +
-    ggplot2::geom_segment(ggplot2::aes(x = 1.9, y = one05, 
-                                       xend = 2.1, yend = one05)) +
-    ggplot2::geom_segment(ggplot2::aes(x = 1.9, y = one95, 
-                                       xend = 2.1, yend = one95))
-  
-  plotResult[[type]] <- plot
-  
-  }
-  
- return(plotResult)
-}
-
diff --git a/inst/shiny/PLPViewer/modules/download.R b/inst/shiny/PLPViewer/modules/download.R
deleted file mode 100644
index 56f352895..000000000
--- a/inst/shiny/PLPViewer/modules/download.R
+++ /dev/null
@@ -1,122 +0,0 @@
-downloadViewer <- function(id) {
-  
-  ns <- shiny::NS(id)
-  
-  shiny::div(
-    
-    shinydashboard::box(
-      title = "Development R Package", 
-      status = 'info', 
-      solidHeader = T,
-      shiny::p("Click here to download an R package that contains all the settings requires to replicate the model development using any OMOP CDM database."),
-      shiny::actionButton(
-        inputId = ns('downloadPackageDev'), 
-        label = "Download Development"
-      )
-    ),
-    
-    shinydashboard::box(
-      title = "Validation R Package", 
-      status = 'info', 
-      solidHeader = T,
-      shiny::p("Click here to download an R package that contains all the settings requires to validate the existing model using any OMOP CDM database."),
-      shiny::actionButton(
-        inputId = ns('downloadPackageVal'), 
-        label = "Download Validation"
-      )
-      
-    )
-  )
-}
-
-downloadServer <- function(id) {
-  shiny::moduleServer(
-    id,
-    function(input, output, session) {
-      
-      shiny::observeEvent(
-        input$downloadPackageDev, 
-        {
-        dir.create(file.path('/Users/jreps/Downloads', 'devPackage'), recursive = T)
-        #Hydra::hydrate(specifications = specifications, outputFolder = outputPackageLocation)
-          createPackage <- tryCatch(
-            {downLoadSkeleton(
-              outputFolder = file.path('/Users/jreps/Downloads'),
-              packageName = 'devPackage',
-              skeletonType = 'SkeletonPredictionStudy'
-            )#'SkeletonPredictionValidationStudy'
-            }, error = function(e){return(NULL)}
-          )
-        
-        if(!is.null(createPackage)){
-          createPackage <- tryCatch(
-            {replaceName(
-              packageLocation = file.path('/Users/jreps/Downloads', 'devPackage'),
-              packageName = 'devPackage',
-              skeletonType = 'SkeletonPredictionStudy')
-            },
-            error = function(e){return(NULL)}
-          )
-        }
-        
-        
-      })
-      
-    }
-  )
-}
-
-### DOWNLOAD
-
-downLoadSkeleton <- function(
-  outputFolder,
-  packageName,
-  skeletonType = 'SkeletonPredictionStudy'
-){
-  utils::download.file(
-    url = paste0("https://github.com/ohdsi/",skeletonType,"/archive/master.zip"), 
-    destfile = file.path(outputFolder, "package.zip")
-  )
-  # unzip the .zip file
-  utils::unzip(
-    zipfile = file.path(outputFolder, "package.zip"), 
-    exdir = outputFolder
-  )
-  file.rename( 
-    from = file.path(outputFolder, paste0(skeletonType, '-master')),
-    to = file.path(outputFolder,  packageName)
-  )
-  unlink(file.path(outputFolder, "package.zip"))
-  return(file.path(outputFolder, packageName))
-}
-
-# change name
-replaceName <- function(
-  packageLocation = getwd(),
-  packageName = 'ValidateRCRI',
-  skeletonType = 'SkeletonPredictionValidationStudy'
-){
-  
-  filesToRename <- c(paste0(skeletonType,".Rproj"),paste0("R/",skeletonType,".R"))
-  for(f in filesToRename){
-    ParallelLogger::logInfo(paste0('Renaming ', f))
-    fnew <- gsub(skeletonType, packageName, f)
-    file.rename(from = file.path(packageLocation,f), to = file.path(packageLocation,fnew))
-  }
-  
-  filesToEdit <- c(
-    file.path(packageLocation,"DESCRIPTION"),
-    file.path(packageLocation,"README.md"),
-    file.path(packageLocation,"extras/CodeToRun.R"),
-    dir(file.path(packageLocation,"R"), full.names = T)
-  )
-  for( f in filesToEdit ){
-    ParallelLogger::logInfo(paste0('Editing ', f))
-    x <- readLines(f)
-    y <- gsub( skeletonType, packageName, x )
-    cat(y, file=f, sep="\n")
-    
-  }
-  
-  return(packageName)
-}
diff --git a/inst/shiny/PLPViewer/modules/netBenefit.R b/inst/shiny/PLPViewer/modules/netBenefit.R
deleted file mode 100644
index 9ad7cc132..000000000
--- a/inst/shiny/PLPViewer/modules/netBenefit.R
+++ /dev/null
@@ -1,132 +0,0 @@
-nbViewer <- function(id) {
-  ns <- shiny::NS(id)
-  
-  shiny::div(
-    
-    shiny::fluidRow(
-      shinydashboard::box(
-        status = 'info', 
-        width = 12,
-        title = 'Select net benefit type to view:',
-        solidHeader = TRUE,
-        shiny::uiOutput(ns('nbSelect'))
-      )
-    ),
-  
-    shiny::fluidRow(
-      shinydashboard::box(
-        status = 'info', 
-        width = 6,
-        title = 'Net Benefit Plot',
-        solidHeader = TRUE,
-        side = "right",
-        shinycssloaders::withSpinner(
-          shiny::plotOutput(ns('nbPlot'))
-        )
-      ),
-      
-      shinydashboard::box(
-        status = 'info', 
-        width = 6,
-        title = 'Summary',
-        solidHeader = TRUE,
-        DT::dataTableOutput(ns('nbTable'))
-      )
-    )
-  )
-}
-
-nbServer <- function(id, plpResult) {
-  shiny::moduleServer(
-    id,
-    function(input, output, session) {
-      
-      output$nbSelect = shiny::renderUI({
-        shiny::selectInput(
-          inputId = session$ns('nbSelectInput'), 
-          label = 'Type:', 
-          choices = unique(plpResult()$performanceEvaluation$thresholdSummary$evaluation), 
-          multiple = F, 
-          selectize=FALSE
-        )
-      })
-      
-      output$nbTable <- DT::renderDataTable({
-        if(is.null(plpResult()$performanceEvaluation)){
-          return(NULL)
-        } else{
-          result <- extractNetBenefit(
-            performanceEvaluation = plpResult()$performanceEvaluation, 
-            type=trimws(input$nbSelectInput)
-          )
-          unique(result)
-          result$treatAll <- format(result$treatAll, digits = 2, scientific = F)
-          result$netBenefit <- format(result$netBenefit, digits = 2, scientific = F)
-          result
-        }
-      })
-      
-      output$nbPlot <- shiny::renderPlot({
-        if(is.null(plpResult()$performanceEvaluation)){
-          return(NULL)
-        } else{
-          result <- extractNetBenefit(
-            performanceEvaluation = plpResult()$performanceEvaluation, 
-            type=trimws(input$nbSelectInput)
-          )
-          result <- unique(result)
-          ind <- !is.na(result$netBenefit) & is.finite(result$netBenefit) & !is.null(result$netBenefit) & is.finite(result$pt)
-          
-          df2 <- tidyr::pivot_longer(
-            data = result, 
-            cols = colnames(result)[colnames(result) != 'pt'], 
-            names_to = 'variable', 
-            values_to = 'value'
-          )
-          
-          
-          ggplot2::ggplot(
-            df2, 
-            ggplot2::aes(x=pt, 
-              y=value, 
-              group=variable, 
-              color = variable
-            )
-          ) +
-            ggplot2::geom_line(ggplot2::aes(linetype=variable))+
-            ggplot2::geom_point(ggplot2::aes(shape=variable))
-        }
-      })
-      
-    }
-  )
-}
-
-
-
-extractNetBenefit <- function(performanceEvaluation, type=NULL, modelId=NULL){
-  data <- performanceEvaluation$thresholdSummary
-  
-  if(!is.null(type)){
-    if(!is.null(data$evaluation[1])){
-      data <- data[data$evaluation==type,]
-    }
-  }
-  
-  pt <- data$predictionThreshold
-  TP <- data$truePositiveCount
-  FP <- data$falsePositiveCount
-  n <- data$positiveCount + data$negativeCount
-  
-  treatAll <- data$trueCount/n-data$falseCount/n*(pt/(1-pt))
-  
-  if(!is.null(modelId[1])){
-    netbenefit <- data.frame(modelId=modelId, pt=pt, netBenefit=TP/n-(FP/n)*(pt/(1-pt)),
-                             treatAll=treatAll)
-  }else{
-    netbenefit <- data.frame(pt=pt, netBenefit=TP/n-(FP/n)*(pt/(1-pt)),
-                             treatAll=treatAll)
-  }
-  
-  return(netbenefit)
-}
diff --git a/inst/shiny/PLPViewer/modules/settings.R b/inst/shiny/PLPViewer/modules/settings.R
deleted file mode 100644
index 4abbd26e2..000000000
--- a/inst/shiny/PLPViewer/modules/settings.R
+++ /dev/null
@@ -1,120 +0,0 @@
-settingsViewer <- function(id) {
-  ns <- shiny::NS(id)
-  shiny::div(
-  shiny::h3('Model Settings: ',
-            shiny::a("help", href="https://ohdsi.github.io/PatientLevelPrediction/reference/index.html", target="_blank")
-  ),
-  DT::dataTableOutput(ns('modelTable')),
-  
-  shiny::h3('Population Settings: ',
-            shiny::a("help", href="https://ohdsi.github.io/PatientLevelPrediction/reference/createStudyPopulation.html", target="_blank")
-  ),
-  DT::dataTableOutput(ns('populationTable')),
-  
-  shiny::h3('Covariate Settings: ',
-            shiny::a("help", href="http://ohdsi.github.io/FeatureExtraction/reference/createCovariateSettings.html", target="_blank")
-  ),
-  DT::dataTableOutput(ns('covariateTable')),
-  
-  shiny::h3("Hyper-parameters"),
-  DT::dataTableOutput(ns('hpTable')),
-  shiny::h3("Attrition"),
-  DT::dataTableOutput(ns('attritionTable'))
-  )
-}
-
-setingsServer <- function(id, plpResult) {
-  shiny::moduleServer(
-    id,
-    function(input, output, session) {
-      
-      # input tables
-      output$modelTable <- DT::renderDataTable(
-        formatModSettings(plpResult()$model$settings$modelSettings  )
-        )
-      output$covariateTable <- DT::renderDataTable(
-        formatCovSettings(plpResult()$model$settings$covariateSettings)
-        )
-      output$populationTable <- DT::renderDataTable(
-        formatPopSettings(plpResult()$model$settings$populationSettings)
-        )
-      
-      output$hpTable <- DT::renderDataTable(
-        DT::datatable(
-          as.data.frame(
-            plpResult()$model[[
-              which(
-                names(plpResult()$model) %in% c('validationDetails','trainDetails')
-              )
-            ]]$hyperParamSearch
-          ),
-          options = list(scrollX = TRUE),
-          colnames = 'Fold AUROC'
-        )
-      )
-      
-      output$attritionTable <- DT::renderDataTable(
-        plpResult()$model[[
-          which(
-            names(plpResult()$model) %in% c('validationDetails','trainDetails')
-            )
-          ]]$attrition
-      )
-      
-  
-    }
-  )
-}
-
-
-# helpers
-# format modelSettings
-formatModSettings <- function(modelSettings){
-  modelset <- data.frame(Setting = c(names(modelSettings$finalModelParameters)),
-                         Value = c(unlist(lapply(modelSettings$finalModelParameters, 
-                                                                     function(x) paste0(x, collapse=','))))
-    )
-  row.names(modelset) <- NULL
-  return(modelset)
-}
-
-# format covariateSettings
-formatCovSettings <- function(covariateSettings){
-  
-  if(class(covariateSettings)=='covariateSettings'){
-    covariateSettings <- list(covariateSettings)
-  }
-  
-  #code for when multiple covariateSettings
-  covariates <- c() 
-  for(i in 1:length(covariateSettings)){
-    covariatesTemp <- data.frame(
-      fun = attr(covariateSettings[[i]],'fun'),
-      setting = i,
-      covariateName = names(covariateSettings[[i]]), 
-      SettingValue = unlist(
-        lapply(
-          covariateSettings[[i]], 
-          function(x) paste0(x, collapse='-')
-        )
-      )
-    )
-    covariates  <- rbind(covariates,covariatesTemp)
-  }
-  row.names(covariates) <- NULL
-  return(covariates)
-}
-
-# format populationSettings
-formatPopSettings <- function(populationSettings){
-  population <- populationSettings
-  population$attrition <- NULL # remove the attrition as result and not setting
-  population <- data.frame(Setting = names(population), 
-                           Value = unlist(lapply(population, 
-                                                 function(x) paste0(x, 
-                                                                    collapse='-')))
-  ) 
-  row.names(population) <- NULL
-  return(population)
-}
-
diff --git a/inst/shiny/PLPViewer/modules/summaryTable.R b/inst/shiny/PLPViewer/modules/summaryTable.R
deleted file mode 100644
index 161828e9e..000000000
--- a/inst/shiny/PLPViewer/modules/summaryTable.R
+++ /dev/null
@@ -1,89 +0,0 @@
-summaryViewer <- function(id) {
-  ns <- shiny::NS(id)
-  DT::dataTableOutput(ns('summaryTable'))
-}
-
-summaryServer <- function(
-  id, 
-  summaryTable, 
-  cNamesExclude = c('studyId','Analysis','analysisId','resultId','researcherId','addExposureDaysToStart','addExposureDaysToEnd', 'plpResultLocation', 'plpResultLoad')
-) {
-  shiny::moduleServer(
-    id,
-    function(input, output, session) {
-      
-      # check if this makes drpdwn filter
-      summaryTable$T <- as.factor(summaryTable$T)
-      summaryTable$O <- as.factor(summaryTable$O)
-      
-      output$summaryTable <- DT::renderDataTable(
-        DT::datatable(
-          summaryTable[,!colnames(summaryTable)%in%cNamesExclude],
-          rownames= FALSE, 
-          selection = 'single', 
-          filter = 'top',
-          extensions = 'Buttons', 
-          options = list(
-            dom = 'Blfrtip' , 
-            buttons = c(I('colvis'), 'copy', 'excel', 'pdf' ),
-            scrollX = TRUE
-            #pageLength = 100, lengthMenu=c(10, 50, 100,200)
-          ),
-          
-          container = htmltools::withTags(
-            table(
-              class = 'display', 
-              thead(
-                #tags$th(title=active_columns[i], colnames(data)[i])
-                tr(apply(
-                  data.frame(
-                    colnames = c(
-                      'Dev', 
-                      'Val', 
-                      'T',
-                      'O', 
-                      'Model',
-                      'Covariate setting',
-                      'TAR', 
-                      'AUROC', 
-                      'AUPRC', 
-                      'T Size', 
-                      'O Count',
-                      'Val (%)', 
-                      'O Incidence (%)', 
-                      'timeStamp'
-                    ), 
-                    labels = c('Database used to develop the model', 
-                      'Database used to evaluate model', 
-                      'Target population - the patients you want to predict risk for',
-                      'Outcome - what you want to predict', 
-                      'Model type',
-                      'Id for the covariate/settings used',
-                      'Time-at-risk period', 
-                      'Area under the reciever operating characteristics (test or validation)', 
-                      'Area under the precision recall curve (test or validation)',
-                      'Target population size in the data', 
-                      'Outcome count in the data',
-                      'The percentage of data used to evaluate the model', 
-                      'Percentage of target population that have outcome during time-at-risk',
-                      'date and time of execution')
-                  ), 1,
-                  function(x) th(title=x[2], x[1])
-                )
-                )
-              )
-            )
-          )
-          
-        )
-      )
-      
-      selectedRow <- shiny::reactive({
-        input$summaryTable_rows_selected
-      })
-      
-      return(selectedRow)
-      
-    }
-  )
-}
diff --git a/inst/shiny/PLPViewer/modules/validation.R b/inst/shiny/PLPViewer/modules/validation.R
deleted file mode 100644
index 8c30c17e5..000000000
--- a/inst/shiny/PLPViewer/modules/validation.R
+++ /dev/null
@@ -1,477 +0,0 @@
-validationViewer <- function(id) {
-  ns <- shiny::NS(id)
-  
-  shiny::div(
-    style = "font-size:70%",
-    shiny::p('Select one or more rows to generate comparison ROC and calibration plots'),
-    DT::dataTableOutput(ns('validationTable')), 
-    
-    shiny::fluidRow(
-      shinydashboard::box(
-        status = 'info',
-        title = "Roc Plot",
-        solidHeader = TRUE,
-        shinycssloaders::withSpinner(shiny::plotOutput(ns('valRoc')))
-      ),
-      shinydashboard::box(
-        status = 'info',
-        title = "Calibration Plot",
-        solidHeader = TRUE,
-        side = "right",
-        shinycssloaders::withSpinner(shiny::plotOutput(ns('valCal')))
-      )
-    )
-  )
-}
-
-validationServer <- function(
-  id, 
-  plpResult,
-  result,
-  validation,
-  inputType,
-  useDatabase,
-  summaryTable,
-  resultRow,
-  con, 
-  mySchema,
-  connectionDetails,
-  targetDialect = NULL,
-  myTableAppend = NULL
-) {
-  shiny::moduleServer(
-    id,
-    function(input, output, session) {
-      
-      if (useDatabase == F){
-        validationTable <- shiny::reactive(
-          dplyr::filter(
-            summaryTable,
-            Analysis == summaryTable[resultRow(),'Analysis']
-          )
-        )
-      }
-      else{
-        # validationTable <- shiny::reactive(getValSummary(con, mySchema, summaryTable[filterIndex(),'Analysis'][trueRow()]))
-        validationTable <- shiny::reactive(
-          getValSummary(
-            con, 
-            mySchema, 
-            modelId = summaryTable[resultRow(),'Analysis'],
-            targetDialect = targetDialect, 
-            myTableAppend = myTableAppend
-          )
-        )
-      }
-      
-      #shiny::reactive({print(validationTable())})
-      #output$validationTable <- DT::renderDataTable(dplyr::select(validationTable(),c(Analysis, Dev, Val, AUC)), rownames= FALSE)
-      output$validationTable <- DT::renderDataTable({
-        
-        if(nrow(validationTable())>0){
-          
-          cind <- c('Analysis','T','O', 'Val', 'AUROC','calibrationInLarge intercept', 'T Size', 'O Count','Val (%)')%in%colnames(validationTable())
-          validationTable()[,c('Analysis','T','O', 'Val', 'AUROC','calibrationInLarge intercept', 'T Size', 'O Count','Val (%)')[cind]]
-        } else{
-          NULL
-        }
-      }, 
-        escape = FALSE, 
-        filter = 'top', 
-        extensions = 'Buttons', 
-        options = list(
-          dom = 'Blfrtip', 
-          scrollX = TRUE
-        ),
-        rownames= FALSE 
-      ) #options = list(filter = 'top'))
-      
-      # need to modify this for non-database results!
-      valtemplist <- list()
-      valResult <- shiny::reactive({
-        
-        valTable <- validationTable()[input$validationTable_rows_selected,,]
-        if(nrow(valTable)>0){
-          names <- valTable[, "Val"]
-          Ts <- valTable[, "T"]
-          Os <- valTable[, "O"]
-          for (i in 1:nrow(valTable)){
-            
-            #make i reactive
-            iReact <- shiny::reactiveVal(i)
-            
-            valtemplist[[i]] <- getPlpResult(
-              result,
-              validation,
-              valTable, 
-              inputType, 
-              iReact, 
-              val = T, 
-              mySchema = mySchema, 
-              connectionDetails = connectionDetails, 
-              targetDialect = targetDialect, 
-              myTableAppend = myTableAppend
-            )
-          }
-          list(
-            results = valtemplist, 
-            databaseName = names, 
-            Ts=Ts, 
-            Os=Os
-          )
-        }else{
-          list(
-            results = list(list()), 
-            databaseName = '', 
-            Ts='', 
-            Os=''
-          )
-        }
-      })
-      
-      output$valRoc <- shiny::renderPlot({
-        
-        if(is.null(valResult()$results[[1]]$performanceEvaluation)){
-          return(NULL)
-        } else{
-          plotRocs(
-            evaluationList = valResult()$results, 
-            modelNames = paste0(1:length(valResult()$Ts),':',substr(valResult()$Ts,1,5),'-',substr(valResult()$Os,1,5),'-', substr(valResult()$databaseName,1,5))
-          )
-        }
-      })
-      output$valCal <- shiny::renderPlot({
-        
-        if(is.null(valResult()$results[[1]]$performanceEvaluation)){
-          return(NULL)
-        } else{
-          plotCalsSmooth(
-            evaluationList = valResult()$results, 
-            modelNames =  paste0(1:length(valResult()$Ts),':',substr(valResult()$Ts,1,5),'-',substr(valResult()$Os,1,5),'-', substr(valResult()$databaseName,1,5))
-          )
-        }
-        
-      })
-      
-    }
-  )
-}
-
-
-
-# helper for multiple roc plots
-plotRocs <- function(
-  evaluationList,
-  modelNames, 
-  type= NULL, 
-  fileName=NULL
-){
-  if(class(evaluationList)!='list'){
-    stop('Need to enter a list')
-  }
-  
-  if("thresholdSummary" %in% names(evaluationList[[1]]) ){
-    evaluationList <- evaluationList
-  } else if("performanceEvaluation" %in% names(evaluationList[[1]]) ){
-    evaluationList <- lapply(evaluationList, function(x) x$performanceEvaluation)
-  } else {
-    stop('Wrong evaluationList')
-  }
-  
-  if(missing(modelNames)){
-    modelNames <- paste0('Model ', 1:length(evaluationList))
-  }
-  
-  createSteps <- function(
-    evaluation, 
-    type, 
-    name
-  ){
-    
-    if(is.null(type)){
-      if(length(unique(evaluation$thresholdSummary$evaluation)) > 1){
-        ind <- evaluation$thresholdSummary$evaluation%in%c('Test','validation')
-        x<- evaluation$thresholdSummary[ind,c('falsePositiveRate','sensitivity')]
-      } else {
-        x<- evaluation$thresholdSummary[,c('falsePositiveRate','sensitivity')]
-      }
-    } else {
-      ind <- evaluation$thresholdSummary$evaluation == type
-      x <- evaluation$thresholdSummary[ind,c('falsePositiveRate','sensitivity')]
-    }
-    
-    x <- x[order(x$falsePositiveRate, x$sensitivity),]
-    
-    # add the bit to get the step
-    stepsExtra <- cbind(x[-1,1], x[-nrow(x),2])
-    colnames(stepsExtra) <- colnames(x)
-    x <- rbind(c(1,1), x, stepsExtra, c(0,0))
-    x <- x[order(x$falsePositiveRate, x$sensitivity),]
-    
-    x$model <- name
-    return(x)
-  }
-  
-  stepVals <- lapply(
-    1:length(evaluationList), 
-    function(i){
-      createSteps(
-        evaluationList[[i]], 
-        type=type[i], 
-        name=modelNames[i]
-        )
-    }
-  )
-  data <- do.call(rbind, stepVals)
-  
-  plot <- ggplot2::ggplot(
-    data = data, 
-    ggplot2::aes(
-      x = falsePositiveRate, 
-      y = sensitivity, 
-      color = model
-    )
-  ) +
-    ggplot2::geom_polygon(
-      ggplot2::aes(fill = model), 
-      alpha = 0.2
-    ) +
-    ggplot2::geom_line(size = 1) +
-    ggplot2::geom_abline(
-      intercept = 0, 
-      slope = 1,
-      linetype = 2
-    ) +
-    ggplot2::scale_x_continuous(
-      "1 - specificity", 
-      limits=c(0,1)
-    ) +
-    ggplot2::scale_y_continuous(
-      "Sensitivity", 
-      limits=c(0,1)
-    ) +
-    ggplot2::scale_color_discrete(name = 'Result') +
-    ggplot2::scale_fill_discrete(guide = FALSE)
-  
-  if (!is.null(fileName)){
-    ggplot2::ggsave(fileName, plot, width = 5, height = 4.5, dpi = 400)
-  }
-  
-  return(plot)
-}
-
-plotCals <- function(
-  evaluationList,
-  modelNames, 
-  type = NULL, 
-  fileName = NULL
-){
-  
-  if("calibrationSummary" %in% names(evaluationList[[1]]) ){
-    evaluationList <- evaluationList
-  }else if("performanceEvaluation" %in% names(evaluationList[[1]]) ){
-    evaluationList <- lapply(evaluationList, function(x) x$performanceEvaluation)
-  } else{
-    stop('Wrong evaluationList')
-  }
-  
-  if(missing(modelNames)){
-    modelNames <- paste0('Model ', 1:length(evaluationList))
-  }
-  
-  calVal <- function(
-    evaluation, 
-    type, 
-    name
-  ){
-    
-    if(is.null(type)){
-      if(length(unique(evaluation$calibrationSummary$evaluation)) > 1){
-        ind <- evaluation$calibrationSummary$evaluation%in%c('Test','validation')
-        x<- evaluation$calibrationSummary[ind,c('averagePredictedProbability','observedIncidence','PersonCountAtRisk')]
-      } else{
-        x<- evaluation$calibrationSummary[,c('averagePredictedProbability','observedIncidence','PersonCountAtRisk')]
-      }
-    } else{
-      ind <- evaluation$calibrationSummary$evaluation == type
-      x<- evaluation$calibrationSummary[ind,c('averagePredictedProbability','observedIncidence','PersonCountAtRisk')]
-    }
-    
-    cis <- apply(
-      x, 
-      1,
-      function(x){
-        stats::binom.test(
-          x[2]*x[3], 
-          x[3], 
-          alternative = c("two.sided"), 
-          conf.level = 0.95
-        )$conf.int
-      }
-    )
-    x$lci <- cis[1,]
-    x$uci <- cis[2,]
-    x$model <- name
-    return(x)
-  }
-  
-  calVal <- lapply(1:length(evaluationList), function(i) calVal(evaluationList[[i]], type=type[i], name=modelNames[i]))
-  data <- do.call(rbind, calVal)
-  
-  maxes <- max(max(data$averagePredictedProbability), max(data$observedIncidence))*1.1
-  
-  limits <- ggplot2::aes(
-    ymax = uci, 
-    ymin= lci
-  )
-  
-  plot <- ggplot2::ggplot(
-    data = data,
-    ggplot2::aes(
-      x = averagePredictedProbability, 
-      y = observedIncidence,
-      color = model
-    )
-  ) +
-    ggplot2::geom_point(size=2) +
-    ggplot2::geom_errorbar(limits) +
-    ggplot2::geom_line() +
-    ggplot2::geom_abline(
-      intercept = 0, 
-      slope = 1, 
-      linetype = 5, 
-      size=0.4,
-      show.legend = TRUE
-    ) +
-    ggplot2::scale_x_continuous("Average Predicted Probability") +
-    ggplot2::scale_y_continuous("Observed Fraction With Outcome") +
-    ggplot2::coord_cartesian(
-      xlim = c(0, maxes), 
-      ylim = c(0,maxes)
-    ) +
-    ggplot2::scale_color_discrete(name = 'Result')
-  
-  if (!is.null(fileName)){
-    ggplot2::ggsave(fileName, plot, width = 5, height = 4.5, dpi = 400)
-  }
-  
-  return(plot)
-}
-
-
-
-plotCalsSmooth <- function(
-  evaluationList,
-  modelNames, 
-  type = NULL
-  ){
-  
-  if("calibrationSummary" %in% names(evaluationList[[1]]) ){
-    evaluationList <- evaluationList
-  }else if("performanceEvaluation" %in% names(evaluationList[[1]]) ){
-    evaluationList <- lapply(evaluationList, function(x) x$performanceEvaluation)
-  } else{
-    stop('Wrong evaluationList')
-  }
-  
-  if(missing(modelNames))
-    modelNames <- paste0('Model ', 1:length(evaluationList))
-  
-  calVal <- function(
-    evaluation, 
-    type, 
-    name
-    ){
-    
-    if(is.null(type)){
-      if(length(unique(evaluation$calibrationSummary$evaluation)) > 1){
-        ind <- evaluation$calibrationSummary$evaluation %in% c('Test','validation')
-        data <- evaluation$calibrationSummary[ind,c('averagePredictedProbability','observedIncidence','PersonCountAtRisk')]
-      } else{
-        data <- evaluation$calibrationSummary[,c('averagePredictedProbability','observedIncidence','PersonCountAtRisk')]
-      }
-    } else{
-      ind <- evaluation$calibrationSummary$evaluation==type
-      data <- evaluation$calibrationSummary[ind,c('averagePredictedProbability','observedIncidence','PersonCountAtRisk')]
-    }
-    
-    maxes <- max(max(data$averagePredictedProbability), max(data$observedIncidence))*1.1
-    
-    fit <- stats::loess(data$observedIncidence ~ data$averagePredictedProbability, degree = 1)
-    smoothData <- data.frame(
-      p = seq(0,maxes,0.0001), 
-      y = stats::predict(fit, seq(0,maxes,0.0001)), 
-      model = name
-    )
-    smoothData <- smoothData[!is.na(smoothData$y),]
-    
-    return(smoothData)
-  }
-  
-  getVal <- function(
-    evaluation, 
-    type, 
-    name
-    ){
-    
-    if(is.null(type)){
-      if(length(unique(evaluation$calibrationSummary$evaluation)) > 1){
-        ind <- evaluation$calibrationSummary$evaluation %in% c('Test','validation')
-        data <- evaluation$calibrationSummary[ind, c('averagePredictedProbability','observedIncidence')]
-      } else{
-        data <- evaluation$calibrationSummary[, c('averagePredictedProbability','observedIncidence')]
-      }
-    } else{
-      ind <- evaluation$calibrationSummary$evaluation == type
-      data <- evaluation$calibrationSummary[ind, c('averagePredictedProbability','observedIncidence')]
-    }
-    
-
-    values <- data.frame(
-      p = data$averagePredictedProbability, 
-      y = data$observedIncidence, 
-      model = name
-    )
-    
-    values <- values[seq(1, nrow(values), 10),]
-    
-    return(values)
-  }
-  
-  calVal<- lapply(1:length(evaluationList), function(i) calVal(evaluationList[[i]], type=type[i], name=modelNames[i]))
-  smoothData <- do.call(rbind, calVal)
-  
-  values <- do.call(rbind,lapply(1:length(evaluationList), function(i) getVal(evaluationList[[i]], type=type[i], name=modelNames[i])))
-  
-  plot <- ggplot2::ggplot(
-    data = smoothData, 
-    ggplot2::aes(
-      x = .data$p, 
-      y = .data$y, 
-      color = .data$model
-      )
-    ) +
-    ggplot2::geom_line(
-      ggplot2::aes(
-        linetype = "Loess"),
-        size = 1,
-        show.legend = T
-      ) +
-    ggplot2::geom_abline(
-      intercept = 0, 
-      slope = 1, 
-      linetype = 5, 
-      size=0.4,
-      show.legend = TRUE, 
-      color = "black"
-      ) +
-    ggplot2::geom_point(data = values) +
-    ggplot2::labs(
-      x = "Average Predicted Probability", 
-      y = "Observed Fraction With Outcome"
-      ) 
-    #ggplot2::scale_color_discrete(name = 'Result')
-  
-  return(plot)
-}
-
diff --git a/inst/shiny/PLPViewer/processing.R b/inst/shiny/PLPViewer/processing.R
deleted file mode 100644
index 3c04c4b77..000000000
--- a/inst/shiny/PLPViewer/processing.R
+++ /dev/null
@@ -1,297 +0,0 @@
-settingsNames <- c('analysisId','modelSettings','covariateSetting', 'targetName', 'outcomeName',
-  'populationSetting','modelSettingName')
-
-getSummary  <- function(result,inputType,validation){
-  if(inputType == 'plpResult' || inputType == 'plpNoClass'){
-    
-    sumTab <- getSummaryFromObject(result)
-    
-    if(!is.null(validation)){ # what about a list of validations?
-      sumTab <- rbind(sumTab, getSummaryFromObject(validation))
-    }
-    
-  } else if( inputType == 'file') {
-    sumTab <- summaryPlpAnalyses(result)
-  } 
-  
-  #remove empty rows
-  emptyInd <- is.na(sumTab[,'AUROC'])
-  if(sum(emptyInd)>0){
-    sumTab <- sumTab[!emptyInd,]
-  }
-  
-  columnsOfInt <- c('analysisId',
-    'devDatabase', 'valDatabase',
-    'targetName', 'outcomeName',
-    'modelSettingName','covariateSetting', 
-    'TAR', 'AUROC','AUPRC',
-    'populationSize', 'outcomeCount',
-    'valPercent', 'incidence',
-    'timeStamp', 'plpResultLocation', 'plpResultLoad'
-    )
-  
-  # add missing columns
-  if(sum(!columnsOfInt %in% colnames(sumTab))>0){
-    missInd <- columnsOfInt[!columnsOfInt %in% colnames(sumTab)]
-    for(i in 1:length(missInd)){
-      sumTab[,missInd[i]] <- 'NA'
-    }
-  }
-  
-  sumTab <- sumTab[,columnsOfInt]
-  
-  colnames(sumTab) <- c(
-    'Analysis','Dev', 'Val', 'T', 'O','Model','covariateSetting', 
-    'TAR', 'AUROC','AUPRC', 'T Size','O Count','% used for Eval','O Incidence (%)',
-    'timeStamp', 'plpResultLocation', 'plpResultLoad'
-  )
-  
-  return(sumTab)
-} 
-
-
-getSummaryFromObject <- function(result, analysisId = NULL){
-  
-  timeV <- ifelse(is.null(result$executionSummary$ExecutionDateTime), '2000-01-01', result$executionSummary$ExecutionDateTime)
-
-  TAR <- getTAR(result$model$settings$populationSettings)
-  eval <- as.data.frame(result$performanceEvaluation$evaluationStatistics)
-  
-  for(i in 1:ncol(eval)){
-    eval[,i] <- unlist(eval[,i])
-  }
-  
-  if(is.null(eval$evaluation)){
-    eval$evaluation <- 'Test'
-  }
-  if(length(unique(eval$evaluation)) == 1){
-    eval$evaluation <- 'Test'
-  }
-  
-  eval <- tidyr::pivot_wider( 
-    data = eval %>% 
-      dplyr::mutate(variable = paste(.data$evaluation, .data$metric, sep = '_')) %>% 
-      dplyr::select(-.data$evaluation, -.data$metric), 
-    names_from = 'variable', 
-    values_from = 'value'
-      ) 
-  
-  #eval <- reshape2::dcast(
-  #  data = eval, 
-  #  formula = . ~ evaluation + metric, 
-  #  value.var = 'value' 
-  #  )
-  
-  AUC <- paste0(
-    signif(as.double(eval$Test_AUROC),3),
-    ' (',
-    signif(as.double(eval$`Test_95% lower AUROC`),3),
-    '-',
-    signif(as.double(eval$`Test_95% upper AUROC`),3),
-    ')'
-    )
-  
-  AUPRC <- signif(as.double(eval$Test_AUPRC),3)
-  
-  populationSize <- ifelse(is.null(eval$Train_populationSize), 0, as.double(eval$Train_populationSize)) + as.double(eval$Test_populationSize)
-  outcomeCount <- ifelse(is.null(eval$Train_outcomeCount), 0, as.double(eval$Train_outcomeCount)) + as.double(eval$Test_outcomeCount)
-  valPercent <- round(as.double(eval$Test_populationSize)/populationSize*100)
-  incidence <- signif(100*outcomeCount/populationSize ,3)
-  
-  if(!is.null(result$model$trainDetails)){
-    devDatabase <- ifelse(is.null(result$model$trainDetails$cdmDatabaseSchema),'Missing',result$model$trainDetails$cdmDatabaseSchema)
-    valDatabase <- devDatabase
-  } else{
-    devDatabase <- ifelse(is.null(result$model$validationDetails$developmentDatabase),'Missing',result$model$validationDetails$developmentDatabase)
-    valDatabase <- ifelse(is.null(result$model$validationDetails$cdmDatabaseSchema),'Missing',result$model$validationDetails$cdmDatabaseSchema)
-}
-  allRes <- data.frame(analysisId = ifelse(is.null(analysisId), ifelse(is.null(result$analysisRef$analysisId), 'None', result$analysisRef$analysisId), analysisId),
-                       devDatabase = devDatabase,
-                       valDatabase = valDatabase,
-                       cohortName = 'T', # needed?
-                       outcomeName = 'O', # needed?
-                       modelSettingName = result$model$settings$modelSettings$model,
-                       #covariateSetting = 1,
-                       TAR = TAR,
-                       AUROC = AUC,
-                       AUPRC = AUPRC,
-                       populationSize = populationSize,
-                       outcomeCount = outcomeCount,
-                       valPercent = valPercent,
-                       incidence = incidence,
-                       timeStamp = timeV,
-                       plpResultLocation = 'NULL', 
-                       plpResultLoad = 'NULL'
-  )
-  
-  return(allRes)
-}
-
-
-
-summaryPlpAnalyses <- function(analysesLocation){ 
-  # loads the analyses and validations to get summaries
-  #========================================
-  settings <- utils::read.csv(file.path(analysesLocation,'settings.csv'))
-  settings <- settings[,!colnames(settings)%in%c('plpDataFolder','studyPopFile','plpResultFolder')]
-  settings$analysisId <- gsub('Analysis_','', settings$analysisId) # fixing if Analysis_id in settings
-  settings$analysisId <- paste0('Analysis_',  settings$analysisId)
-  
-  analysisIds <- dir(file.path(analysesLocation), recursive = F, full.names = T)
-  analysisIds <- analysisIds[grep('Analysis_',analysisIds)]
-  
-  devPerformance <- do.call(rbind,lapply(file.path(analysisIds), getPerformance))
-  
-  # updated this
-  devPerformance <- merge(settings[,settingsNames[settingsNames %in% colnames(settings)]],
-                          devPerformance[, !colnames(devPerformance) %in% c('cohortName','outcomeName')[c('cohortName','outcomeName') %in% colnames(settings)]], by='analysisId', all.x=T)
-  
-  validationLocation <- file.path(analysesLocation,'Validation')
-  if(length(dir(validationLocation))>0){
-    valPerformances <- c()
-    valDatabases <- dir(validationLocation, recursive = F, full.names = T)
-    if(length(grep('plplog.txt', valDatabases))>0){
-      valDatabases <- valDatabases[-grep('plplog.txt', valDatabases)]
-    }
-    for(valDatabase in valDatabases){
-      
-      valAnalyses <-  dir(valDatabase, recursive = F, full.names = T)
-      valAnalyses <-  valAnalyses[grep('Analysis_', valAnalyses)]
-      valPerformance <- do.call(rbind,lapply(file.path(valAnalyses), function(x) getPerformance(x)))
-      
-      valSettings <- settings[,settingsNames[settingsNames %in% colnames(settings)]] # removed TAR bits
-      valPerformance <- merge(
-        valSettings,
-        valPerformance[, !colnames(valPerformance) %in% c('cohortName','outcomeName')[c('cohortName','outcomeName') %in% colnames(settings)]],
-          by='analysisId'
-      )
-      valPerformance <- valPerformance[,colnames(devPerformance)] # make sure same order
-      valPerformances <- rbind(valPerformances, valPerformance)
-    }
-    
-    if(ncol(valPerformances)==ncol(devPerformance)){
-      allPerformance <- rbind(devPerformance,valPerformances)
-    } else{
-      stop('Issue with dev and val performance data.frames')
-    }
-  } else {
-    allPerformance <- devPerformance
-  }
-  
-  return(allPerformance)
-}
-
-getPerformance <- function(analysisLocation){
-  
-  getType <- function(analysisLocation){
-    
-    if(file.exists(file.path(analysisLocation, 'plpResult.rds'))){
-      return('rds')
-    }
-    
-    if(file.exists(file.path(analysisLocation, 'validationResult.rds'))){
-      return('rds')
-    }
-    
-    if(dir.exists(file.path(analysisLocation, 'plpResult'))){
-      return('runPlp')
-    }
-    
-    if(dir.exists(file.path(analysisLocation, 'validationResult'))){
-      return('runPlp')
-    }
-    
-    
-    if(dir.exists(file.path(analysisLocation, 'performanceEvaluation'))){
-      return('csv')
-    }
-    
-    return('none')
-    
-  }
-  
-  analysisId <- strsplit(analysisLocation, '/')[[1]]
-  analysisId <- analysisId[length(analysisId)]
-  
-  type <- getType(analysisLocation) # csv, rds, runPlp
-  print(type)
-  
-  if(type == 'csv'){
-    
-    require(PatientLevelPrediction)
-    res <- PatientLevelPrediction::loadPlpShareable(file.path(analysisLocation))
-    result <- getSummaryFromObject(result = res, analysisId = analysisId)
-    location <- file.path(analysisLocation)
-    plpResultLoad <- 'loadPlpShareable'
-    
-  } else if(type == 'rds'){
-    
-    # read rds here
-    res <- readRDS(file.path(analysisLocation,'plpResult.rds'))
-    result <- getSummaryFromObject(result = res, analysisId = analysisId)
-    location <- file.path(analysisLocation, 'plpResult.rds')
-    plpResultLoad <- 'readRDS'
-    
-  } else if(type == 'runPlp'){
-    
-    location <- file.path(analysisLocation, 'plpResult')
-    if(!dir.exists(location)){
-      location <- file.path(analysisLocation, 'validationResult')
-    }
-    
-    require(PatientLevelPrediction)
-    res <- loadPlpResult(location)
-    result <- getSummaryFromObject(result = res, analysisId = analysisId)
-    plpResultLoad <- 'loadPlpResult'
-    
-  } else{
-    # return empty result
-    analysisId <- strsplit(analysisLocation, '/')[[1]]
-    result <- data.frame(
-      AnalysisId = analysisId, 
-      devDatabase = 'missing',
-      valDatabase = 'missing',
-      targetName = 'T', # NEEDED?
-      outcomeName = 'O', # NEEDED?
-      modelSettingName = 'none',
-      #covariateSetting = 1,
-      TAR = '?',
-      AUROC = 0.000,
-      AUPRC = 0,
-      populationSize = 0,
-      outcomeCount = 0,
-      valPercent = 0,
-      incidence = 0,
-      timeStamp = as.Date('1900-01-01'),
-      plpResultLocation = '', 
-      plpResultLoad = 'loadPlpResult'
-      )
-    location <- ''
-    plpResultLoad <- 'loadPlpResult'
-
-  }
-  
-  result$plpResultLocation <- location
-  result$plpResultLoad <- plpResultLoad
-  
-  #remove settings
-  result[,!colnames(result) %in% settingsNames]
-  
-  return(result)
-}
-
-
-getTAR <- function(x){
-  starttar <- unique(x$startAnchor)
-  if(is.null(starttar)){
-    starttar <- ifelse(unique(x$addExposureDaysToStart), 'cohort end','cohort start')
-  }
-  endtar <- unique(x$endAnchor)
-  if(is.null(endtar)){
-    endtar <- ifelse(unique(x$addExposureDaysToEnd), 'cohort end','cohort start')
-  }
-  TAR <- paste0('(',starttar,' + ',unique(x$riskWindowStart),') - (', endtar,' + ',unique(x$riskWindowEnd),')')
-  return(TAR)
-  #return('cohort start + 1 - cohort start + 365')
-}
-
diff --git a/inst/shiny/PLPViewer/server.R b/inst/shiny/PLPViewer/server.R
deleted file mode 100644
index afe359d6a..000000000
--- a/inst/shiny/PLPViewer/server.R
+++ /dev/null
@@ -1,169 +0,0 @@
-# @file server.R
-#
-# Copyright 2021 Observational Health Data Sciences and Informatics
-#
-# This file is part of PatientLevelPrediction
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-source("helpers.R")
-source("emptyPlot.R")
-
-source("modules/summaryTable.R")
-source("modules/covariateSummary.R")
-source("modules/settings.R")
-source("modules/cutoff.R")
-source("modules/discrimination.R")
-source("modules/calibration.R")
-source("modules/netBenefit.R")
-source("modules/validation.R")
-source("modules/download.R")
-
-server <- shiny::shinyServer(function(input, output, session) {
-  session$onSessionEnded(shiny::stopApp)
-
-  #=============
-  # sidebar menu
-  #=============
-  if(useDatabase == F){
-    
-    output$sidebarMenu <- shinydashboard::renderMenu(shinydashboard::sidebarMenu(id ='menu',
-                                                                                 addInfo(shinydashboard::menuItem("Description", tabName = "Description", icon = shiny::icon("home")), "DescriptionInfo"),
-                                                                                 addInfo(shinydashboard::menuItem("Results", tabName = "Summary", icon = shiny::icon("table")), "SummaryInfo"),
-                                                                                 #addInfo(shinydashboard::menuItem("Log", tabName = "Log", icon = shiny::icon("list")), "LogInfo"),
-                                                                                 addInfo(shinydashboard::menuItem("Data Info", tabName = "DataInfo", icon = shiny::icon("database")), "DataInfoInfo"),
-                                                                                 addInfo(shinydashboard::menuItem("Help", tabName = "Help", icon = shiny::icon("info")), "HelpInfo")
-    ))
-  } else {
-    
-    shiny::observe({
-      studyId <- shiny::parseQueryString(session$clientData$url_search)[['studyId']]
-      
-      print(paste0('StudyId: ', studyId))
-      if(!is.null(studyId)){
-        summaryTable <- summaryTable[summaryTable$studyId == studyId, ]
-      }
-
-      })
-    
-    output$sidebarMenu <- shinydashboard::renderMenu(shinydashboard::sidebarMenu(id ='menu',
-                                                                                 addInfo(shinydashboard::menuItem("Description", tabName = "Description", icon = shiny::icon("home")), "DescriptionInfo"),
-                                                                                 addInfo(shinydashboard::menuItem("Results", tabName = "Summary", icon = shiny::icon("table")), "SummaryInfo"),
-                                                                                addInfo(shinydashboard::menuItem("Data Info", tabName = "DataInfo", icon = shiny::icon("database")), "DataInfoInfo"),
-                                                                                 addInfo(shinydashboard::menuItem("Help", tabName = "Help", icon = shiny::icon("info")), "HelpInfo")
-    ))
-  }
-  
-  # ===========================================
-  #  RESULT viewer
-  # ===========================================
-  
-  # use the summary module to select a result via row selection
-  resultRow <- summaryServer('sumTab', summaryTable)
-  
-  # change to single model explore tab when summary table row is selected
-  shiny::observeEvent(resultRow(), {
-    shiny::updateTabsetPanel(session, "allView", selected = "Explore Selected Model")
-  })
-  
-  # this loads all the results
-  plpResult <- shiny::reactive({getPlpResult(result,
-                                             validation,
-                                             summaryTable, 
-                                             inputType, 
-                                             val = F, 
-                                             resultRow,
-                                             mySchema = mySchema, 
-                                             connectionDetails = connectionDetails,
-                                             targetDialect = targetDialect, 
-                                             myTableAppend = myTableAppend)})
-  
-  
-  # ===========================================
-  #  Single Result Exploring Modules
-  # ===========================================
-  
-  covariateSummaryServer('covariateSummary',
-                         plpResult,
-                         summaryTable, 
-                         resultRow, 
-                         mySchema, 
-                         con,
-                         inputSingleView = input$singleView,
-                         myTableAppend = myTableAppend, 
-                         targetDialect = targetDialect) 
-  
-  setingsServer('settings', 
-                plpResult)
-  
-  cutoffServer('cutoff', 
-               plpResult)
-  
-  discriminationServer('discrimination', 
-                       plpResult)
-  
-  calibrationServer('calibration', 
-                    plpResult) 
-  
-  nbServer('netBenefit', 
-           plpResult) 
-  
-  validationServer('validation', 
-                   result,
-                   validation,
-                   plpResult = plpResult,
-                   inputType = inputType,
-                   useDatabase = useDatabase,
-                   summaryTable = summaryTable,
-                   resultRow = resultRow,
-                   con = con, 
-                   mySchema = mySchema,
-                   connectionDetails = connectionDetails,
-                   myTableAppend = myTableAppend, 
-                   targetDialect = targetDialect) 
-  
-  
-  downloadServer('download')
-  #=======================
-  # get researcher info
-  #=======================
-  output$researcherInfo <- shiny::renderTable(plpResult()$researcherInfo)
-  
-  # HELPER INFO
-  shiny::observeEvent(input$DescriptionInfo, {
-    showInfoBox("Description", "html/Description.html")
-  })
-  shiny::observeEvent(input$SummaryInfo, {
-    showInfoBox("Summary", "html/Summary.html")
-  })
-  shiny::observeEvent(input$PerformanceInfo, {
-    showInfoBox("Performance", "html/Performance.html")
-  })
-  shiny::observeEvent(input$ModelInfo, {
-    showInfoBox("Model", "html/Model.html")
-  })
-  shiny::observeEvent(input$LogInfo, {
-    showInfoBox("Log", "html/Log.html")
-  })
-  shiny::observeEvent(input$SettingsInfo, {
-    showInfoBox("Settings", "html/Settings.html")
-  })
-  shiny::observeEvent(input$DataInfoInfo, {
-    showInfoBox("DataInfo", "html/DataInfo.html")
-  })
-  shiny::observeEvent(input$HelpInfo, {
-    showInfoBox("HelpInfo", "html/Help.html")
-  })
-  
- 
-})
diff --git a/inst/shiny/PLPViewer/ui.R b/inst/shiny/PLPViewer/ui.R
deleted file mode 100644
index ebee9550f..000000000
--- a/inst/shiny/PLPViewer/ui.R
+++ /dev/null
@@ -1,163 +0,0 @@
-# @file Ui.R
-#
-# Copyright 2021 Observational Health Data Sciences and Informatics
-#
-# This file is part of PatientLevelPrediction
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-source("modules/summaryTable.R")
-source("modules/covariateSummary.R")
-source("modules/settings.R")
-source("modules/cutoff.R")
-source("modules/discrimination.R")
-source("modules/calibration.R")
-source("modules/netBenefit.R")
-source("modules/validation.R")
-source("modules/download.R")
-
-addInfo <- function(item, infoId) {
-  infoTag <- tags$small(
-    class = "badge pull-right action-button",
-    style = "padding: 1px 6px 2px 6px; background-color: steelblue;",
-    type = "button", 
-    id = infoId,
-    "i"
-  )
-  item$children[[1]]$children <- append(item$children[[1]]$children, list(infoTag))
-  return(item)
-}
-
-ui <- shinydashboard::dashboardPage(
-  skin = 'black',
-  
-  shinydashboard::dashboardHeader(
-    title = "PLP Viewer", 
-    tags$li(
-      div(
-        img(
-          src = 'logo.png',
-          title = "OHDSI PLP", 
-          height = "40px", 
-          width = "40px"),
-        style = "padding-top:0px; padding-bottom:0px;"
-      ),
-      class = "dropdown"
-    )
-  ), 
-  
-  shinydashboard::dashboardSidebar(
-    shinydashboard::sidebarMenuOutput("sidebarMenu")
-  ), # end sidebar
-  
-  shinydashboard::dashboardBody(
-    shinydashboard::tabItems(
-      
-      # help tab
-      shinydashboard::tabItem(
-        tabName = "Help",
-        shiny::h2("Information"),
-        shiny::p("Click on a row to explore the results for that model.  When you wish to explore a different model, then select the new result row and the tabs will be updated."),
-        shiny::a("Demo Video", href = 'https://youtu.be/StpV40yl1UE', target='_blank')
-      ),
-      
-      # First tab content
-      shinydashboard::tabItem(
-        tabName = "Description",
-        shiny::includeMarkdown(path = pathToMd)
-      ),
-      shinydashboard::tabItem(
-        tabName = "DataInfo",
-        shiny::includeMarkdown(path = "./www/dataInfo.md")
-      ),
-      shinydashboard::tabItem(
-        tabName = "Summary",
-        # do this inside tabs:
-        shiny::tabsetPanel(
-          id = 'allView',
-          shiny::tabPanel(
-            "All Models Summary",  
-            summaryViewer('sumTab') 
-          ),
-          
-          shiny::tabPanel(
-            "Explore Selected Model",
-            
-            shiny::tabsetPanel(
-              id = 'singleView',
-              shiny::tabPanel(
-                "Development Settings",
-                settingsViewer('settings')
-              ),
-              
-              shiny::tabPanel(
-                "Model",
-                covariateSummaryViewer('covariateSummary')
-              ),
-              
-              shiny::tabPanel(
-                "Threshold Dependant", 
-                cutoffViewer('cutoff')
-              ), 
-              
-              shiny::tabPanel(
-                "Discrimination",  
-                discriminationViewer('discrimination')
-              ),
-              
-              shiny::tabPanel(
-                "Calibration", 
-                calibrationViewer('calibration')
-              ),
-              
-              shiny::tabPanel(
-                "Net Benefit", 
-                nbViewer('netBenefit')
-              ),
-              
-              shiny::tabPanel(
-                "Validation",
-                validationViewer('validation')
-              ),
-              
-              shiny::tabPanel(
-                "Developer Info",
-                shinydashboard::box(status = 'info',
-                  title = "Developer Info",
-                  solidHeader = TRUE,
-                  side = "right",
-                  shiny::tableOutput('researcherInfo')
-                )
-              ),
-              
-              shiny::tabPanel(
-                "Download Model",
-                downloadViewer('download')
-              )
-              
-            )
-          )
-          
-          
-          
-          
-        )
-      )
-      
-      
-      
-      
-      
-    )
-  )
-)
diff --git a/inst/shiny/PLPViewer/www/about.png b/inst/shiny/PLPViewer/www/about.png
deleted file mode 100644
index 878a509e2..000000000
Binary files a/inst/shiny/PLPViewer/www/about.png and /dev/null differ
diff --git a/inst/shiny/PLPViewer/www/custom.css b/inst/shiny/PLPViewer/www/custom.css
deleted file mode 100644
index 26730bbd9..000000000
--- a/inst/shiny/PLPViewer/www/custom.css
+++ /dev/null
@@ -1,81 +0,0 @@
-.plotly.html-widget.html-widget-output.shiny-bound-output.js-plotly-plot {
-  z-index: 22;
-  position: relative; 
-}
-
-.plotlybars {
-  padding: 0 10px;
-  vertical-align: bottom;
-  width: 100%;
-  height: 100%;
-  overflow: hidden;
-  position: relative;
-  box-sizing: border-box;
-}
-
-.plotlybars-wrapper {
-  width: 165px;
-  height: 100px;
-  margin: 0 auto;
-  left: 0;
-  right: 0;
-  position: absolute;
-  z-index: 1;
-}
-
-.plotlybars-text {
-  color: #447adb;
-  font-family: 'Open Sans', verdana, arial, sans-serif;
-  font-size: 80%;
-  text-align: center;
-  margin-top: 5px;
-}
-
-.plotlybars-bar {
-  background-color: #447adb;
-  height: 100%;
-  width: 13.3%;
-  position: absolute;
-
-  -webkit-transform: translateZ(0);
-  transform: translateZ(0);
-
-  animation-duration: 2s;
-  animation-iteration-count: infinite;
-  animation-direction: normal;
-  animation-timing-function: linear;
-
-  -webkit-animation-duration: 2s;
-  -webkit-animation-iteration-count: infinite;
-  -webkit-animation-direction: normal;
-  -webkit-animation-timing-function: linear;
-}
-
-.b1 { left: 0%; top: 88%; animation-name: b1; -webkit-animation-name: b1; }
-.b2 { left: 14.3%; top: 76%; animation-name: b2; -webkit-animation-name: b2; }
-.b3 { left: 28.6%; top: 16%; animation-name: b3; -webkit-animation-name: b3; }
-.b4 { left: 42.9%; top: 40%; animation-name: b4; -webkit-animation-name: b4; }
-.b5 { left: 57.2%; top: 26%; animation-name: b5; -webkit-animation-name: b5; }
-.b6 { left: 71.5%; top: 67%; animation-name: b6; -webkit-animation-name: b6; }
-.b7 { left: 85.8%; top: 89%; animation-name: b7; -webkit-animation-name: b7; }
-
-@keyframes b1 { 0% { top: 88%; } 44% { top: 0%; } 94% { top: 100%; } 100% { top: 88%; } }
-@-webkit-keyframes b1 { 0% { top: 88%; } 44% { top: 0%; } 94% { top: 100%; } 100% { top: 88%; } }
-
-@keyframes b2 { 0% { top: 76%; } 38% { top: 0%; } 88% { top: 100%; } 100% { top: 76%; } }
-@-webkit-keyframes b2 { 0% { top: 76%; } 38% { top: 0%; } 88% { top: 100%; } 100% { top: 76%; } }
-
-@keyframes b3 { 0% { top: 16%; } 8% { top: 0%; } 58% { top: 100%; } 100% { top: 16%; } }
-@-webkit-keyframes b3 { 0% { top: 16%; } 8% { top: 0%; } 58% { top: 100%; } 100% { top: 16%; } }
-
-@keyframes b4 { 0% { top: 40%; } 20% { top: 0%; } 70% { top: 100%; } 100% { top: 40%; } }
-@-webkit-keyframes b4 { 0% { top: 40%; } 20% { top: 0%; } 70% { top: 100%; } 100% { top: 40%; } }
-
-@keyframes b5 { 0% { top: 26%; } 13% { top: 0%; } 63% { top: 100%; } 100% { top: 26%; } }
-@-webkit-keyframes b5 { 0% { top: 26%; } 13% { top: 0%; } 63% { top: 100%; } 100% { top: 26%; } }
-
-@keyframes b6 { 0% { top: 67%; } 33.5% { top: 0%; } 83% { top: 100%; } 100% { top: 67%; } }
-@-webkit-keyframes b6 { 0% { top: 67%; } 33.5% { top: 0%; } 83% { top: 100%; } 100% { top: 67%; } }
-
-@keyframes b7 { 0% { top: 89%; } 44.5% { top: 0%; } 94.5% { top: 100%; } 100% { top: 89%; } }
-@-webkit-keyframes b7 { 0% { top: 89%; } 44.5% { top: 0%; } 94.5% { top: 100%; } 100% { top: 89%; } }
diff --git a/inst/shiny/PLPViewer/www/dataInfo.md b/inst/shiny/PLPViewer/www/dataInfo.md
deleted file mode 100644
index aacec89c0..000000000
--- a/inst/shiny/PLPViewer/www/dataInfo.md
+++ /dev/null
@@ -1,26 +0,0 @@
-# Data Information 
-
-#
-
-The following databases were used in this study:
-
-
-
-| Database       | Name | Country  | Type | Years
-|--------|----------|-----|-----|-----|
-| 
-*OptumDoD* | Optum® De-Identified Clinformatic Data Mart Database – Date of Death (DOD) &nbsp; &nbsp;  | USA &nbsp; &nbsp; | Claims &nbsp; &nbsp; | 2000-2019 |
-| 
-*CCAE* &nbsp; &nbsp; | IBM MarketScan® Commercial Database &nbsp; &nbsp; | USA &nbsp; &nbsp; | Claims | 2000-2019 |
-| 
-*MDCD* &nbsp; &nbsp; | IBM MarketScan® Multi-State Medicaid Database &nbsp; &nbsp;  | USA &nbsp; &nbsp; | Claims | 2006-2019 |
-| 
-*MDCR* &nbsp; &nbsp; | IBM MarketScan® Medicare Supplemental Database &nbsp; &nbsp;  | USA &nbsp; &nbsp; | Claims | 2000-2019 |
-| 
-*JMDC* &nbsp; &nbsp; | Japan Medical Data Center &nbsp; &nbsp; | Japan &nbsp; &nbsp; | Claims &nbsp; &nbsp; | 2000-2019 |
-| 
-*optumEhr* &nbsp; &nbsp; | Optum® de-identified Electronic Health Record Dataset &nbsp; &nbsp;  | USA | EHR &nbsp; &nbsp; | 2006-2019 |
-| 
-
-
-All databases obtained IRB approval or used deidentified data that was considered exempt from IRB approval.
diff --git a/inst/shiny/PLPViewer/www/favicon.ico b/inst/shiny/PLPViewer/www/favicon.ico
deleted file mode 100644
index 849a1fa44..000000000
Binary files a/inst/shiny/PLPViewer/www/favicon.ico and /dev/null differ
diff --git a/inst/shiny/PLPViewer/www/libraryDescription.md b/inst/shiny/PLPViewer/www/libraryDescription.md
deleted file mode 100644
index e9f4976e0..000000000
--- a/inst/shiny/PLPViewer/www/libraryDescription.md
+++ /dev/null
@@ -1,21 +0,0 @@
-### OHDSI PatientLevelPrediction Model Repository
-
-This shiny application is an interface for viewing, uploading and downloading healthcare prediction models. 
-
-Our aim is to improve the reproducibility and transparency of prediction models.  Reproducibiliy is improved by the ability to access code that can be run against any OMOP common data model data to implement and evaluate any model in this repository. Transparency is improved by providing all the details for the model development and performance within the app. This will help us decide which models to trust by examining extensive external validation results. 
-
-### How to use?
-
-Click on the 'Library' option in the left-hand menu to explore the models in the repository.  You can select a model to explore further by selecting the corresponding row in the library table.  This will then populate the settings/performance/model views.
-
-### How to download a model?
-
-Click on the 'download development' to download an R package that can be executed on any OMOP common data model data to replicate the model development process in your data.   Click the 'download validation' to download an R package that can be executed on any OMOP common data model data to validate the choosen model in your data.  You can then upload the new model or validation results to the repo.
-
-### How to upload results?
-
-Click the 'upload' left-hand menu option, fill out the form and click submit.  The results will be reviewed and you will recieve and email once the results are in the repository.
-
-### Questions?
-
-Please send any questions to: ...@...com
diff --git a/inst/shiny/PLPViewer/www/logo.png b/inst/shiny/PLPViewer/www/logo.png
deleted file mode 100644
index c6307af6b..000000000
Binary files a/inst/shiny/PLPViewer/www/logo.png and /dev/null differ
diff --git a/inst/shiny/PLPViewer/www/shinyAbout.md b/inst/shiny/PLPViewer/www/shinyAbout.md
deleted file mode 100644
index 9d7af1821..000000000
--- a/inst/shiny/PLPViewer/www/shinyAbout.md
+++ /dev/null
@@ -1,24 +0,0 @@
-Introduction:
-
-Development Status: 
-
-Below is the abstract of the manuscript that summarizes the findings:
-
-Background:
-
-Methods:
-
-Results:
-
-Discussion:
-
-
-Below are links for study-related artifacts that have been made available as part of this study:
-
-**Protocol:** 
-
-### Packages ###
-
-- OHDSI model development: 
-- OHDSI model validation: 
-- Existing model validation: 
diff --git a/inst/shiny/PLPViewer/www/shinyDescription.md b/inst/shiny/PLPViewer/www/shinyDescription.md
deleted file mode 100644
index 40ea11e43..000000000
--- a/inst/shiny/PLPViewer/www/shinyDescription.md
+++ /dev/null
@@ -1,49 +0,0 @@
-### PLP Study Title 
-
-**Development Status: Under Development**
-
-
-
-### Information
-
-
-
-This shiny application contains the results of [add information about the study]. 
-
-During manuscript development and the subsequent review period, these results are considered under embargo and should not be disclosed without explicit permission and consent from the authors.
-
-
-
-Below are links for study-related artifacts that have been made available as part of this study:
-
-
-
-**Protocol:** [link](https://...)
-
-
-
-### Abstract 
-
-
-Below is the abstract of the manuscript that summarizes the findings:
-
-
-
-**Background:**  
-
-
-**Methods:** 
-
-**Findings:** 
-
-
-**Interpretation:**
-
-
-
-### Study Packages
-
-- Model validation: [link](https://github.com/ohdsi-studies/...)
-
-
-
diff --git a/inst/shinyConfig.json b/inst/shinyConfig.json
new file mode 100644
index 000000000..f2a674932
--- /dev/null
+++ b/inst/shinyConfig.json
@@ -0,0 +1,32 @@
+{
+  "shinyModules": [
+    {
+      "id": "about",
+      "tabName": "About",
+      "tabText": "About",
+      "shinyModulePackage": "OhdsiShinyModules",
+      "uiFunction": "aboutViewer",
+      "serverFunction": "aboutServer",
+      "databaseConnectionKeyService": null,
+      "databaseConnectionKeyUsername": null,
+      "infoBoxFile": "aboutHelperFile()",
+      "icon": "info",
+      "keyring": true,
+      "order": 1
+    },
+    {
+      "id": "prediction",
+      "tabName": "Prediction",
+      "tabText": "Prediction",
+      "shinyModulePackage": "OhdsiShinyModules",
+      "uiFunction": "predictionViewer",
+      "serverFunction": "predictionServer",
+      "databaseConnectionKeyService": "resultDatabaseDetails",
+      "databaseConnectionKeyUsername": "prediction",
+      "infoBoxFile": "predictionHelperFile()",
+      "icon": "chart-line",
+      "keyring": false,
+      "order": 2
+    }
+  ]
+}
diff --git a/inst/shinyConfigUpdate.json b/inst/shinyConfigUpdate.json
new file mode 100644
index 000000000..95756dc68
--- /dev/null
+++ b/inst/shinyConfigUpdate.json
@@ -0,0 +1,26 @@
+{
+  "shinyModules": [
+    {
+      "id": "about",
+      "tabName": "About",
+      "tabText": "About",
+      "shinyModulePackage": "OhdsiShinyModules",
+      "uiFunction": "aboutViewer",
+      "serverFunction": "aboutServer",
+      "infoBoxFile": "aboutHelperFile()",
+      "icon": "info",
+      "order": 1
+    },
+    {
+      "id": "prediction",
+      "tabName": "Prediction",
+      "tabText": "Prediction",
+      "shinyModulePackage": "OhdsiShinyModules",
+      "uiFunction": "patientLevelPredictionViewer",
+      "serverFunction": "patientLevelPredictionServer",
+      "infoBoxFile": "patientLevelPredictionHelperFile()",
+      "icon": "chart-line",
+      "order": 2
+    }
+  ]
+}
diff --git a/inst/sql/postgresql/Migration_1-store_version.sql b/inst/sql/postgresql/Migration_1-store_version.sql
new file mode 100644
index 000000000..bd4ab3a61
--- /dev/null
+++ b/inst/sql/postgresql/Migration_1-store_version.sql
@@ -0,0 +1,16 @@
+-- Database migrations for verion 6.0.10
+-- This migration updates the schema:
+ -- 1. to store the patient level prediction version
+ -- 2. Add a migrations table for supporting database migrations
+
+{DEFAULT @package_version = package_version}
+{DEFAULT @migration = migration}
+{DEFAULT @table_prefix = ''}
+
+-- Create table indicating version number of ddl
+DROP TABLE IF EXISTS @database_schema.@table_prefix@package_version;
+
+--HINT DISTRIBUTE ON RANDOM
+CREATE TABLE @database_schema.@table_prefix@package_version (
+    version_number VARCHAR(50) PRIMARY KEY
+);
\ No newline at end of file
diff --git a/inst/sql/sql_server/CreateCohorts.sql b/inst/sql/sql_server/CreateCohorts.sql
index 141d9da92..20279339f 100644
--- a/inst/sql/sql_server/CreateCohorts.sql
+++ b/inst/sql/sql_server/CreateCohorts.sql
@@ -17,7 +17,7 @@ limitations under the License.
 {DEFAULT @cohort_database_schema = 'CDM_SIM' }
 {DEFAULT @cohort_table = 'drug_era' }
 {DEFAULT @cdm_version = '5'}
-{DEFAULT @cohort_id = '' }
+{DEFAULT @target_id = '' }
 {DEFAULT @study_start_date = '' }
 {DEFAULT @study_end_date = '' }
 {DEFAULT @first_only = FALSE}
@@ -77,15 +77,15 @@ FROM ( -- first_only
 }
 FROM ( -- raw_cohorts 
 	SELECT subject_id,
-			   @cohort_id AS cohort_definition_id,
+			   @target_id AS cohort_definition_id,
 		     cohort_start_date,
 		     cohort_end_date
 	FROM @cohort_database_schema.@cohort_table cohort_table
 	
 {@cdm_version == "4"} ? {	
-	WHERE cohort_concept_id IN (@cohort_id)
+	WHERE cohort_concept_id IN (@target_id)
 } : {
-	WHERE cohort_definition_id IN (@cohort_id)
+	WHERE cohort_definition_id IN (@target_id)
 }
 	) raw_cohorts
 {@first_only} ? {
diff --git a/inst/sql/sql_server/GetCohorts.sql b/inst/sql/sql_server/GetCohorts.sql
index e9189b2b1..54d4423ea 100644
--- a/inst/sql/sql_server/GetCohorts.sql
+++ b/inst/sql/sql_server/GetCohorts.sql
@@ -3,9 +3,9 @@
 SELECT cast(row_id as int) row_id,
 	subject_id,
 {@cdm_version == "4"} ? {	
-	cohort_concept_id AS cohort_id,
+	cohort_concept_id AS target_id,
 } : {
-	cohort_definition_id AS cohort_id,
+	cohort_definition_id AS target_id,
 }
 	cohort_start_date,
 	days_from_obs_start,
diff --git a/inst/sql/sql_server/PlpResultTables.sql b/inst/sql/sql_server/PlpResultTables.sql
index 11597b82d..feb5cf2ed 100644
--- a/inst/sql/sql_server/PlpResultTables.sql
+++ b/inst/sql/sql_server/PlpResultTables.sql
@@ -1,31 +1,39 @@
--- do we need this
-CREATE TABLE @my_schema.@string_to_appendstudies (
-    study_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
-	  study_name varchar(100),
-    study_description varchar(1000)
+-- this links the PLP cohort_definition_id to the COHORT_DEFINITION
+CREATE TABLE @my_schema.@string_to_appendcohorts ( 
+    cohort_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY, -- 
+    cohort_definition_id int NOT NULL, -- the atlas id check type
+    cohort_name VARCHAR(MAX) NOT NULL
 );
 
-CREATE TABLE @my_schema.@string_to_appendcohorts (
-    cohort_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
-	atlas_id bigint,
-    cohort_name char(100) NOT NULL,
-    cohort_json VARCHAR(MAX) NOT NULL
+-- NEW - needs to match cohort generator COHORT_DEFINITION
+CREATE TABLE @my_schema.@string_to_appendCOHORT_DEFINITION (
+    cohort_definition_id int, -- check type
+    cohort_name VARCHAR(MAX) NOT NULL,
+    description VARCHAR(MAX),
+    json VARCHAR(MAX),
+    sql_command VARCHAR(MAX)
 );
 
-CREATE TABLE @my_schema.@string_to_appendresearchers (
-    researcher_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
-    researcher_name char(100) NOT NULL,
-    researcher_email char(100) NOT NULL,
-    researcher_affiliation char(250) NOT NULL
+-- link the database_id in the results with the database_meta_data_id
+CREATE TABLE @my_schema.@string_to_appenddatabase_details ( -- DATABASE_META_DATA
+    database_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
+    database_meta_data_id varchar(MAX) -- databaseId strategus 
 );
 
-CREATE TABLE @my_schema.@string_to_appenddatabase_details (
-    database_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
-    database_name char(100) NOT NULL,
-    database_acronym char(20) NOT NULL,
-	  database_version int NOT NULL,
-    database_description char(1000) NOT NULL,
-    database_type char(20) NOT NULL
+-- NEW - needs to match stragegus DATABASE_META_DATA
+CREATE TABLE @my_schema.@string_to_appendDATABASE_META_DATA (
+    database_id varchar(MAX) PRIMARY KEY,
+    cdm_source_name varchar(MAX) NOT NULL,
+    cdm_source_abbreviation varchar(MAX) NOT NULL,
+    CDM_HOLDER varchar(MAX), 
+    SOURCE_DESCRIPTION varchar(MAX),
+    SOURCE_DOCUMENTATION_REFERENCE varchar(MAX),
+    CDM_ETL_REFERENCE varchar(MAX), 
+    SOURCE_RELEASE_DATE varchar(MAX), -- not date due to sqlite and consistency
+    CDM_RELEASE_DATE varchar(MAX), -- not date due to sqlite and consistency
+    CDM_VERSION varchar(MAX),
+    VOCABULARY_VERSION varchar(MAX),
+    MAX_OBS_PERIOD_END_DATE varchar(MAX) -- not date due to sqlite and consistency
 );
 
 CREATE TABLE @my_schema.@string_to_appendtars (
@@ -91,11 +99,9 @@ CREATE TABLE  @my_schema.@string_to_appendmodel_designs (
 	  split_setting_id int NOT NULL, -- new
 	  feature_engineering_setting_id int NOT NULL, -- new
 	  tidy_covariates_setting_id int NOT NULL, -- new
-    researcher_id int NOT NULL,
     FOREIGN KEY (target_id) REFERENCES @my_schema.@string_to_appendcohorts(cohort_id),
     FOREIGN KEY (outcome_id) REFERENCES @my_schema.@string_to_appendcohorts(cohort_id),
-    FOREIGN KEY (researcher_id) REFERENCES @my_schema.@string_to_appendresearchers(researcher_id),
-	  FOREIGN KEY (tar_id) REFERENCES @my_schema.@string_to_appendtars(tar_id),
+    FOREIGN KEY (tar_id) REFERENCES @my_schema.@string_to_appendtars(tar_id),
 	  FOREIGN KEY (population_setting_id) REFERENCES @my_schema.@string_to_appendpopulation_settings(population_setting_id),
     FOREIGN KEY (model_setting_id) REFERENCES @my_schema.@string_to_appendmodel_settings(model_setting_id),
 	  FOREIGN KEY (covariate_setting_id) REFERENCES @my_schema.@string_to_appendcovariate_settings(covariate_setting_id),
@@ -106,45 +112,72 @@ CREATE TABLE  @my_schema.@string_to_appendmodel_designs (
 	  FOREIGN KEY (tidy_covariates_setting_id) REFERENCES @my_schema.@string_to_appendtidy_covariates_settings(tidy_covariates_setting_id) -- new
 );
 
--- diagnostics holder (will add more tables)
+-- diagnostics holder
 CREATE TABLE  @my_schema.@string_to_appenddiagnostics(
    diagnostic_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
    model_design_id int,
-   researcher_id int,
    database_id int NOT NULL,
-   execution_date_time DATETIME2,
-   FOREIGN KEY (researcher_id) REFERENCES @my_schema.@string_to_appendresearchers(researcher_id),
+   execution_date_time VARCHAR(100),
    FOREIGN KEY (model_design_id) REFERENCES @my_schema.@string_to_appendmodel_designs(model_design_id),
    FOREIGN KEY (database_id) REFERENCES @my_schema.@string_to_appenddatabase_details(database_id)
 );
-
+CREATE TABLE  @my_schema.@string_to_appenddiagnostic_summary(
+   diagnostic_id int NOT NULL,
+   probast_id varchar(50),
+   result_value varchar(50),
+   FOREIGN KEY (diagnostic_id) REFERENCES @my_schema.@string_to_appenddiagnostics(diagnostic_id)
+);
+CREATE TABLE  @my_schema.@string_to_appenddiagnostic_predictors( -- call this kmplot
+   diagnostic_id int NOT NULL,
+   days_to_event int,
+   outcome_at_time int,
+   observed_at_start_of_day bigint,
+   probast_id varchar(50),
+   input_type varchar(50),
+   FOREIGN KEY (diagnostic_id) REFERENCES @my_schema.@string_to_appenddiagnostics(diagnostic_id)
+);
+CREATE TABLE  @my_schema.@string_to_appenddiagnostic_participants(
+   diagnostic_id int NOT NULL,
+   design varchar(50),
+   metric varchar(50),
+   value float,
+   probast_id varchar(50),
+   FOREIGN KEY (diagnostic_id) REFERENCES @my_schema.@string_to_appenddiagnostics(diagnostic_id)
+);
+CREATE TABLE  @my_schema.@string_to_appenddiagnostic_outcomes(
+   diagnostic_id int NOT NULL,
+   xvalue int,
+   outcome_percent float,
+   aggregation varchar(50),
+   probast_id varchar(50),
+   input_type varchar(50),
+   FOREIGN KEY (diagnostic_id) REFERENCES @my_schema.@string_to_appenddiagnostics(diagnostic_id)
+);
+CREATE TABLE  @my_schema.@string_to_appenddiagnostic_designs(
+   diagnostic_id int NOT NULL,
+   probast_id varchar(50),
+   value varchar(50),
+   FOREIGN KEY (diagnostic_id) REFERENCES @my_schema.@string_to_appenddiagnostics(diagnostic_id)
+);
+-- end diagnostics
 
 -- results
 CREATE TABLE  @my_schema.@string_to_appendmodels(
     model_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
-    researcher_id int,
     analysis_id varchar(50),
     model_design_id int,
     database_id int NOT NULL,
-	  hyper_param_search VARCHAR(MAX), -- new this contains the hyperparameter performances
-    plp_model_file char(50) NOT NULL,
-	  execution_date_time DATETIME2,
+    model_type VARCHAR(50),
+    plp_model_file text NOT NULL, -- reference to saved model location
+	  train_details VARCHAR(MAX), -- new this contains all the trainDetails
+	  preprocessing VARCHAR(MAX), -- new this contains the preprocessing required
+	  execution_date_time VARCHAR(100),
 	  training_time VARCHAR(100), -- previously new
 	  intercept float,
-	  require_dense_matrix char(1), -- new
-	  FOREIGN KEY (researcher_id) REFERENCES @my_schema.@string_to_appendresearchers(researcher_id),
 	  FOREIGN KEY (model_design_id) REFERENCES @my_schema.@string_to_appendmodel_designs(model_design_id),
     FOREIGN KEY (database_id) REFERENCES @my_schema.@string_to_appenddatabase_details(database_id)
 );
 
--- do we need this?
-CREATE TABLE @my_schema.@string_to_appendstudy_models (
-    study_id int,
-	  model_id int,
-	  FOREIGN KEY (study_id) REFERENCES @my_schema.@string_to_appendstudies(study_id),
-    FOREIGN KEY (model_id) REFERENCES @my_schema.@string_to_appendmodels(model_id)
-);
-
 -- make this relcaibration specific?
 CREATE TABLE  @my_schema.@string_to_appendrecalibrations (
     recalibration_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
@@ -156,42 +189,44 @@ CREATE TABLE  @my_schema.@string_to_appendrecalibrations (
 	  FOREIGN KEY (recalibrated_model_id) REFERENCES @my_schema.@string_to_appendmodels(model_id)
 );
 
-CREATE TABLE  @my_schema.@string_to_appendresults (
-    result_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
-    model_id int NOT NULL,
-    researcher_id int NOT NULL,
-    database_id int NOT NULL,
+CREATE TABLE  @my_schema.@string_to_appendperformances (
+    performance_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
+    model_design_id int NOT NULL,
+    development_database_id int NOT NULL,
+    validation_database_id int NOT NULL,
     target_id int NOT NULL,
     outcome_id int NOT NULL,
     tar_id int NOT NULL,
     plp_data_setting_id int NOT NULL, -- added
 	  population_setting_id int NOT NULL,
-    execution_date_time DATETIME2,
+	  model_development int NOT NULL, -- added
+    execution_date_time VARCHAR(100),
     plp_version char(10),
-    FOREIGN KEY (model_id) REFERENCES @my_schema.@string_to_appendmodels(model_id),
-    FOREIGN KEY (researcher_id) REFERENCES @my_schema.@string_to_appendresearchers(researcher_id),
-    FOREIGN KEY (database_id) REFERENCES @my_schema.@string_to_appenddatabase_details(database_id),
+    FOREIGN KEY (model_design_id) REFERENCES @my_schema.@string_to_appendmodel_designs(model_design_id),
+    FOREIGN KEY (development_database_id) REFERENCES @my_schema.@string_to_appenddatabase_details(database_id),
+    FOREIGN KEY (validation_database_id) REFERENCES @my_schema.@string_to_appenddatabase_details(database_id),
     FOREIGN KEY (target_id) REFERENCES @my_schema.@string_to_appendcohorts(cohort_id),
     FOREIGN KEY (outcome_id) REFERENCES @my_schema.@string_to_appendcohorts(cohort_id),
     FOREIGN KEY (tar_id) REFERENCES @my_schema.@string_to_appendtars(tar_id),
     FOREIGN KEY (plp_data_setting_id) REFERENCES @my_schema.@string_to_appendplp_data_settings(plp_data_setting_id), -- new
 	  FOREIGN KEY (population_setting_id) REFERENCES @my_schema.@string_to_appendpopulation_settings(population_setting_id)
+	  
 );
 
 -- new
 CREATE TABLE @my_schema.@string_to_appendattrition (
-	  result_id int NOT NULL,
+	  performance_id int NOT NULL,
 	  outcome_id int,
 	  description varchar(1000),
-	  target_count int,
+	  target_count int, -- is this still target?
 	  unique_people int,
 	  outcomes int,
-    FOREIGN KEY (result_id) REFERENCES @my_schema.@string_to_appendresults(result_id)
+    FOREIGN KEY (performance_id) REFERENCES @my_schema.@string_to_appendperformances(performance_id)
 );
 
 CREATE TABLE @my_schema.@string_to_appendprediction_distribution (
     --distribution_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
-	  result_id int NOT NULL,
+	  performance_id int NOT NULL,
     evaluation char(10),
     class_label int,
     person_count int,
@@ -204,12 +239,12 @@ CREATE TABLE @my_schema.@string_to_appendprediction_distribution (
     p_75_predicted_probability float,
     p_95_predicted_probability float,
     max_predicted_probability float,
-    FOREIGN KEY (result_id) REFERENCES @my_schema.@string_to_appendresults(result_id)
+    FOREIGN KEY (performance_id) REFERENCES @my_schema.@string_to_appendperformances(performance_id)
 );
 
 CREATE TABLE @my_schema.@string_to_appendcovariate_summary(
     --cov_sum_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
-	result_id int NOT NULL,
+	performance_id int NOT NULL,
     covariate_id bigint NOT NULL,
 	covariate_name VARCHAR(1000) NOT NULL,
     concept_id int NOT NULL,
@@ -224,11 +259,11 @@ CREATE TABLE @my_schema.@string_to_appendcovariate_summary(
     with_outcome_covariate_mean float NOT NULL,
 	with_outcome_covariate_st_dev float NOT NULL,
     standardized_mean_diff float NOT NULL,
-    FOREIGN KEY (result_id) REFERENCES @my_schema.@string_to_appendresults(result_id)
+    FOREIGN KEY (performance_id) REFERENCES @my_schema.@string_to_appendperformances(performance_id)
 );
 CREATE TABLE @my_schema.@string_to_appendthreshold_summary(
     --threshold_summary_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
-    result_id int NOT NULL,
+    performance_id int NOT NULL,
     evaluation char(10),
     prediction_threshold float,
     preference_threshold float,
@@ -253,12 +288,12 @@ CREATE TABLE @my_schema.@string_to_appendthreshold_summary(
     positive_likelihood_ratio float,
     negative_likelihood_ratio float,
     diagnostic_odds_ratio float,
-    FOREIGN KEY (result_id) REFERENCES @my_schema.@string_to_appendresults(result_id)
+    FOREIGN KEY (performance_id) REFERENCES @my_schema.@string_to_appendperformances(performance_id)
 );
 
 CREATE TABLE @my_schema.@string_to_appendcalibration_summary(
     --calibration_summary_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
-    result_id int NOT NULL,
+    performance_id int NOT NULL,
     evaluation char(10),
     prediction_threshold float,
     person_count_at_risk int,
@@ -271,21 +306,21 @@ CREATE TABLE @my_schema.@string_to_appendcalibration_summary(
     p_75_predicted_probability float,
     max_predicted_probability float,
     observed_incidence float,
-    FOREIGN KEY (result_id) REFERENCES @my_schema.@string_to_appendresults(result_id)
+    FOREIGN KEY (performance_id) REFERENCES @my_schema.@string_to_appendperformances(performance_id)
 );
 
 CREATE TABLE @my_schema.@string_to_appendevaluation_statistics (
     --evaluation_stat_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
-	result_id int NOT NULL,
+	performance_id int NOT NULL,
 	evaluation char(10),
   metric varchar(50),
 	value float,
-    FOREIGN KEY (result_id) REFERENCES @my_schema.@string_to_appendresults(result_id)
+    FOREIGN KEY (performance_id) REFERENCES @my_schema.@string_to_appendperformances(performance_id)
 );
 
 CREATE TABLE @my_schema.@string_to_appenddemographic_summary(
     --demographic_summary_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
-    result_id int NOT NULL,
+    performance_id int NOT NULL,
     evaluation char(10),
     age_group char(20),
     gen_group char(20),
@@ -298,5 +333,6 @@ CREATE TABLE @my_schema.@string_to_appenddemographic_summary(
     p_50_predicted_probability float,
     p_75_predicted_probability float,
     max_predicted_probability float,
-    FOREIGN KEY (result_id) REFERENCES @my_schema.@string_to_appendresults(result_id)
+    FOREIGN KEY (performance_id) REFERENCES @my_schema.@string_to_appendperformances(performance_id)
 );
+
diff --git a/inst/sql/sql_server/UpdateVersionNumber.sql b/inst/sql/sql_server/UpdateVersionNumber.sql
new file mode 100644
index 000000000..216233f88
--- /dev/null
+++ b/inst/sql/sql_server/UpdateVersionNumber.sql
@@ -0,0 +1,5 @@
+{DEFAULT @package_version = package_version}
+{DEFAULT @version_number = '6.0.10'}
+
+DELETE FROM @database_schema.@table_prefix@package_version;
+INSERT INTO @database_schema.@table_prefix@package_version (version_number) VALUES ('@version_number');
diff --git a/inst/sql/sql_server/migrations/Migration_1-store_version.sql b/inst/sql/sql_server/migrations/Migration_1-store_version.sql
new file mode 100644
index 000000000..bd4ab3a61
--- /dev/null
+++ b/inst/sql/sql_server/migrations/Migration_1-store_version.sql
@@ -0,0 +1,16 @@
+-- Database migrations for verion 6.0.10
+-- This migration updates the schema:
+ -- 1. to store the patient level prediction version
+ -- 2. Add a migrations table for supporting database migrations
+
+{DEFAULT @package_version = package_version}
+{DEFAULT @migration = migration}
+{DEFAULT @table_prefix = ''}
+
+-- Create table indicating version number of ddl
+DROP TABLE IF EXISTS @database_schema.@table_prefix@package_version;
+
+--HINT DISTRIBUTE ON RANDOM
+CREATE TABLE @database_schema.@table_prefix@package_version (
+    version_number VARCHAR(50) PRIMARY KEY
+);
\ No newline at end of file
diff --git a/inst/sql/sqlite/PlpResultTables.sql b/inst/sql/sqlite/PlpResultTables.sql
new file mode 100644
index 000000000..93bd63ac5
--- /dev/null
+++ b/inst/sql/sqlite/PlpResultTables.sql
@@ -0,0 +1,338 @@
+-- this links the PLP cohort_definition_id to the COHORT_DEFINITION
+CREATE TABLE @my_schema.@string_to_appendcohorts ( 
+    cohort_id INTEGER PRIMARY KEY AUTOINCREMENT, -- 
+    cohort_definition_id int NOT NULL, -- the atlas id check type
+    cohort_name VARCHAR(MAX) NOT NULL
+);
+
+-- NEW - needs to match cohort generator COHORT_DEFINITION
+CREATE TABLE @my_schema.@string_to_appendCOHORT_DEFINITION (
+    cohort_definition_id int NOT NULL, -- check type
+    cohort_name VARCHAR(MAX) NOT NULL,
+    description VARCHAR(MAX),
+    json VARCHAR(MAX),
+    sql_command VARCHAR(MAX)
+);
+
+-- link the database_id in the results with the database_meta_data_id
+CREATE TABLE @my_schema.@string_to_appenddatabase_details ( -- DATABASE_META_DATA
+    database_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    database_meta_data_id varchar(MAX) -- databaseId strategus 
+);
+
+-- NEW - needs to match stragegus DATABASE_META_DATA
+CREATE TABLE @my_schema.@string_to_appendDATABASE_META_DATA (
+    database_id varchar(MAX) PRIMARY KEY,
+    cdm_source_name varchar(MAX) NOT NULL,
+    cdm_source_abbreviation varchar(MAX) NOT NULL,
+    CDM_HOLDER varchar(MAX), 
+    SOURCE_DESCRIPTION varchar(MAX),
+    SOURCE_DOCUMENTATION_REFERENCE varchar(MAX),
+    CDM_ETL_REFERENCE varchar(MAX), 
+    SOURCE_RELEASE_DATE varchar(MAX), -- not date due to sqlite and consistency
+    CDM_RELEASE_DATE varchar(MAX), -- not date due to sqlite and consistency
+    CDM_VERSION varchar(MAX),
+    VOCABULARY_VERSION varchar(MAX),
+    MAX_OBS_PERIOD_END_DATE varchar(MAX) -- not date due to sqlite and consistency
+);
+
+
+CREATE TABLE @my_schema.@string_to_appendtars (
+    tar_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    tar_start_day int NOT NULL,
+	  tar_start_anchor VARCHAR(20) NOT NULL,
+    tar_end_day int NOT NULL,
+	  tar_end_anchor VARCHAR(20) NOT NULL
+);
+
+CREATE TABLE @my_schema.@string_to_appendpopulation_settings(
+    population_setting_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    population_settings_json VARCHAR(MAX) NOT NULL
+);
+
+CREATE TABLE @my_schema.@string_to_appendcovariate_settings(
+    covariate_setting_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    covariate_settings_json VARCHAR(MAX) NOT NULL
+);
+
+CREATE TABLE @my_schema.@string_to_appendmodel_settings(
+    model_setting_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    model_type VARCHAR(50),
+    model_settings_json VARCHAR(MAX)
+);
+
+CREATE TABLE @my_schema.@string_to_appendsplit_settings( -- was training_settings
+    split_setting_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    split_settings_json VARCHAR(MAX)
+);
+
+CREATE TABLE @my_schema.@string_to_appendplp_data_settings( -- new
+    plp_data_setting_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    plp_data_settings_json VARCHAR(MAX)
+);
+
+CREATE TABLE @my_schema.@string_to_appendfeature_engineering_settings( -- new
+    feature_engineering_setting_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    feature_engineering_settings_json VARCHAR(MAX)
+);
+
+CREATE TABLE @my_schema.@string_to_appendtidy_covariates_settings( -- new
+    tidy_covariates_setting_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    tidy_covariates_settings_json VARCHAR(MAX)
+);
+
+CREATE TABLE @my_schema.@string_to_appendsample_settings( -- new
+    sample_setting_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    sample_settings_json VARCHAR(MAX)
+);
+
+CREATE TABLE  @my_schema.@string_to_appendmodel_designs (
+    model_design_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    --model_name CHAR(50) NOT NULL,
+    target_id int NOT NULL,
+    outcome_id int NOT NULL,
+    tar_id int NOT NULL,
+    plp_data_setting_id int NOT NULL, -- new
+	  population_setting_id int NOT NULL,
+	  model_setting_id int NOT NULL,
+	  covariate_setting_id int NOT NULL,
+	  sample_setting_id int NOT NULL, -- new
+	  split_setting_id int NOT NULL, -- new
+	  feature_engineering_setting_id int NOT NULL, -- new
+	  tidy_covariates_setting_id int NOT NULL, -- new
+    FOREIGN KEY (target_id) REFERENCES @string_to_appendcohorts(cohort_id),
+    FOREIGN KEY (outcome_id) REFERENCES @string_to_appendcohorts(cohort_id),
+    FOREIGN KEY (tar_id) REFERENCES @string_to_appendtars(tar_id),
+	  FOREIGN KEY (population_setting_id) REFERENCES @string_to_appendpopulation_settings(population_setting_id),
+    FOREIGN KEY (model_setting_id) REFERENCES @string_to_appendmodel_settings(model_setting_id),
+	  FOREIGN KEY (covariate_setting_id) REFERENCES @string_to_appendcovariate_settings(covariate_setting_id),
+	  FOREIGN KEY (sample_setting_id) REFERENCES @string_to_appendsample_settings(sample_setting_id),  -- new
+	  FOREIGN KEY (split_setting_id) REFERENCES @string_to_appendsplit_settings(split_setting_id),  -- new
+	  FOREIGN KEY (plp_data_setting_id) REFERENCES @string_to_appendplp_data_settings(plp_data_setting_id), -- new
+	  FOREIGN KEY (feature_engineering_setting_id) REFERENCES @string_to_appendfeature_engineering_settings(feature_engineering_setting_id), -- new
+	  FOREIGN KEY (tidy_covariates_setting_id) REFERENCES @string_to_appendtidy_covariates_settings(tidy_covariates_setting_id) -- new
+);
+
+-- diagnostics holder
+CREATE TABLE  @my_schema.@string_to_appenddiagnostics(
+   diagnostic_id INTEGER PRIMARY KEY AUTOINCREMENT,
+   model_design_id int,
+   database_id int NOT NULL,
+   execution_date_time VARCHAR(100),
+   FOREIGN KEY (model_design_id) REFERENCES @string_to_appendmodel_designs(model_design_id),
+   FOREIGN KEY (database_id) REFERENCES @string_to_appenddatabase_details(database_id)
+);
+CREATE TABLE  @my_schema.@string_to_appenddiagnostic_summary(
+   diagnostic_id int NOT NULL,
+   probast_id varchar(50),
+   result_value varchar(50),
+   FOREIGN KEY (diagnostic_id) REFERENCES @string_to_appenddiagnostics(diagnostic_id)
+);
+CREATE TABLE  @my_schema.@string_to_appenddiagnostic_predictors( -- call this kmplot
+   diagnostic_id int NOT NULL,
+   days_to_event int,
+   outcome_at_time int,
+   observed_at_start_of_day bigint,
+   probast_id varchar(50),
+   input_type varchar(50),
+   FOREIGN KEY (diagnostic_id) REFERENCES @string_to_appenddiagnostics(diagnostic_id)
+);
+CREATE TABLE  @my_schema.@string_to_appenddiagnostic_participants(
+   diagnostic_id int NOT NULL,
+   design varchar(50),
+   metric varchar(50),
+   value float,
+   probast_id varchar(50),
+   FOREIGN KEY (diagnostic_id) REFERENCES @string_to_appenddiagnostics(diagnostic_id)
+);
+CREATE TABLE  @my_schema.@string_to_appenddiagnostic_outcomes(
+   diagnostic_id int NOT NULL,
+   xvalue int,
+   outcome_percent float,
+   aggregation varchar(50),
+   probast_id varchar(50),
+   input_type varchar(50),
+   FOREIGN KEY (diagnostic_id) REFERENCES @string_to_appenddiagnostics(diagnostic_id)
+);
+CREATE TABLE  @my_schema.@string_to_appenddiagnostic_designs(
+   diagnostic_id int NOT NULL,
+   probast_id varchar(50),
+   value varchar(50),
+   FOREIGN KEY (diagnostic_id) REFERENCES @string_to_appenddiagnostics(diagnostic_id)
+);
+-- end diagnostics
+
+-- results
+CREATE TABLE  @my_schema.@string_to_appendmodels(
+    model_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    analysis_id varchar(50),
+    model_design_id int,
+    database_id int NOT NULL,
+    model_type VARCHAR(50),
+    plp_model_file VARCHAR(MAX) NOT NULL, -- reference to saved model location
+	  train_details VARCHAR(MAX), -- new this contains all the trainDetails
+	  preprocessing VARCHAR(MAX), -- new this contains the preprocessing required
+	  execution_date_time VARCHAR(100),
+	  training_time VARCHAR(100), -- previously new
+	  intercept float,
+	  FOREIGN KEY (model_design_id) REFERENCES @string_to_appendmodel_designs(model_design_id),
+    FOREIGN KEY (database_id) REFERENCES @string_to_appenddatabase_details(database_id)
+);
+
+-- make this relcaibration specific?
+CREATE TABLE  @my_schema.@string_to_appendrecalibrations (
+    recalibration_id INTEGER PRIMARY KEY AUTOINCREMENT,
+	  original_model_id int NOT NULL,
+	  recalibrated_model_id int NOT NULL,
+	  recalibration_type varchar(15),
+	  recalibration_json varchar(MAX),
+	  FOREIGN KEY (original_model_id) REFERENCES @string_to_appendmodels(model_id),
+	  FOREIGN KEY (recalibrated_model_id) REFERENCES @string_to_appendmodels(model_id)
+);
+
+CREATE TABLE  @my_schema.@string_to_appendperformances (
+    performance_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    model_design_id int NOT NULL,
+    development_database_id int NOT NULL,
+    validation_database_id int NOT NULL,
+    target_id int NOT NULL,
+    outcome_id int NOT NULL,
+    tar_id int NOT NULL,
+    plp_data_setting_id int NOT NULL, -- added
+	  population_setting_id int NOT NULL,
+	  model_development int NOT NULL, -- added
+    execution_date_time VARCHAR(100),
+    plp_version char(10),
+    FOREIGN KEY (model_design_id) REFERENCES @string_to_appendmodels_designs(model_design_id),
+    FOREIGN KEY (development_database_id) REFERENCES @string_to_appenddatabase_details(database_id),
+    FOREIGN KEY (validation_database_id) REFERENCES @string_to_appenddatabase_details(database_id),
+    FOREIGN KEY (target_id) REFERENCES @string_to_appendcohorts(cohort_id),
+    FOREIGN KEY (outcome_id) REFERENCES @string_to_appendcohorts(cohort_id),
+    FOREIGN KEY (tar_id) REFERENCES @string_to_appendtars(tar_id),
+    FOREIGN KEY (plp_data_setting_id) REFERENCES @string_to_appendplp_data_settings(plp_data_setting_id), -- new
+	  FOREIGN KEY (population_setting_id) REFERENCES @string_to_appendpopulation_settings(population_setting_id)
+);
+
+-- new
+CREATE TABLE @my_schema.@string_to_appendattrition (
+	  performance_id int NOT NULL,
+	  outcome_id int,
+	  description varchar(1000),
+	  target_count int, -- is this still target?
+	  unique_people int,
+	  outcomes int,
+    FOREIGN KEY (performance_id) REFERENCES @string_to_appendperformances(performance_id)
+);
+
+CREATE TABLE @my_schema.@string_to_appendprediction_distribution (
+    --distribution_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
+	  performance_id int NOT NULL,
+    evaluation VARCHAR(10),
+    class_label int,
+    person_count int,
+    average_predicted_probability float,
+    st_dev_predicted_probability float,
+    min_predicted_probability float,
+    p_05_predicted_probability float,
+    p_25_predicted_probability float,
+    median_predicted_probability float,
+    p_75_predicted_probability float,
+    p_95_predicted_probability float,
+    max_predicted_probability float,
+    FOREIGN KEY (performance_id) REFERENCES @string_to_appendperformances(performance_id)
+);
+
+CREATE TABLE @my_schema.@string_to_appendcovariate_summary(
+    --cov_sum_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
+	performance_id int NOT NULL,
+    covariate_id bigint NOT NULL,
+	covariate_name VARCHAR(1000) NOT NULL,
+    concept_id int NOT NULL,
+    covariate_value float NOT NULL,
+    covariate_count int NOT NULL,
+	covariate_mean float NOT NULL,
+	covariate_st_dev float NOT NULL,
+	with_no_outcome_covariate_count int NOT NULL,
+    with_no_outcome_covariate_mean float NOT NULL,
+	with_no_outcome_covariate_st_dev float NOT NULL,
+    with_outcome_covariate_count int NOT NULL,
+    with_outcome_covariate_mean float NOT NULL,
+	with_outcome_covariate_st_dev float NOT NULL,
+    standardized_mean_diff float NOT NULL,
+    FOREIGN KEY (performance_id) REFERENCES @string_to_appendperformances(performance_id)
+);
+CREATE TABLE @my_schema.@string_to_appendthreshold_summary(
+    --threshold_summary_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
+    performance_id int NOT NULL,
+    evaluation VARCHAR(10),
+    prediction_threshold float,
+    preference_threshold float,
+    positive_count int,
+    negative_count int,
+    true_count int,
+    false_count int,
+    true_positive_count int,
+    true_negative_count int,
+    false_positive_count int,
+    false_negative_count int,
+    f_1_score float,
+    accuracy float,
+    sensitivity float,
+    false_negative_rate float,
+    false_positive_rate float,
+    specificity float,
+    positive_predictive_value float,
+    false_discovery_rate float,
+    negative_predictive_value float,
+    false_omission_rate float,
+    positive_likelihood_ratio float,
+    negative_likelihood_ratio float,
+    diagnostic_odds_ratio float,
+    FOREIGN KEY (performance_id) REFERENCES @string_to_appendperformances(performance_id)
+);
+
+CREATE TABLE @my_schema.@string_to_appendcalibration_summary(
+    --calibration_summary_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
+    performance_id int NOT NULL,
+    evaluation VARCHAR(10),
+    prediction_threshold float,
+    person_count_at_risk int,
+    person_count_with_outcome int,
+    average_predicted_probability float,
+    st_dev_predicted_probability float,
+    min_predicted_probability float,
+    p_25_predicted_probability float,
+    median_predicted_probability float,
+    p_75_predicted_probability float,
+    max_predicted_probability float,
+    observed_incidence float,
+    FOREIGN KEY (performance_id) REFERENCES @string_to_appendperformances(performance_id)
+);
+
+CREATE TABLE @my_schema.@string_to_appendevaluation_statistics (
+    --evaluation_stat_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
+	performance_id int NOT NULL,
+	evaluation VARCHAR(10),
+  metric varchar(50),
+	value float,
+    FOREIGN KEY (performance_id) REFERENCES @string_to_appendperformances(performance_id)
+);
+
+CREATE TABLE @my_schema.@string_to_appenddemographic_summary(
+    --demographic_summary_id int GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY,
+    performance_id int NOT NULL,
+    evaluation VARCHAR(10),
+    age_group VARCHAR(20),
+    gen_group VARCHAR(20),
+    person_count_at_risk int,
+    person_count_with_outcome int,
+    average_predicted_probability float,
+    st_dev_predicted_probability float,
+    min_predicted_probability float,
+    p_25_predicted_probability float,
+    p_50_predicted_probability float,
+    p_75_predicted_probability float,
+    max_predicted_probability float,
+    FOREIGN KEY (performance_id) REFERENCES @string_to_appendperformances(performance_id)
+);
+
diff --git a/inst/sql/sqlite/migrations/Migration_1-store_version.sql b/inst/sql/sqlite/migrations/Migration_1-store_version.sql
new file mode 100644
index 000000000..bd4ab3a61
--- /dev/null
+++ b/inst/sql/sqlite/migrations/Migration_1-store_version.sql
@@ -0,0 +1,16 @@
+-- Database migrations for verion 6.0.10
+-- This migration updates the schema:
+ -- 1. to store the patient level prediction version
+ -- 2. Add a migrations table for supporting database migrations
+
+{DEFAULT @package_version = package_version}
+{DEFAULT @migration = migration}
+{DEFAULT @table_prefix = ''}
+
+-- Create table indicating version number of ddl
+DROP TABLE IF EXISTS @database_schema.@table_prefix@package_version;
+
+--HINT DISTRIBUTE ON RANDOM
+CREATE TABLE @database_schema.@table_prefix@package_version (
+    version_number VARCHAR(50) PRIMARY KEY
+);
\ No newline at end of file
diff --git a/man/MapIds.Rd b/man/MapIds.Rd
new file mode 100644
index 000000000..e10d16ba3
--- /dev/null
+++ b/man/MapIds.Rd
@@ -0,0 +1,19 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/Formatting.R
+\name{MapIds}
+\alias{MapIds}
+\title{Map covariate and row Ids so they start from 1}
+\usage{
+MapIds(covariateData, cohort = NULL, mapping = NULL)
+}
+\arguments{
+\item{covariateData}{a covariateData object}
+
+\item{cohort}{if specified rowIds restricted to the ones in cohort}
+
+\item{mapping}{A pre defined mapping to use}
+}
+\description{
+this functions takes covariate data and a cohort/population and remaps 
+the covariate and row ids, restricts to pop and saves/creates mapping
+}
diff --git a/man/PatientLevelPrediction.Rd b/man/PatientLevelPrediction.Rd
index 560e9be95..8bc15fc71 100644
--- a/man/PatientLevelPrediction.Rd
+++ b/man/PatientLevelPrediction.Rd
@@ -2,8 +2,32 @@
 % Please edit documentation in R/PatientLevelPrediction.R
 \docType{package}
 \name{PatientLevelPrediction}
+\alias{PatientLevelPrediction-package}
 \alias{PatientLevelPrediction}
 \title{PatientLevelPrediction}
 \description{
 A package for running predictions using data in the OMOP CDM
 }
+\seealso{
+Useful links:
+\itemize{
+  \item \url{https://ohdsi.github.io/PatientLevelPrediction}
+  \item \url{https://github.com/OHDSI/PatientLevelPrediction}
+  \item Report bugs at \url{https://github.com/OHDSI/PatientLevelPrediction/issues}
+}
+
+}
+\author{
+\strong{Maintainer}: Jenna Reps \email{jreps@its.jnj.com}
+
+Authors:
+\itemize{
+  \item Martijn Schuemie
+  \item Marc Suchard
+  \item Patrick Ryan
+  \item Peter Rijnbeek
+  \item Egill Fridgeirsson
+}
+
+}
+\keyword{internal}
diff --git a/man/addDiagnosePlpToDatabase.Rd b/man/addDiagnosePlpToDatabase.Rd
new file mode 100644
index 000000000..483575654
--- /dev/null
+++ b/man/addDiagnosePlpToDatabase.Rd
@@ -0,0 +1,39 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/uploadToDatabaseDiagnostics.R
+\name{addDiagnosePlpToDatabase}
+\alias{addDiagnosePlpToDatabase}
+\title{Insert a diagnostic result into a PLP result schema database}
+\usage{
+addDiagnosePlpToDatabase(
+  diagnosePlp,
+  connectionDetails,
+  databaseSchemaSettings,
+  cohortDefinitions,
+  databaseList = NULL,
+  overWriteIfExists = T
+)
+}
+\arguments{
+\item{diagnosePlp}{An object of class \code{diagnosePlp}}
+
+\item{connectionDetails}{A connection details created by using the
+function \code{createConnectionDetails} in the
+\code{DatabaseConnector} package.}
+
+\item{databaseSchemaSettings}{A object created by \code{createDatabaseSchemaSettings} with all the settings specifying the result tables}
+
+\item{cohortDefinitions}{A set of one or more cohorts extracted using ROhdsiWebApi::exportCohortDefinitionSet()}
+
+\item{databaseList}{(Optional) If you wish to overwrite the settings in the plp object use \code{createdatabaseList} to specify the databases}
+
+\item{overWriteIfExists}{(default: T) Whether to delete existing results and overwrite them}
+}
+\value{
+Returns NULL but uploads the diagnostic into the database schema specified in databaseSchemaSettings
+}
+\description{
+This function inserts a diagnostic result into the result schema
+}
+\details{
+This function can be used to upload a diagnostic result into a database
+}
diff --git a/man/addMultipleDiagnosePlpToDatabase.Rd b/man/addMultipleDiagnosePlpToDatabase.Rd
new file mode 100644
index 000000000..3d679c026
--- /dev/null
+++ b/man/addMultipleDiagnosePlpToDatabase.Rd
@@ -0,0 +1,36 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/uploadToDatabaseDiagnostics.R
+\name{addMultipleDiagnosePlpToDatabase}
+\alias{addMultipleDiagnosePlpToDatabase}
+\title{Insert mutliple diagnosePlp results saved to a directory into a PLP result schema database}
+\usage{
+addMultipleDiagnosePlpToDatabase(
+  connectionDetails,
+  databaseSchemaSettings,
+  cohortDefinitions,
+  databaseList = NULL,
+  resultLocation
+)
+}
+\arguments{
+\item{connectionDetails}{A connection details created by using the
+function \code{createConnectionDetails} in the
+\code{DatabaseConnector} package.}
+
+\item{databaseSchemaSettings}{A object created by \code{createDatabaseSchemaSettings} with all the settings specifying the result tables}
+
+\item{cohortDefinitions}{(list) A list of cohortDefinitions (each list must contain: name, id)}
+
+\item{databaseList}{(Optional) ...}
+
+\item{resultLocation}{The location of the diagnostic results}
+}
+\value{
+Returns NULL but uploads multiple diagnosePlp results into the database schema specified in databaseSchemaSettings
+}
+\description{
+This function inserts diagnosePlp results into the result schema
+}
+\details{
+This function can be used to upload diagnosePlp results into a database
+}
diff --git a/man/addMultipleRunPlpToDatabase.Rd b/man/addMultipleRunPlpToDatabase.Rd
new file mode 100644
index 000000000..696fc81b3
--- /dev/null
+++ b/man/addMultipleRunPlpToDatabase.Rd
@@ -0,0 +1,42 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/uploadToDatabase.R
+\name{addMultipleRunPlpToDatabase}
+\alias{addMultipleRunPlpToDatabase}
+\title{Populate the PatientLevelPrediction results tables}
+\usage{
+addMultipleRunPlpToDatabase(
+  connectionDetails,
+  databaseSchemaSettings = createDatabaseSchemaSettings(resultSchema = "main"),
+  cohortDefinitions,
+  databaseList = NULL,
+  resultLocation = NULL,
+  resultLocationVector,
+  modelSaveLocation
+)
+}
+\arguments{
+\item{connectionDetails}{A connection details created by using the
+function \code{createConnectionDetails} in the
+\code{DatabaseConnector} package.}
+
+\item{databaseSchemaSettings}{A object created by \code{createDatabaseSchemaSettings} with all the settings specifying the result tables}
+
+\item{cohortDefinitions}{A set of one or more cohorts extracted using ROhdsiWebApi::exportCohortDefinitionSet()}
+
+\item{databaseList}{(Optional) A list created by \code{createDatabaseList} to specify the databases}
+
+\item{resultLocation}{(string) location of directory where the main package results were saved}
+
+\item{resultLocationVector}{(only used when resultLocation is missing) a vector of locations with development or validation results}
+
+\item{modelSaveLocation}{The location of the file system for saving the models in a subdirectory}
+}
+\value{
+Returns NULL but uploads all the results in resultLocation to the PatientLevelPrediction result tables in resultSchema
+}
+\description{
+This function formats and uploads results that have been generated via an ATLAS prediction package into a database
+}
+\details{
+This function can be used upload PatientLevelPrediction results into a database
+}
diff --git a/man/addRunPlpToDatabase.Rd b/man/addRunPlpToDatabase.Rd
new file mode 100644
index 000000000..c77dc3eaf
--- /dev/null
+++ b/man/addRunPlpToDatabase.Rd
@@ -0,0 +1,39 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/uploadToDatabase.R
+\name{addRunPlpToDatabase}
+\alias{addRunPlpToDatabase}
+\title{Function to add the run plp (development or validation) to database}
+\usage{
+addRunPlpToDatabase(
+  runPlp,
+  connectionDetails,
+  databaseSchemaSettings,
+  cohortDefinitions,
+  modelSaveLocation,
+  databaseList = NULL
+)
+}
+\arguments{
+\item{runPlp}{An object of class \code{runPlp} or class \code{externalValidatePlp}}
+
+\item{connectionDetails}{A connection details created by using the
+function \code{createConnectionDetails} in the
+\code{DatabaseConnector} package.}
+
+\item{databaseSchemaSettings}{A object created by \code{createDatabaseSchemaSettings} with all the settings specifying the result tables}
+
+\item{cohortDefinitions}{A set of one or more cohorts extracted using ROhdsiWebApi::exportCohortDefinitionSet()}
+
+\item{modelSaveLocation}{The location of the directory that models will be saved to}
+
+\item{databaseList}{(Optional) If you want to change the database name then used \code{createDatabaseList} to specify the database settings but use the same cdmDatabaseId was model development/validation}
+}
+\value{
+Returns a data.frame with the database details
+}
+\description{
+This function adds a runPlp or external validation result into a database
+}
+\details{
+This function is used when inserting results into the PatientLevelPrediction database results schema
+}
diff --git a/man/computeGridPerformance.Rd b/man/computeGridPerformance.Rd
new file mode 100644
index 000000000..35a1b9ee2
--- /dev/null
+++ b/man/computeGridPerformance.Rd
@@ -0,0 +1,22 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/SklearnClassifier.R
+\name{computeGridPerformance}
+\alias{computeGridPerformance}
+\title{Computes grid performance with a specified performance function}
+\usage{
+computeGridPerformance(prediction, param, performanceFunct = "computeAuc")
+}
+\arguments{
+\item{prediction}{a dataframe with predictions and outcomeCount per rowId}
+
+\item{param}{a list of hyperparameters}
+
+\item{performanceFunct}{a string specifying which performance function to use
+. Default ``'compute_AUC'``}
+}
+\value{
+A list with overview of the performance
+}
+\description{
+Computes grid performance with a specified performance function
+}
diff --git a/man/configurePython.Rd b/man/configurePython.Rd
index 9d014fa94..99e8d79aa 100644
--- a/man/configurePython.Rd
+++ b/man/configurePython.Rd
@@ -4,12 +4,14 @@
 \alias{configurePython}
 \title{Sets up a virtual environment to use for PLP (can be conda or python)}
 \usage{
-configurePython(envname = "PLP", envtype = NULL)
+configurePython(envname = "PLP", envtype = NULL, condaPythonVersion = "3.11")
 }
 \arguments{
 \item{envname}{A string for the name of the virtual environment (default is 'PLP')}
 
 \item{envtype}{An option for specifying the environment as'conda' or 'python'.  If NULL then the default is 'conda' for windows users and 'python' for non-windows users}
+
+\item{condaPythonVersion}{String, Python version to use when creating a conda environment}
 }
 \description{
 Sets up a virtual environment to use for PLP (can be conda or python)
diff --git a/man/createDatabaseDetails.Rd b/man/createDatabaseDetails.Rd
index fdcd1ae40..f848843cd 100644
--- a/man/createDatabaseDetails.Rd
+++ b/man/createDatabaseDetails.Rd
@@ -8,14 +8,16 @@ createDatabaseDetails(
   connectionDetails,
   cdmDatabaseSchema,
   cdmDatabaseName,
+  cdmDatabaseId,
   tempEmulationSchema = cdmDatabaseSchema,
   cohortDatabaseSchema = cdmDatabaseSchema,
   cohortTable = "cohort",
   outcomeDatabaseSchema = cdmDatabaseSchema,
   outcomeTable = "cohort",
-  cohortId = NULL,
+  targetId = NULL,
   outcomeIds = NULL,
-  cdmVersion = 5
+  cdmVersion = 5,
+  cohortId = NULL
 )
 }
 \arguments{
@@ -28,7 +30,9 @@ instance. Requires read permissions to this database. On SQL
 Server, this should specifiy both the database and the schema,
 so for example 'cdm_instance.dbo'.}
 
-\item{cdmDatabaseName}{A string with a shareable name of the database (this will be shown to OHDSI researchers if the results get transported)}
+\item{cdmDatabaseName}{A string with the name of the database - this is used in the shiny app and when externally validating models to name the result list and to specify the folder name when saving validation results (defaults to cdmDatabaseSchema if not specified)}
+
+\item{cdmDatabaseId}{A string with a unique identifier for the database and version - this is stored in the plp object for future reference and used by the shiny app (defaults to cdmDatabaseSchema if not specified)}
 
 \item{tempEmulationSchema}{For dmbs like Oracle only: the name of the database schema where you
 want all temporary tables to be managed. Requires
@@ -50,12 +54,13 @@ this database.}
 outcomeTable has format of COHORT table: COHORT_DEFINITION_ID,
 SUBJECT_ID, COHORT_START_DATE, COHORT_END_DATE.}
 
-\item{cohortId}{An integer specifying the cohort id for the target cohort}
+\item{targetId}{An integer specifying the cohort id for the target cohort}
 
 \item{outcomeIds}{A single integer or vector of integers specifying the cohort ids for the outcome cohorts}
 
-\item{cdmVersion}{Define the OMOP CDM version used: currently support "4" and
-"5".}
+\item{cdmVersion}{Define the OMOP CDM version used: currently support "4" and "5".}
+
+\item{cohortId}{(depreciated: use targetId) old input for the target cohort id}
 }
 \value{
 A list with the the database specific settings (this is used by the runMultiplePlp function and the skeleton packages)
diff --git a/man/createDatabaseList.Rd b/man/createDatabaseList.Rd
new file mode 100644
index 000000000..4e9d5393c
--- /dev/null
+++ b/man/createDatabaseList.Rd
@@ -0,0 +1,24 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/uploadToDatabase.R
+\name{createDatabaseList}
+\alias{createDatabaseList}
+\title{Create a list with the database details and database meta data entries}
+\usage{
+createDatabaseList(cdmDatabaseSchemas, cdmDatabaseNames, databaseRefIds = NULL)
+}
+\arguments{
+\item{cdmDatabaseSchemas}{(string vector) A vector of the cdmDatabaseSchemas used in the study - if the schemas are not unique per database please also specify databaseRefId}
+
+\item{cdmDatabaseNames}{Sharable names for the databases}
+
+\item{databaseRefIds}{(string vector) Unique database identifiers - what you specified as cdmDatabaseId in \code{PatientLevelPrediction::createDatabaseDetails()} when developing the models}
+}
+\value{
+Returns a data.frame with the database details
+}
+\description{
+This function creates a list with the database details and database meta data entries used in the study
+}
+\details{
+This function is used when inserting database details into the PatientLevelPrediction database results schema
+}
diff --git a/man/createDatabaseSchemaSettings.Rd b/man/createDatabaseSchemaSettings.Rd
new file mode 100644
index 000000000..cd58c7da2
--- /dev/null
+++ b/man/createDatabaseSchemaSettings.Rd
@@ -0,0 +1,43 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/uploadToDatabase.R
+\name{createDatabaseSchemaSettings}
+\alias{createDatabaseSchemaSettings}
+\title{Create the PatientLevelPrediction database result schema settings}
+\usage{
+createDatabaseSchemaSettings(
+  resultSchema = "main",
+  tablePrefix = "",
+  targetDialect = "sqlite",
+  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"),
+  cohortDefinitionSchema = resultSchema,
+  tablePrefixCohortDefinitionTables = tablePrefix,
+  databaseDefinitionSchema = resultSchema,
+  tablePrefixDatabaseDefinitionTables = tablePrefix
+)
+}
+\arguments{
+\item{resultSchema}{(string) The name of the database schema with the result tables.}
+
+\item{tablePrefix}{(string) A string that appends to the PatientLevelPrediction result tables}
+
+\item{targetDialect}{(string) The database management system being used}
+
+\item{tempEmulationSchema}{(string) The temp schema used when the database management system is oracle}
+
+\item{cohortDefinitionSchema}{(string) The name of the database schema with the cohort definition tables (defaults to resultSchema).}
+
+\item{tablePrefixCohortDefinitionTables}{(string) A string that appends to the cohort definition tables}
+
+\item{databaseDefinitionSchema}{(string) The name of the database schema with the database definition tables (defaults to resultSchema).}
+
+\item{tablePrefixDatabaseDefinitionTables}{(string) A string that appends to the database definition tables}
+}
+\value{
+Returns a list of class 'plpDatabaseResultSchema' with all the database settings
+}
+\description{
+This function specifies where the results schema is and lets you pick a different schema for the cohorts and databases
+}
+\details{
+This function can be used to specify the database settings used to upload PatientLevelPrediction results into a database
+}
diff --git a/man/createDefaultSplitSetting.Rd b/man/createDefaultSplitSetting.Rd
index a3b67f2e1..b9020bd45 100644
--- a/man/createDefaultSplitSetting.Rd
+++ b/man/createDefaultSplitSetting.Rd
@@ -3,7 +3,7 @@
 \name{createDefaultSplitSetting}
 \alias{createDefaultSplitSetting}
 \title{Create the settings for defining how the plpData are split into test/validation/train sets using 
-default splitting functions (either random stratified by outcome, time or subject splitting).}
+default splitting functions (either random stratified by outcome, time or subject splitting)}
 \usage{
 createDefaultSplitSetting(
   testFraction = 0.25,
@@ -24,9 +24,9 @@ If not set train is equal to 1 - test}
 \item{nfold}{(numeric) An integer > 1 specifying the number of folds used in cross validation}
 
 \item{type}{(character) Choice of:  \itemize{
-\item{'stratified'}{ Each data point is randomly assigned into the test or a train fold set but this is done stratified such that the outcome rate is consistent in each partition }
-\item{'time')}{ Older data are assigned into the training set and newer data are assigned into the test set}
-\item{'subject'}{ Data are partitioned by subject, if a subject is in the data more than once, all the data points for the subject are assigned either into the test data or into the train data (not both).}
+\item'stratified' Each data point is randomly assigned into the test or a train fold set but this is done stratified such that the outcome rate is consistent in each partition 
+\item'time' Older data are assigned into the training set and newer data are assigned into the test set
+\item'subject' Data are partitioned by subject, if a subject is in the data more than once, all the data points for the subject are assigned either into the test data or into the train data (not both).
 }}
 }
 \value{
@@ -34,7 +34,7 @@ An object of class \code{splitSettings}
 }
 \description{
 Create the settings for defining how the plpData are split into test/validation/train sets using 
-default splitting functions (either random stratified by outcome, time or subject splitting).
+default splitting functions (either random stratified by outcome, time or subject splitting)
 }
 \details{
 Returns an object of class \code{splitSettings} that specifies the splitting function that will be called and the settings
diff --git a/man/createFeatureEngineeringSettings.Rd b/man/createFeatureEngineeringSettings.Rd
index c43b4cd2d..0b7a0a8d8 100644
--- a/man/createFeatureEngineeringSettings.Rd
+++ b/man/createFeatureEngineeringSettings.Rd
@@ -8,7 +8,7 @@ createFeatureEngineeringSettings(type = "none")
 }
 \arguments{
 \item{type}{(character) Choice of:  \itemize{
-\item{'none'}{ No feature engineering - this is the default }
+\item'none' No feature engineering - this is the default 
 }}
 }
 \value{
diff --git a/man/createLearningCurve.Rd b/man/createLearningCurve.Rd
index b8013559f..e6c113a13 100644
--- a/man/createLearningCurve.Rd
+++ b/man/createLearningCurve.Rd
@@ -15,7 +15,7 @@ createLearningCurve(
   populationSettings = createStudyPopulationSettings(),
   splitSettings = createDefaultSplitSetting(),
   trainFractions = c(0.25, 0.5, 0.75),
-  trainEvents = c(500, 1000, 1500),
+  trainEvents = NULL,
   sampleSettings = createSampleSettings(),
   featureEngineeringSettings = createFeatureEngineeringSettings(),
   preprocessSettings = createPreprocessSettings(minFraction = 0.001, normalize = T),
@@ -37,17 +37,12 @@ data extracted from the CDM.}
 
 \item{modelSettings}{An object of class \code{modelSettings} created using one of the function:
 \itemize{
-\item{setLassoLogisticRegression()}{ A lasso logistic regression model}
-\item{setGradientBoostingMachine()}{ A gradient boosting machine}
-\item{setAdaBoost()}{ An ada boost model}
-\item{setRandomForest()}{ A random forest model}
-\item{setDecisionTree()}{ A decision tree model}
-\item{setCovNN())}{ A convolutional neural network model}
-\item{setCIReNN()}{ A recurrent neural network model}
-\item{setMLP()}{ A neural network model}
-\item{setDeepNN()}{ A deep neural network model}
-\item{setKNN()}{ A KNN model}
-
+\item \code{setLassoLogisticRegression()} A lasso logistic regression model
+\item \code{setGradientBoostingMachine()} A gradient boosting machine
+\item \code{setAdaBoost()} An ada boost model
+\item \code{setRandomForest()} A random forest model
+\item \code{setDecisionTree()} A decision tree model
+\item \code{setKNN()} A KNN model
 }}
 
 \item{saveDirectory}{The path to the directory where the results will be saved (if NULL uses working directory)}
@@ -70,7 +65,7 @@ Therefore, it is recommended to provide \code{trainEvents} rather than
 \code{trainFractions}. Note, providing \code{trainEvents} will override
 your input to \code{trainFractions}. The format should be as follows:
 \itemize{
-  \item{ \code{c(500, 1000, 1500) } - a list of training events}
+  \item \code{c(500, 1000, 1500) } - a list of training events
 }}
 
 \item{sampleSettings}{An object of type \code{sampleSettings} that specifies any under/over sampling to be done.
diff --git a/man/createLogSettings.Rd b/man/createLogSettings.Rd
index 0ff82b6f1..83476c541 100644
--- a/man/createLogSettings.Rd
+++ b/man/createLogSettings.Rd
@@ -9,12 +9,12 @@ createLogSettings(verbosity = "DEBUG", timeStamp = T, logName = "runPlp Log")
 \arguments{
 \item{verbosity}{Sets the level of the verbosity. If the log level is at or higher in priority than the logger threshold, a message will print. The levels are:
 \itemize{
-\item{DEBUG}{Highest verbosity showing all debug statements}
-\item{TRACE}{Showing information about start and end of steps}
-\item{INFO}{Show informative information (Default)}
-\item{WARN}{Show warning messages}
-\item{ERROR}{Show error messages}
-\item{FATAL}{Be silent except for fatal errors}
+\item DEBUG Highest verbosity showing all debug statements
+\item TRACE Showing information about start and end of steps
+\item INFO Show informative information (Default)
+\item WARN Show warning messages
+\item ERROR Show error messages
+\item FATAL Be silent except for fatal errors
 }}
 
 \item{timeStamp}{If TRUE a timestamp will be added to each logging statement. Automatically switched on for TRACE level.}
diff --git a/man/createModelDesign.Rd b/man/createModelDesign.Rd
index dbae17f6e..593a0d8c9 100644
--- a/man/createModelDesign.Rd
+++ b/man/createModelDesign.Rd
@@ -14,6 +14,8 @@ createModelDesign(
   sampleSettings = NULL,
   preprocessSettings = NULL,
   modelSettings = NULL,
+  splitSettings = createDefaultSplitSetting(type = "stratified", testFraction = 0.25,
+    trainFraction = 0.75, splitSeed = 123, nfold = 3),
   runCovariateSummary = T
 )
 }
@@ -36,6 +38,8 @@ createModelDesign(
 
 \item{modelSettings}{The model settings such as \code{setLassoLogisticRegression()}}
 
+\item{splitSettings}{The train/validation/test splitting used by all analyses created using \code{createDefaultSplitSetting()}}
+
 \item{runCovariateSummary}{Whether to run the covariateSummary}
 }
 \value{
diff --git a/man/createPlpResultTables.Rd b/man/createPlpResultTables.Rd
index 765b870cb..8bdd6fa6c 100644
--- a/man/createPlpResultTables.Rd
+++ b/man/createPlpResultTables.Rd
@@ -1,41 +1,39 @@
 % Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/uploadPlpDbResults.R
+% Please edit documentation in R/uploadToDatabase.R
 \name{createPlpResultTables}
 \alias{createPlpResultTables}
 \title{Create the results tables to store PatientLevelPrediction models and results into a database}
 \usage{
 createPlpResultTables(
-  conn,
-  resultSchema,
+  connectionDetails,
   targetDialect = "postgresql",
-  deleteExistingTables = T,
+  resultSchema,
+  deleteTables = T,
   createTables = T,
-  stringAppendToTables = "",
+  tablePrefix = "",
   tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"),
   testFile = NULL
 )
 }
 \arguments{
-\item{conn}{A connection to a database created by using the
-function \code{connect} in the
-\code{DatabaseConnector} package.}
-
-\item{resultSchema}{The name of the database schema that the result tables will be created.}
+\item{connectionDetails}{The database connection details}
 
 \item{targetDialect}{The database management system being used}
 
-\item{deleteExistingTables}{If true any existing tables matching the PatientLevelPrediction result tables names will be deleted}
+\item{resultSchema}{The name of the database schema that the result tables will be created.}
+
+\item{deleteTables}{If true any existing tables matching the PatientLevelPrediction result tables names will be deleted}
 
 \item{createTables}{If true the PatientLevelPrediction result tables will be created}
 
-\item{stringAppendToTables}{A string that appends to the PatientLevelPrediction result tables}
+\item{tablePrefix}{A string that appends to the PatientLevelPrediction result tables}
 
 \item{tempEmulationSchema}{The temp schema used when the database management system is oracle}
 
 \item{testFile}{(used for testing) The location of an sql file with the table creation code}
 }
 \value{
-Returns NULL but creates the required tables into the specified database schema.
+Returns NULL but creates the required tables into the specified database schema(s).
 }
 \description{
 This function executes a large set of SQL statements to create tables that can store models and results
diff --git a/man/createPreprocessSettings.Rd b/man/createPreprocessSettings.Rd
index 17ae28208..6ce8ff1f7 100644
--- a/man/createPreprocessSettings.Rd
+++ b/man/createPreprocessSettings.Rd
@@ -2,7 +2,7 @@
 % Please edit documentation in R/PreprocessingData.R
 \name{createPreprocessSettings}
 \alias{createPreprocessSettings}
-\title{Create the settings for preprocessing the trainData using \code{ }.}
+\title{Create the settings for preprocessing the trainData.}
 \usage{
 createPreprocessSettings(
   minFraction = 0.001,
@@ -21,7 +21,7 @@ createPreprocessSettings(
 An object of class \code{preprocessingSettings}
 }
 \description{
-Create the settings for preprocessing the trainData using \code{ }.
+Create the settings for preprocessing the trainData.
 }
 \details{
 Returns an object of class \code{preprocessingSettings} that specifies how to preprocess the training data
diff --git a/man/createSampleSettings.Rd b/man/createSampleSettings.Rd
index 48855ee98..9c8d0b918 100644
--- a/man/createSampleSettings.Rd
+++ b/man/createSampleSettings.Rd
@@ -13,9 +13,9 @@ createSampleSettings(
 }
 \arguments{
 \item{type}{(character) Choice of:  \itemize{
-\item{'none'}{ No sampling is applied - this is the default }
-\item{'underSample')}{Undersample the non-outcome class to make the data more ballanced}
-\item{'overSample'}{Oversample the outcome class by adding in each outcome multiple times}
+\item 'none' No sampling is applied - this is the default
+\item 'underSample' Undersample the non-outcome class to make the data more ballanced
+\item 'overSample' Oversample the outcome class by adding in each outcome multiple times
 }}
 
 \item{numberOutcomestoNonOutcomes}{(numeric) An numeric specifying the require number of non-outcomes per outcome}
diff --git a/man/createSplineSettings.Rd b/man/createSplineSettings.Rd
new file mode 100644
index 000000000..36a9ae09d
--- /dev/null
+++ b/man/createSplineSettings.Rd
@@ -0,0 +1,24 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/FeatureEngineering.R
+\name{createSplineSettings}
+\alias{createSplineSettings}
+\title{Create the settings for adding a spline for continuous variables}
+\usage{
+createSplineSettings(continousCovariateId, knots, analysisId = 683)
+}
+\arguments{
+\item{continousCovariateId}{The covariateId to apply splines to}
+
+\item{knots}{Either number of knots of vector of split values}
+
+\item{analysisId}{The analysisId to use for the spline covariates}
+}
+\value{
+An object of class \code{featureEngineeringSettings}
+}
+\description{
+Create the settings for adding a spline for continuous variables
+}
+\details{
+Returns an object of class \code{featureEngineeringSettings} that specifies the sampling function that will be called and the settings
+}
diff --git a/man/createStratifiedImputationSettings.Rd b/man/createStratifiedImputationSettings.Rd
new file mode 100644
index 000000000..8332506dd
--- /dev/null
+++ b/man/createStratifiedImputationSettings.Rd
@@ -0,0 +1,22 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/FeatureEngineering.R
+\name{createStratifiedImputationSettings}
+\alias{createStratifiedImputationSettings}
+\title{Create the settings for adding a spline for continuous variables}
+\usage{
+createStratifiedImputationSettings(covariateId, ageSplits = NULL)
+}
+\arguments{
+\item{covariateId}{The covariateId that needs imputed values}
+
+\item{ageSplits}{A vector of age splits in years to create age groups}
+}
+\value{
+An object of class \code{featureEngineeringSettings}
+}
+\description{
+Create the settings for adding a spline for continuous variables
+}
+\details{
+Returns an object of class \code{featureEngineeringSettings} that specifies how to do stratified imputation
+}
diff --git a/man/createTempModelLoc.Rd b/man/createTempModelLoc.Rd
new file mode 100644
index 000000000..916c955ec
--- /dev/null
+++ b/man/createTempModelLoc.Rd
@@ -0,0 +1,11 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/HelperFunctions.R
+\name{createTempModelLoc}
+\alias{createTempModelLoc}
+\title{Create a temporary model location}
+\usage{
+createTempModelLoc()
+}
+\description{
+Create a temporary model location
+}
diff --git a/man/createValidationDesign.Rd b/man/createValidationDesign.Rd
new file mode 100644
index 000000000..f54b6aa78
--- /dev/null
+++ b/man/createValidationDesign.Rd
@@ -0,0 +1,34 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/ExternalValidatePlp.R
+\name{createValidationDesign}
+\alias{createValidationDesign}
+\title{createValidationDesign - Define the validation design for external validation}
+\usage{
+createValidationDesign(
+  targetId,
+  outcomeId,
+  populationSettings,
+  restrictPlpDataSettings,
+  plpModelList,
+  recalibrate = NULL,
+  runCovariateSummary = TRUE
+)
+}
+\arguments{
+\item{targetId}{The targetId of the target cohort to validate on}
+
+\item{outcomeId}{The outcomeId of the outcome cohort to validate on}
+
+\item{populationSettings}{A list of population restriction settings created by \code{createPopulationSettings}}
+
+\item{restrictPlpDataSettings}{A list of plpData restriction settings created by \code{createRestrictPlpDataSettings}}
+
+\item{plpModelList}{A list of plpModels objects created by \code{runPlp} or a path to such objects}
+
+\item{recalibrate}{A vector of characters specifying the recalibration method to apply,}
+
+\item{runCovariateSummary}{whether to run the covariate summary for the validation data}
+}
+\description{
+createValidationDesign - Define the validation design for external validation
+}
diff --git a/man/diagnoseMultiplePlp.Rd b/man/diagnoseMultiplePlp.Rd
new file mode 100644
index 000000000..a8a1c6ff7
--- /dev/null
+++ b/man/diagnoseMultiplePlp.Rd
@@ -0,0 +1,40 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/DiagnosePlp.R
+\name{diagnoseMultiplePlp}
+\alias{diagnoseMultiplePlp}
+\title{Run a list of predictions diagnoses}
+\usage{
+diagnoseMultiplePlp(
+  databaseDetails = createDatabaseDetails(),
+  modelDesignList = list(createModelDesign(targetId = 1, outcomeId = 2, modelSettings =
+    setLassoLogisticRegression()), createModelDesign(targetId = 1, outcomeId = 3,
+    modelSettings = setLassoLogisticRegression())),
+  cohortDefinitions = NULL,
+  logSettings = createLogSettings(verbosity = "DEBUG", timeStamp = T, logName =
+    "diagnosePlp Log"),
+  saveDirectory = getwd()
+)
+}
+\arguments{
+\item{databaseDetails}{The database settings created using \code{createDatabaseDetails()}}
+
+\item{modelDesignList}{A list of model designs created using \code{createModelDesign()}}
+
+\item{cohortDefinitions}{A list of cohort definitions for the target and outcome cohorts}
+
+\item{logSettings}{The setting spexcifying the logging for the analyses created using \code{createLogSettings()}}
+
+\item{saveDirectory}{Name of the folder where all the outputs will written to.}
+}
+\value{
+A data frame with the following columns: \tabular{ll}{ \verb{analysisId} \tab The unique identifier
+for a set of analysis choices.\cr \verb{targetId} \tab The ID of the target cohort populations.\cr
+\verb{outcomeId} \tab The ID of the outcomeId.\cr \verb{dataLocation} \tab The location where the plpData was saved 
+ \cr \verb{the settings ids} \tab The ids for all other settings used for model development.\cr }
+}
+\description{
+Run a list of predictions diagnoses
+}
+\details{
+This function will run all specified prediction design diagnoses as defined using .
+}
diff --git a/man/diagnosePlp.Rd b/man/diagnosePlp.Rd
new file mode 100644
index 000000000..e62e999ea
--- /dev/null
+++ b/man/diagnosePlp.Rd
@@ -0,0 +1,84 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/DiagnosePlp.R
+\name{diagnosePlp}
+\alias{diagnosePlp}
+\title{diagnostic - Investigates the prediction problem settings - use before training a model}
+\usage{
+diagnosePlp(
+  plpData = NULL,
+  outcomeId,
+  analysisId,
+  populationSettings,
+  splitSettings = createDefaultSplitSetting(),
+  sampleSettings = createSampleSettings(),
+  saveDirectory = NULL,
+  featureEngineeringSettings = createFeatureEngineeringSettings(),
+  modelSettings = setLassoLogisticRegression(),
+  logSettings = createLogSettings(verbosity = "DEBUG", timeStamp = T, logName =
+    "diagnosePlp Log"),
+  preprocessSettings = createPreprocessSettings()
+)
+}
+\arguments{
+\item{plpData}{An object of type \code{plpData} - the patient level prediction
+data extracted from the CDM.  Can also include an initial population as 
+plpData$popualtion.}
+
+\item{outcomeId}{(integer) The ID of the outcome.}
+
+\item{analysisId}{(integer) Identifier for the analysis. It is used to create, e.g., the result folder. Default is a timestamp.}
+
+\item{populationSettings}{An object of type \code{populationSettings} created using \code{createStudyPopulationSettings} that
+specifies how the data class labels are defined and addition any exclusions to apply to the 
+plpData cohort}
+
+\item{splitSettings}{An object of type \code{splitSettings} that specifies how to split the data into train/validation/test.
+The default settings can be created using \code{createDefaultSplitSetting}.}
+
+\item{sampleSettings}{An object of type \code{sampleSettings} that specifies any under/over sampling to be done.
+The default is none.}
+
+\item{saveDirectory}{The path to the directory where the results will be saved (if NULL uses working directory)}
+
+\item{featureEngineeringSettings}{An object of \code{featureEngineeringSettings} specifying any feature engineering to be learned (using the train data)}
+
+\item{modelSettings}{An object of class \code{modelSettings} created using one of the function:
+\itemize{
+\item setLassoLogisticRegression() A lasso logistic regression model
+\item setGradientBoostingMachine() A gradient boosting machine
+\item setAdaBoost() An ada boost model
+\item setRandomForest() A random forest model
+\item setDecisionTree() A decision tree model
+\item setKNN() A KNN model
+
+}}
+
+\item{logSettings}{An object of \code{logSettings} created using \code{createLogSettings} 
+specifying how the logging is done}
+
+\item{preprocessSettings}{An object of \code{preprocessSettings}. This setting specifies the minimum fraction of 
+target population who must have a covariate for it to be included in the model training                            
+and whether to normalise the covariates before training}
+}
+\value{
+An object containing the model or location where the model is save, the data selection settings, the preprocessing
+and training settings as well as various performance measures obtained by the model.
+
+\item{distribution}{list for each O of a data.frame containing: i) Time to observation end distribution, ii) Time from observation start distribution, iii) Time to event distribution and iv) Time from last prior event to index distribution (only for patients in T who have O before index) }
+\item{incident}{list for each O of incidence of O in T during TAR}
+\item{characterization}{list for each O of Characterization of T, TnO, Tn~O}
+}
+\description{
+This function runs a set of prediction diagnoses to help pick a suitable T, O, TAR and determine 
+whether the prediction problem is worth executing.
+}
+\details{
+Users can define set of Ts, Os, databases and population settings.  A list of data.frames containing details such as
+follow-up time distribution, time-to-event information, characteriszation details, time from last prior event, 
+observation time distribution.
+}
+\examples{
+\dontrun{
+#******** EXAMPLE 1 ********* 
+} 
+}
diff --git a/man/diagnostic.Rd b/man/diagnostic.Rd
deleted file mode 100644
index c9cd53063..000000000
--- a/man/diagnostic.Rd
+++ /dev/null
@@ -1,59 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/Diagnostics.R
-\name{diagnostic}
-\alias{diagnostic}
-\title{diagnostic - Investigates the prediction problem settings - use before training a model}
-\usage{
-diagnostic(
-  plpData = NULL,
-  cdmDatabaseName = "none",
-  cohortName,
-  outcomeNames,
-  databaseDetails,
-  restrictPlpDataSettings,
-  populationSettings,
-  outputFolder = NULL,
-  minCellCount = 5
-)
-}
-\arguments{
-\item{plpData}{The data object to do the diagnostic on - if NULL you need to specify the connection settings below}
-
-\item{cdmDatabaseName}{The name of the database being diagnosed}
-
-\item{cohortName}{Name of the target cohort}
-
-\item{outcomeNames}{Vector of outcome names}
-
-\item{databaseDetails}{(only used is plpData is NULL) The database details created using \code{createDatabaseDetails}}
-
-\item{restrictPlpDataSettings}{(only used is plpData is NULL) The restrictPlpDataSettings created using \code{createRestrictPlpDataSettings}}
-
-\item{populationSettings}{The population setting details created using \code{createPopulationSettings}}
-
-\item{outputFolder}{Location to save results for shiny app}
-
-\item{minCellCount}{The minimum count that will be displayed}
-}
-\value{
-An object containing the model or location where the model is save, the data selection settings, the preprocessing
-and training settings as well as various performance measures obtained by the model.
-
-\item{distribution}{list for each O of a data.frame containing: i) Time to observation end distribution, ii) Time from observation start distribution, iii) Time to event distribution and iv) Time from last prior event to index distribution (only for patients in T who have O before index) }
-\item{incident}{list for each O of incidence of O in T during TAR}
-\item{characterization}{list for each O of Characterization of T, TnO, Tn~O}
-}
-\description{
-This function runs a set of prediction diagnoses to help pick a suitable T, O, TAR and determine 
-whether the prediction problem is worth executing.
-}
-\details{
-Users can define set of Ts, Os, databases and population settings.  A list of data.frames containing details such as
-follow-up time distribution, time-to-event information, characteriszation details, time from last prior event, 
-observation time distribution.
-}
-\examples{
-\dontrun{
-#******** EXAMPLE 1 ********* 
-} 
-}
diff --git a/man/externalValidateDbPlp.Rd b/man/externalValidateDbPlp.Rd
index 9582ba332..ad3755bb5 100644
--- a/man/externalValidateDbPlp.Rd
+++ b/man/externalValidateDbPlp.Rd
@@ -22,7 +22,7 @@ externalValidateDbPlp(
 
 \item{settings}{A settings object of class \code{validationSettings} created using \code{createValidationSettings}}
 
-\item{logSettings}{An object of \code{logSettings} created using \code{createLogSettings} 
+\item{logSettings}{An object of \code{logSettings} created using \code{createLogSettings}
 specifying how the logging is done}
 
 \item{outputFolder}{The directory to save the validation results to (subfolders are created per database in validationDatabaseDetails)}
@@ -34,6 +34,6 @@ A list containing the performance for each validation_schema
 This function extracts data using a user specified connection and cdm_schema, applied the model and then calcualtes the performance
 }
 \details{
-Users need to input a trained model (the output of runPlp()) and new database connections. The function will return a list of length equal to the 
+Users need to input a trained model (the output of runPlp()) and new database connections. The function will return a list of length equal to the
 number of cdm_schemas input with the performance on the new data
 }
diff --git a/man/extractDatabaseToCsv.Rd b/man/extractDatabaseToCsv.Rd
new file mode 100644
index 000000000..98b4a6f08
--- /dev/null
+++ b/man/extractDatabaseToCsv.Rd
@@ -0,0 +1,37 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/SaveLoadPlp.R
+\name{extractDatabaseToCsv}
+\alias{extractDatabaseToCsv}
+\title{Exports all the results from a database into csv files}
+\usage{
+extractDatabaseToCsv(
+  conn = NULL,
+  connectionDetails,
+  databaseSchemaSettings = createDatabaseSchemaSettings(resultSchema = "main"),
+  csvFolder,
+  minCellCount = 5,
+  sensitiveColumns = getPlpSensitiveColumns(),
+  fileAppend = NULL
+)
+}
+\arguments{
+\item{conn}{The connection to the database with the results}
+
+\item{connectionDetails}{The connectionDetails for the result database}
+
+\item{databaseSchemaSettings}{The result database schema settings}
+
+\item{csvFolder}{Location to save the csv files}
+
+\item{minCellCount}{The min value to show in cells that are sensitive (values less than this value will be replaced with -1)}
+
+\item{sensitiveColumns}{A named list (name of table columns belong to) with a list of columns to apply the minCellCount to.}
+
+\item{fileAppend}{If set to a string this will be appended to the start of the csv file names}
+}
+\description{
+Exports all the results from a database into csv files
+}
+\details{
+Extracts the results from a database into a set of csv files
+}
diff --git a/man/fitPlp.Rd b/man/fitPlp.Rd
index 8ed1a5e98..5f52edb8b 100644
--- a/man/fitPlp.Rd
+++ b/man/fitPlp.Rd
@@ -4,7 +4,7 @@
 \alias{fitPlp}
 \title{fitPlp}
 \usage{
-fitPlp(trainData, modelSettings, search = "grid", analysisId)
+fitPlp(trainData, modelSettings, search = "grid", analysisId, analysisPath)
 }
 \arguments{
 \item{trainData}{An object of type \code{TrainData} created using \code{splitData}
@@ -12,27 +12,27 @@ data extracted from the CDM.}
 
 \item{modelSettings}{An object of class \code{modelSettings} created using one of the function:
 \itemize{
-\item{logisticRegressionModel()}{ A lasso logistic regression model}
-\item{GBMclassifier()}{ A gradient boosting machine}
-\item{RFclassifier()}{ A random forest model}
-\item{GLMclassifier ()}{ A generalised linear model}
-\item{KNNclassifier()}{ A KNN model}
+\item setLassoLogisticRegression() A lasso logistic regression model
+\item setGradientBoostingMachine() A gradient boosting machine
+\item setRandomForest() A random forest model
+\item setKNN() A KNN model
 }}
 
 \item{search}{The search strategy for the hyper-parameter selection (currently not used)}
 
 \item{analysisId}{The id of the analysis}
+
+\item{analysisPath}{The path of the analysis}
 }
 \value{
 An object of class \code{plpModel} containing:
 
 \item{model}{The trained prediction model}
-\item{modelLoc}{The path to where the model is saved (if saved)}
-\item{trainAuc}{The AUC obtained on the training set}
-\item{trainCalibration}{The calibration obtained on the training set}
-\item{modelSettings}{A list specifiying the model, preprocessing, outcomeId and cohortId}
-\item{metaData}{The model meta data}
-\item{trainingTime}{The time taken to train the classifier}
+\item{preprocessing}{The preprocessing required when applying the model}
+\item{prediction}{The cohort data.frame with the predicted risk column added}
+\item{modelDesign}{A list specifiying the modelDesign settings used to fit the model}
+\item{trainDetails}{The model meta data}
+\item{covariateImportance}{The covariate importance for the model}
 }
 \description{
 Train various models using a default parameter gird search or user specified parameters
diff --git a/man/getCohortCovariateData.Rd b/man/getCohortCovariateData.Rd
index 1014cad53..5c0c515f3 100644
--- a/man/getCohortCovariateData.Rd
+++ b/man/getCohortCovariateData.Rd
@@ -12,8 +12,9 @@ getCohortCovariateData(
   cohortTable = "#cohort_person",
   rowIdField = "row_id",
   aggregated,
-  cohortId,
-  covariateSettings
+  cohortIds,
+  covariateSettings,
+  ...
 )
 }
 \arguments{
@@ -31,9 +32,11 @@ getCohortCovariateData(
 
 \item{aggregated}{whether the covariate should be aggregated}
 
-\item{cohortId}{cohort id for the target population cohort}
+\item{cohortIds}{cohort id for the target cohort}
 
 \item{covariateSettings}{settings for the covariate cohorts and time periods}
+
+\item{...}{additional arguments from FeatureExtraction}
 }
 \value{
 The models will now be in the package
diff --git a/man/insertCsvToDatabase.Rd b/man/insertCsvToDatabase.Rd
new file mode 100644
index 000000000..3b9dd9015
--- /dev/null
+++ b/man/insertCsvToDatabase.Rd
@@ -0,0 +1,34 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/ImportFromCsv.R
+\name{insertCsvToDatabase}
+\alias{insertCsvToDatabase}
+\title{Function to insert results into a database from csvs}
+\usage{
+insertCsvToDatabase(
+  csvFolder,
+  connectionDetails,
+  databaseSchemaSettings,
+  modelSaveLocation,
+  csvTableAppend = ""
+)
+}
+\arguments{
+\item{csvFolder}{The location to the csv folder with the plp results}
+
+\item{connectionDetails}{A connection details for the plp results database that the csv results will be inserted into}
+
+\item{databaseSchemaSettings}{A object created by \code{createDatabaseSchemaSettings} with all the settings specifying the result tables to insert the csv results into}
+
+\item{modelSaveLocation}{The location to save any models from the csv folder - this should be the same location you picked when inserting other models into the database}
+
+\item{csvTableAppend}{A string that appends the csv file names}
+}
+\value{
+Returns a data.frame indicating whether the results were inported into the database
+}
+\description{
+This function converts a folder with csv results into plp objects and loads them into a plp result database
+}
+\details{
+The user needs to have plp csv results in a single folder and an existing plp result database
+}
diff --git a/man/insertModelDesignInDatabase.Rd b/man/insertModelDesignInDatabase.Rd
new file mode 100644
index 000000000..7806d6c95
--- /dev/null
+++ b/man/insertModelDesignInDatabase.Rd
@@ -0,0 +1,33 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/uploadToDatabaseModelDesign.R
+\name{insertModelDesignInDatabase}
+\alias{insertModelDesignInDatabase}
+\title{Insert a model design into a PLP result schema database}
+\usage{
+insertModelDesignInDatabase(
+  object,
+  conn,
+  databaseSchemaSettings,
+  cohortDefinitions
+)
+}
+\arguments{
+\item{object}{An object of class modelDesign, runPlp or externalValidatePlp}
+
+\item{conn}{A connection to a database created by using the
+function \code{connect} in the
+\code{DatabaseConnector} package.}
+
+\item{databaseSchemaSettings}{A object created by \code{createDatabaseSchemaSettings} with all the settings specifying the result tables}
+
+\item{cohortDefinitions}{A set of one or more cohorts extracted using ROhdsiWebApi::exportCohortDefinitionSet()}
+}
+\value{
+Returns NULL but uploads the model design into the database schema specified in databaseSchemaSettings
+}
+\description{
+This function inserts a model design and all the settings into the result schema
+}
+\details{
+This function can be used to upload a model design into a database
+}
diff --git a/man/insertResultsToSqlite.Rd b/man/insertResultsToSqlite.Rd
new file mode 100644
index 000000000..0376c3b9e
--- /dev/null
+++ b/man/insertResultsToSqlite.Rd
@@ -0,0 +1,31 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/uploadToDatabase.R
+\name{insertResultsToSqlite}
+\alias{insertResultsToSqlite}
+\title{Create sqlite database with the results}
+\usage{
+insertResultsToSqlite(
+  resultLocation,
+  cohortDefinitions,
+  databaseList = NULL,
+  sqliteLocation = file.path(resultLocation, "sqlite")
+)
+}
+\arguments{
+\item{resultLocation}{(string) location of directory where the main package results were saved}
+
+\item{cohortDefinitions}{A set of one or more cohorts extracted using ROhdsiWebApi::exportCohortDefinitionSet()}
+
+\item{databaseList}{A list created by \code{createDatabaseList} to specify the databases}
+
+\item{sqliteLocation}{(string) location of directory where the sqlite database will be saved}
+}
+\value{
+Returns the location of the sqlite database file
+}
+\description{
+This function create an sqlite database with the PLP result schema and inserts all results
+}
+\details{
+This function can be used upload PatientLevelPrediction results into an sqlite database
+}
diff --git a/man/launchDiagnosticsExplorer.Rd b/man/launchDiagnosticsExplorer.Rd
deleted file mode 100644
index cd6adf7f7..000000000
--- a/man/launchDiagnosticsExplorer.Rd
+++ /dev/null
@@ -1,22 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/ViewShinyPlp.R
-\name{launchDiagnosticsExplorer}
-\alias{launchDiagnosticsExplorer}
-\title{Launch the Diagnostics Explorer Shiny app}
-\usage{
-launchDiagnosticsExplorer(dataFolder, launch.browser = FALSE)
-}
-\arguments{
-\item{dataFolder}{A folder where the exported zip files with the results are stored.
-Zip files containing results from multiple databases can be placed in the same
-folder.}
-
-\item{launch.browser}{Should the app be launched in your default browser, or in a Shiny window.
-Note: copying to clipboard will not work in a Shiny window.}
-}
-\description{
-Launch the Diagnostics Explorer Shiny app
-}
-\details{
-Launches a Shiny app that allows the user to explore the diagnostics
-}
diff --git a/man/listCartesian.Rd b/man/listCartesian.Rd
new file mode 100644
index 000000000..4c6ffaba8
--- /dev/null
+++ b/man/listCartesian.Rd
@@ -0,0 +1,17 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/SklearnClassifierHelpers.R
+\name{listCartesian}
+\alias{listCartesian}
+\title{Cartesian product}
+\usage{
+listCartesian(allList)
+}
+\arguments{
+\item{allList}{a list of lists}
+}
+\value{
+A list with all possible combinations from the input list of lists
+}
+\description{
+Computes the Cartesian product of all the combinations of elements in a list
+}
diff --git a/man/migrateDataModel.Rd b/man/migrateDataModel.Rd
new file mode 100644
index 000000000..597d37637
--- /dev/null
+++ b/man/migrateDataModel.Rd
@@ -0,0 +1,21 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/DatabaseMigration.R
+\name{migrateDataModel}
+\alias{migrateDataModel}
+\title{Migrate Data model}
+\usage{
+migrateDataModel(connectionDetails, databaseSchema, tablePrefix = "")
+}
+\arguments{
+\item{connectionDetails}{DatabaseConnector connection details object}
+
+\item{databaseSchema}{String schema where database schema lives}
+
+\item{tablePrefix}{(Optional) Use if a table prefix is used before table names (e.g. "cd_")}
+}
+\description{
+Migrate data from current state to next state
+
+It is strongly advised that you have a backup of all data (either sqlite files, a backup database (in the case you
+are using a postgres backend) or have kept the csv/zip files from your data generation.
+}
diff --git a/man/outcomeSurvivalPlot.Rd b/man/outcomeSurvivalPlot.Rd
index 0df4f85d7..39100718d 100644
--- a/man/outcomeSurvivalPlot.Rd
+++ b/man/outcomeSurvivalPlot.Rd
@@ -7,10 +7,10 @@
 outcomeSurvivalPlot(
   plpData,
   outcomeId,
-  populationSettings = createStudyPopulationSettings(binary = T, includeAllOutcomes =
-    T, firstExposureOnly = FALSE, washoutPeriod = 0, removeSubjectsWithPriorOutcome =
-    TRUE, priorOutcomeLookback = 99999, requireTimeAtRisk = F, riskWindowStart = 1,
-    startAnchor = "cohort start", riskWindowEnd = 3650, endAnchor = "cohort start"),
+  populationSettings = createStudyPopulationSettings(binary = T, includeAllOutcomes = T,
+    firstExposureOnly = FALSE, washoutPeriod = 0, removeSubjectsWithPriorOutcome = TRUE,
+    priorOutcomeLookback = 99999, requireTimeAtRisk = F, riskWindowStart = 1, startAnchor
+    = "cohort start", riskWindowEnd = 3650, endAnchor = "cohort start"),
   riskTable = T,
   confInt = T,
   yLabel = "Fraction of those who are outcome free in target population"
diff --git a/man/populatePlpResultTables.Rd b/man/populatePlpResultTables.Rd
deleted file mode 100644
index 3519edecd..000000000
--- a/man/populatePlpResultTables.Rd
+++ /dev/null
@@ -1,97 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/uploadPlpDbResults.R
-\name{populatePlpResultTables}
-\alias{populatePlpResultTables}
-\title{Populate the PatientLevelPrediction results tables}
-\usage{
-populatePlpResultTables(
-  conn,
-  resultSchema,
-  stringAppendToTables = "",
-  targetDialect = "postgresql",
-  tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"),
-  packageName,
-  studyJsonList,
-  studyName = "",
-  studyDescription = "",
-  researcherName = "",
-  researcherEmail = "",
-  researcherOrg = "",
-  databaseName = NULL,
-  databaseAcronym = NULL,
-  databaseVersion = 1,
-  databaseDescription = NULL,
-  databaseType = NULL,
-  valDatabases = list(ccae = list(name = "CCAE", description = "", version = 1, type =
-    "US Claims")),
-  resultLocation = NULL,
-  resultPattern = "",
-  validationLocation = file.path(resultLocation, "Validation"),
-  addInternalValidation = T,
-  addExternalValidation = T,
-  gsubVal = NULL,
-  removePattern = NULL
-)
-}
-\arguments{
-\item{conn}{A connection to a database created by using the
-function \code{connect} in the
-\code{DatabaseConnector} package.}
-
-\item{resultSchema}{(string) The name of the database schema with the result tables.}
-
-\item{stringAppendToTables}{(string) A string that appends to the PatientLevelPrediction result tables}
-
-\item{targetDialect}{(string) The database management system being used}
-
-\item{tempEmulationSchema}{(string) The temp schema used when the database management system is oracle}
-
-\item{packageName}{(string) The name of the ATLAS R package used to generate the results (this is used to extract cohort jsons)}
-
-\item{studyJsonList}{(list) A list of lists per cohort with the cohort_name, cohort_id and cohort_json}
-
-\item{studyName}{(string) A reference study name}
-
-\item{studyDescription}{(string) A description of the study}
-
-\item{researcherName}{(string) Name of the researcher who developed the study}
-
-\item{researcherEmail}{(string) Email of the researcher who developed the study}
-
-\item{researcherOrg}{(string) Organisation of the researcher who developed the study}
-
-\item{databaseName}{(string) name of the database used to develop the model/s}
-
-\item{databaseAcronym}{(string) acronym of the database used to develop the model/s}
-
-\item{databaseVersion}{(int) Version of the database used to develop the model/s}
-
-\item{databaseDescription}{(string) Description of the database used to develop the model/s}
-
-\item{databaseType}{(string) Type of the database used to develop the model/s (e.g., claims)}
-
-\item{valDatabases}{(list) A named list with details of the external validation databases.  Needs to contain: name, description, version, type.}
-
-\item{resultLocation}{(string) location of directory where the main package results were saved}
-
-\item{resultPattern}{(string) A string to match to select models of interest}
-
-\item{validationLocation}{(string) location of directory where the validation package results were saved}
-
-\item{addInternalValidation}{(boolean) Whether the internval validation results should be uploaded}
-
-\item{addExternalValidation}{(boolean) Whether the externval validation results should be uploaded}
-
-\item{gsubVal}{(string) Remove patterns from the result name}
-
-\item{removePattern}{(string) Restrict to result names with this pattern}
-}
-\value{
-Returns NULL but uploads all the results in resultLocation to the PatientLevelPrediction result tables in resultSchema
-}
-\description{
-This function formats and uploads results that have been generated via an ATLAS prediction package into a database
-}
-\details{
-This function can be used upload PatientLevelPrediction results into a database
-}
diff --git a/man/runMultiplePlp.Rd b/man/runMultiplePlp.Rd
index b60663784..22192b5a8 100644
--- a/man/runMultiplePlp.Rd
+++ b/man/runMultiplePlp.Rd
@@ -10,12 +10,11 @@ runMultiplePlp(
     setLassoLogisticRegression()), createModelDesign(targetId = 1, outcomeId = 3,
     modelSettings = setLassoLogisticRegression())),
   onlyFetchData = F,
-  splitSettings = createDefaultSplitSetting(type = "stratified", testFraction = 0.25,
-    trainFraction = 0.75, splitSeed = 123, nfold = 3),
   cohortDefinitions = NULL,
   logSettings = createLogSettings(verbosity = "DEBUG", timeStamp = T, logName =
     "runPlp Log"),
-  saveDirectory = getwd()
+  saveDirectory = getwd(),
+  sqliteLocation = file.path(saveDirectory, "sqlite")
 )
 }
 \arguments{
@@ -25,18 +24,18 @@ runMultiplePlp(
 
 \item{onlyFetchData}{Only fetches and saves the data object to the output folder without running the analysis.}
 
-\item{splitSettings}{The train/validation/test splitting used by all analyses created using \code{createDefaultSplitSetting()}}
-
 \item{cohortDefinitions}{A list of cohort definitions for the target and outcome cohorts}
 
-\item{logSettings}{The setting spexcifying the logging for the analyses created using \code{createLogSettings()}}
+\item{logSettings}{The setting specifying the logging for the analyses created using \code{createLogSettings()}}
 
 \item{saveDirectory}{Name of the folder where all the outputs will written to.}
+
+\item{sqliteLocation}{(optional) The location of the sqlite database with the results}
 }
 \value{
 A data frame with the following columns: \tabular{ll}{ \verb{analysisId} \tab The unique identifier
-for a set of analysis choices.\cr \verb{cohortId} \tab The ID of the target cohort populations.\cr
-\verb{outcomeId} \tab The ID of the outcomeId.\cr \verb{dataLocation} \tab The location where the plpData was saved \cr \verb{evaluationFolder} \tab The name of file containing the evaluation saved as a csv
+for a set of analysis choices.\cr \verb{targetId} \tab The ID of the target cohort populations.\cr
+\verb{outcomeId} \tab The ID of the outcomeId.\cr \verb{dataLocation} \tab The location where the plpData was saved 
  \cr \verb{the settings ids} \tab The ids for all other settings used for model development.\cr }
 }
 \description{
diff --git a/man/runPlp.Rd b/man/runPlp.Rd
index 6d3b99db1..ad000f4b7 100644
--- a/man/runPlp.Rd
+++ b/man/runPlp.Rd
@@ -24,7 +24,8 @@ runPlp(
 }
 \arguments{
 \item{plpData}{An object of type \code{plpData} - the patient level prediction
-data extracted from the CDM.}
+data extracted from the CDM.  Can also include an initial population as 
+plpData$popualtion.}
 
 \item{outcomeId}{(integer) The ID of the outcome.}
 
@@ -50,17 +51,12 @@ and whether to normalise the covariates before training}
 
 \item{modelSettings}{An object of class \code{modelSettings} created using one of the function:
 \itemize{
-\item{setLassoLogisticRegression()}{ A lasso logistic regression model}
-\item{setGradientBoostingMachine()}{ A gradient boosting machine}
-\item{setAdaBoost()}{ An ada boost model}
-\item{setRandomForest()}{ A random forest model}
-\item{setDecisionTree()}{ A decision tree model}
-\item{setCovNN())}{ A convolutional neural network model}
-\item{setCIReNN()}{ A recurrent neural network model}
-\item{setMLP()}{ A neural network model}
-\item{setDeepNN()}{ A deep neural network model}
-\item{setKNN()}{ A KNN model}
-
+\item setLassoLogisticRegression() A lasso logistic regression model
+\item setGradientBoostingMachine() A gradient boosting machine
+\item setAdaBoost() An ada boost model
+\item setRandomForest() A random forest model
+\item setDecisionTree() A decision tree model
+\item setKNN() A KNN model
 }}
 
 \item{logSettings}{An object of \code{logSettings} created using \code{createLogSettings} 
@@ -74,13 +70,12 @@ specifying how the logging is done}
 An object containing the following:
 
  \itemize{
-          \item{inputSettings}{A list containing all the settings used to develop the model}
-          \item{model}{ The developed model of class \code{plpModel}}
-          \item{executionSummary}{ A list containing the hardward details, R package details and execution time}
-          \item{performanceEvaluation}{ Various internal performance metrics in sparse format}
-          \item{prediction}{ The plpData cohort table with the predicted risks added as a column (named value)}
-          \item{covariateSummary)}{ A characterization of the features for patients with and without the outcome during the time at risk}
-          \item{analysisRef}{ A list with details about the analysis}
+          \item model The developed model of class \code{plpModel}
+          \item executionSummary A list containing the hardward details, R package details and execution time
+          \item performanceEvaluation Various internal performance metrics in sparse format
+          \item prediction The plpData cohort table with the predicted risks added as a column (named value)
+          \item covariateSummary A characterization of the features for patients with and without the outcome during the time at risk
+          \item analysisRef A list with details about the analysis
           }
 }
 \description{
diff --git a/man/savePlpAnalysesJson.Rd b/man/savePlpAnalysesJson.Rd
index 1be1cc384..742631e6f 100644
--- a/man/savePlpAnalysesJson.Rd
+++ b/man/savePlpAnalysesJson.Rd
@@ -8,12 +8,15 @@ savePlpAnalysesJson(
   modelDesignList = list(createModelDesign(targetId = 1, outcomeId = 2, modelSettings =
     setLassoLogisticRegression()), createModelDesign(targetId = 1, outcomeId = 3,
     modelSettings = setLassoLogisticRegression())),
+  cohortDefinitions = NULL,
   saveDirectory = NULL
 )
 }
 \arguments{
 \item{modelDesignList}{A list of modelDesigns created using \code{createModelDesign()}}
 
+\item{cohortDefinitions}{A list of the cohortDefinitions (generally extracted from ATLAS)}
+
 \item{saveDirectory}{The directory to save the modelDesignList settings}
 }
 \description{
diff --git a/man/setAdaBoost.Rd b/man/setAdaBoost.Rd
index 0572f7828..971948d00 100644
--- a/man/setAdaBoost.Rd
+++ b/man/setAdaBoost.Rd
@@ -26,7 +26,7 @@ Create setting for AdaBoost with python DecisionTreeClassifier base estimator
 }
 \examples{
 \dontrun{
-model.adaBoost <- setAdaBoost(nEstimators = list(10,50,200), learningRate = list(1, 0.5, 0.1), 
+model.adaBoost <- setAdaBoost(nEstimators = list(10,50,200), learningRate = list(1, 0.5, 0.1),
                               algorithm = list('SAMME.R'), seed = sample(1000000,1)
                               )
 }
diff --git a/man/setDecisionTree.Rd b/man/setDecisionTree.Rd
index 7a32bcf8c..d977d0ee9 100644
--- a/man/setDecisionTree.Rd
+++ b/man/setDecisionTree.Rd
@@ -11,10 +11,10 @@ setDecisionTree(
   minSamplesSplit = list(2, 10),
   minSamplesLeaf = list(10, 50),
   minWeightFractionLeaf = list(0),
-  maxFeatures = list(100, "auto", NULL),
+  maxFeatures = list(100, "sqrt", NULL),
   maxLeafNodes = list(NULL),
   minImpurityDecrease = list(10^-7),
-  classWeight = list(NULL, "balanced"),
+  classWeight = list(NULL),
   seed = sample(1e+06, 1)
 )
 }
@@ -31,7 +31,7 @@ setDecisionTree(
 
 \item{minWeightFractionLeaf}{The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sampleWeight is not provided.}
 
-\item{maxFeatures}{(list) The number of features to consider when looking for the best split (int/'auto'/NULL)}
+\item{maxFeatures}{(list) The number of features to consider when looking for the best split (int/'sqrt'/NULL)}
 
 \item{maxLeafNodes}{(list) Grow a tree with max_leaf_nodes in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes. (int/NULL)}
 
diff --git a/man/setLassoLogisticRegression.Rd b/man/setLassoLogisticRegression.Rd
index f3b1ce51b..533aa53b3 100644
--- a/man/setLassoLogisticRegression.Rd
+++ b/man/setLassoLogisticRegression.Rd
@@ -14,7 +14,8 @@ setLassoLogisticRegression(
   upperLimit = 20,
   lowerLimit = 0.01,
   tolerance = 2e-06,
-  maxIterations = 3000
+  maxIterations = 3000,
+  priorCoefs = NULL
 )
 }
 \arguments{
@@ -37,6 +38,8 @@ setLassoLogisticRegression(
 \item{tolerance}{Numeric: maximum relative change in convergence criterion from successive iterations to achieve convergence}
 
 \item{maxIterations}{Integer: maximum iterations of Cyclops to attempt before returning a failed-to-converge error}
+
+\item{priorCoefs}{Use coefficients from a previous model as starting points for model fit (transfer learning)}
 }
 \description{
 Create setting for lasso logistic regression
diff --git a/man/setLightGBM.Rd b/man/setLightGBM.Rd
new file mode 100644
index 000000000..6380df304
--- /dev/null
+++ b/man/setLightGBM.Rd
@@ -0,0 +1,56 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/LightGBM.R
+\name{setLightGBM}
+\alias{setLightGBM}
+\title{Create setting for gradient boosting machine model using lightGBM (https://github.com/microsoft/LightGBM/tree/master/R-package).}
+\usage{
+setLightGBM(
+  nthread = 20,
+  earlyStopRound = 25,
+  numIterations = c(100),
+  numLeaves = c(31),
+  maxDepth = c(5, 10),
+  minDataInLeaf = c(20),
+  learningRate = c(0.05, 0.1, 0.3),
+  lambdaL1 = c(0),
+  lambdaL2 = c(0),
+  scalePosWeight = 1,
+  isUnbalance = FALSE,
+  seed = sample(1e+07, 1)
+)
+}
+\arguments{
+\item{nthread}{The number of computer threads to use (how many cores do you have?)}
+
+\item{earlyStopRound}{If the performance does not increase over earlyStopRound number of trees then training stops (this prevents overfitting)}
+
+\item{numIterations}{Number of boosting iterations.}
+
+\item{numLeaves}{This hyperparameter sets the maximum number of leaves. Increasing this parameter can lead to higher model complexity and potential overfitting.}
+
+\item{maxDepth}{This hyperparameter sets the maximum depth . Increasing this parameter can also lead to higher model complexity and potential overfitting.}
+
+\item{minDataInLeaf}{This hyperparameter sets the minimum number of data points that must be present in a leaf node. Increasing this parameter can help to reduce overfitting}
+
+\item{learningRate}{This hyperparameter controls the step size at each iteration of the gradient descent algorithm. Lower values can lead to slower convergence but may result in better performance.}
+
+\item{lambdaL1}{This hyperparameter controls L1 regularization, which can help to reduce overfitting by encouraging sparse models.}
+
+\item{lambdaL2}{This hyperparameter controls L2 regularization, which can also help to reduce overfitting by discouraging large weights in the model.}
+
+\item{scalePosWeight}{Controls weight of positive class in loss - useful for imbalanced classes}
+
+\item{isUnbalance}{This parameter cannot be used at the same time with scalePosWeight, choose only one of them. While enabling this should increase the overall performance metric of your model, it will also result in poor estimates of the individual class probabilities.}
+
+\item{seed}{An option to add a seed when training the final model}
+}
+\description{
+Create setting for gradient boosting machine model using lightGBM (https://github.com/microsoft/LightGBM/tree/master/R-package).
+}
+\examples{
+model.lightgbm <- setLightGBM(
+    numLeaves = c(20, 31, 50), maxDepth = c(-1, 5, 10),
+    minDataInLeaf = c(10, 20, 30), learningRate = c(0.05, 0.1, 0.3)
+)
+
+}
diff --git a/man/setMLP.Rd b/man/setMLP.Rd
index 0a70acd25..d6dea6254 100644
--- a/man/setMLP.Rd
+++ b/man/setMLP.Rd
@@ -5,7 +5,7 @@
 \title{Create setting for neural network model with python}
 \usage{
 setMLP(
-  hiddenLayerSizes = list(c(100), c(20, 4)),
+  hiddenLayerSizes = list(c(100), c(20)),
   activation = list("relu"),
   solver = list("adam"),
   alpha = list(0.3, 0.01, 1e-04, 1e-06),
@@ -23,7 +23,7 @@ setMLP(
   validationFraction = list(0.1),
   beta1 = list(0.9),
   beta2 = list(0.999),
-  epsilon = list(1, 0.1, 1e-08),
+  epsilon = list(1e-08),
   nIterNoChange = list(10),
   seed = sample(1e+05, 1)
 )
@@ -33,10 +33,10 @@ setMLP(
 
 \item{activation}{(list) Activation function for the hidden layer.
 \itemize{
-   \item{"identity": no-op activation, useful to implement linear bottleneck, returns f(x) = x}
-   \item{"logistic": the logistic sigmoid function, returns f(x) = 1 / (1 + exp(-x)).}
-   \item{"tanh": the hyperbolic tan function, returns f(x) = tanh(x).}
-   \item{"relu": the rectified linear unit function, returns f(x) = max(0, x)}
+   \item "identity": no-op activation, useful to implement linear bottleneck, returns f(x) = x
+   \item "logistic": the logistic sigmoid function, returns f(x) = 1 / (1 + exp(-x)).
+   \item "tanh": the hyperbolic tan function, returns f(x) = tanh(x).
+   \item "relu": the rectified linear unit function, returns f(x) = max(0, x)
  }}
 
 \item{solver}{(list) The solver for weight optimization. (‘lbfgs’, ‘sgd’, ‘adam’)}
@@ -45,7 +45,7 @@ setMLP(
 
 \item{batchSize}{(list) Size of minibatches for stochastic optimizers. If the solver is ‘lbfgs’, the classifier will not use minibatch. When set to “auto”, batchSize=min(200, n_samples).}
 
-\item{learningRate}{(list) Only used when solver='sgd' Learning rate schedule for weight updates.{‘constant’, ‘invscaling’, ‘adaptive’}, default=’constant’}
+\item{learningRate}{(list) Only used when solver='sgd' Learning rate schedule for weight updates. ‘constant’, ‘invscaling’, ‘adaptive’, default=’constant’}
 
 \item{learningRateInit}{(list) Only used when solver=’sgd’ or ‘adam’. The initial learning rate used. It controls the step-size in updating the weights.}
 
diff --git a/man/setRandomForest.Rd b/man/setRandomForest.Rd
index 2fb1b7b37..360b532ae 100644
--- a/man/setRandomForest.Rd
+++ b/man/setRandomForest.Rd
@@ -11,14 +11,14 @@ setRandomForest(
   minSamplesSplit = list(2, 5),
   minSamplesLeaf = list(1, 10),
   minWeightFractionLeaf = list(0),
-  mtries = list("auto", "log2"),
+  mtries = list("sqrt", "log2"),
   maxLeafNodes = list(NULL),
   minImpurityDecrease = list(0),
   bootstrap = list(TRUE),
   maxSamples = list(NULL, 0.9),
   oobScore = list(FALSE),
   nJobs = list(NULL),
-  classWeight = list("balanced_subsample", NULL),
+  classWeight = list(NULL),
   seed = sample(1e+05, 1)
 )
 }
@@ -35,14 +35,13 @@ setRandomForest(
 
 \item{minWeightFractionLeaf}{(list) The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sampleWeight is not provided.}
 
-\item{mtries}{(list) The number of features to consider when looking for the best split: 
+\item{mtries}{(list) The number of features to consider when looking for the best split:
 \itemize{
-\item{int}{then consider max_features features at each split.}
-\item{float}{then max_features is a fraction and round(max_features * n_features) features are considered at each split}
-\item{'auto'}{then max_features=sqrt(n_features)}
-\item{'sqrt'}{then max_features=sqrt(n_features) (same as “auto”)}
-\item{'log2'}{then max_features=log2(n_features).}
-\item{NULL}{then max_features=n_features}
+\item int then consider max_features features at each split.
+\item float then max_features is a fraction and round(max_features * n_features) features are considered at each split
+\item 'sqrt' then max_features=sqrt(n_features)
+\item 'log2' then max_features=log2(n_features)
+\item NULL then max_features=n_features
 }}
 
 \item{maxLeafNodes}{(list) Grow trees with max_leaf_nodes in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes.}
@@ -57,7 +56,7 @@ setRandomForest(
 
 \item{nJobs}{The number of jobs to run in parallel.}
 
-\item{classWeight}{(list) Weights associated with classes. If not given, all classes are supposed to have weight one. {NULL, “balanced”, “balanced_subsample”}}
+\item{classWeight}{(list) Weights associated with classes. If not given, all classes are supposed to have weight one. NULL, “balanced”, “balanced_subsample”}
 
 \item{seed}{A seed when training the final model}
 }
@@ -66,7 +65,7 @@ Create setting for random forest model with python (very fast)
 }
 \examples{
 \dontrun{
-model.rf <- setRandomForest(mtries=list('auto',5,20),  ntrees=c(10,100), 
+model.rf <- setRandomForest(mtries=list('auto',5,20),  ntrees=c(10,100),
                            maxDepth=c(5,20))
-}                           
+}
 }
diff --git a/man/setSVM.Rd b/man/setSVM.Rd
index 4edccc620..2def0720b 100644
--- a/man/setSVM.Rd
+++ b/man/setSVM.Rd
@@ -12,7 +12,7 @@ setSVM(
   coef0 = list(0),
   shrinking = list(TRUE),
   tol = list(0.001),
-  classWeight = list("balanced", NULL),
+  classWeight = list(NULL),
   cacheSize = 500,
   seed = sample(1e+05, 1)
 )
@@ -24,7 +24,7 @@ setSVM(
 
 \item{degree}{(list) degree of kernel function is significant only in poly, rbf, sigmoid}
 
-\item{gamma}{(list) kernel coefficient for rbf and poly, by default 1/n_features will be taken. {‘scale’, ‘auto’} or float, default=’scale’}
+\item{gamma}{(list) kernel coefficient for rbf and poly, by default 1/n_features will be taken. ‘scale’, ‘auto’ or float, default=’scale’}
 
 \item{coef0}{(list) independent term in kernel function. It is only significant in poly/sigmoid.}
 
diff --git a/man/sklearnFromJson.Rd b/man/sklearnFromJson.Rd
new file mode 100644
index 000000000..402bf1b01
--- /dev/null
+++ b/man/sklearnFromJson.Rd
@@ -0,0 +1,14 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/SklearnToJson.R
+\name{sklearnFromJson}
+\alias{sklearnFromJson}
+\title{Loads sklearn python model from json}
+\usage{
+sklearnFromJson(path)
+}
+\arguments{
+\item{path}{path to the model json file}
+}
+\description{
+Loads sklearn python model from json
+}
diff --git a/man/sklearnToJson.Rd b/man/sklearnToJson.Rd
new file mode 100644
index 000000000..ce5d0b2d0
--- /dev/null
+++ b/man/sklearnToJson.Rd
@@ -0,0 +1,16 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/SklearnToJson.R
+\name{sklearnToJson}
+\alias{sklearnToJson}
+\title{Saves sklearn python model object to json in path}
+\usage{
+sklearnToJson(model, path)
+}
+\arguments{
+\item{model}{a fitted sklearn python model object}
+
+\item{path}{path to the saved model file}
+}
+\description{
+Saves sklearn python model object to json in path
+}
diff --git a/man/splitData.Rd b/man/splitData.Rd
index 065099117..4c03f61b8 100644
--- a/man/splitData.Rd
+++ b/man/splitData.Rd
@@ -26,14 +26,14 @@ Split the plpData into test/train sets using a splitting settings of class \code
 }
 \details{
 Returns a list containing the training data (Train) and optionally the test data (Test).  Train is an Andromeda object containing
-\itemize{\item{covariates}{ a table (rowId, covariateId, covariateValue) containing the covariates for each data point in the train data }
-         \item{covariateRef}{ a table with the covariate information}
-         \item{labels)}{ a table (rowId, outcomeCount, ...) for each data point in the train data (outcomeCount is the class label) }
-         \item{folds}{ a table (rowId, index) specifying which training fold each data point is in.}
+\itemize{\item covariates: a table (rowId, covariateId, covariateValue) containing the covariates for each data point in the train data
+         \item covariateRef: a table with the covariate information
+         \item labels: a table (rowId, outcomeCount, ...) for each data point in the train data (outcomeCount is the class label) 
+         \item folds: a table (rowId, index) specifying which training fold each data point is in.
          } 
 Test is an Andromeda object containing
-\itemize{\item{covariates}{ a table (rowId, covariateId, covariateValue) containing the covariates for each data point in the test data }
-         \item{covariateRef}{ a table with the covariate information}
-         \item{labels)}{ a table (rowId, outcomeCount, ...) for each data point in the test data (outcomeCount is the class label) }
+\itemize{\item covariates: a table (rowId, covariateId, covariateValue) containing the covariates for each data point in the test data 
+         \item covariateRef: a table with the covariate information
+         \item labels: a table (rowId, outcomeCount, ...) for each data point in the test data (outcomeCount is the class label) 
          }
 }
diff --git a/man/validateExternal.Rd b/man/validateExternal.Rd
new file mode 100644
index 000000000..248dd8459
--- /dev/null
+++ b/man/validateExternal.Rd
@@ -0,0 +1,28 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/ExternalValidatePlp.R
+\name{validateExternal}
+\alias{validateExternal}
+\title{externalValidatePlp - Validate model performance on new data}
+\usage{
+validateExternal(
+  validationDesignList,
+  databaseDetails,
+  logSettings,
+  outputFolder
+)
+}
+\arguments{
+\item{validationDesignList}{A list of objects created with \code{createValidationDesign}}
+
+\item{databaseDetails}{A list of objects of class
+\code{databaseDetails} created using \code{createDatabaseDetails}}
+
+\item{logSettings}{An object of \code{logSettings} created
+using \code{createLogSettings}}
+
+\item{outputFolder}{The directory to save the validation results to
+(subfolders are created per database in validationDatabaseDetails)}
+}
+\description{
+externalValidatePlp - Validate model performance on new data
+}
diff --git a/man/validateMultiplePlp.Rd b/man/validateMultiplePlp.Rd
index ed14fc5c8..2ab5bc903 100644
--- a/man/validateMultiplePlp.Rd
+++ b/man/validateMultiplePlp.Rd
@@ -9,18 +9,21 @@ validateMultiplePlp(
   validationDatabaseDetails,
   validationRestrictPlpDataSettings = createRestrictPlpDataSettings(),
   recalibrate = NULL,
+  cohortDefinitions = NULL,
   saveDirectory = NULL
 )
 }
 \arguments{
 \item{analysesLocation}{The location where the multiple plp analyses are}
 
-\item{validationDatabaseDetails}{The validation database settings created using \code{createDatabaseDetails()}}
+\item{validationDatabaseDetails}{A single or list of validation database settings created using \code{createDatabaseDetails()}}
 
 \item{validationRestrictPlpDataSettings}{The settings specifying the extra restriction settings when extracting the data created using \code{createRestrictPlpDataSettings()}.}
 
 \item{recalibrate}{A vector of recalibration methods (currently supports 'RecalibrationintheLarge' and/or 'weakRecalibration')}
 
+\item{cohortDefinitions}{A list of cohortDefinitions}
+
 \item{saveDirectory}{The location to save to validation results}
 }
 \description{
diff --git a/man/viewPlp.Rd b/man/viewPlp.Rd
index 5872efaa7..38ca332eb 100644
--- a/man/viewPlp.Rd
+++ b/man/viewPlp.Rd
@@ -4,12 +4,14 @@
 \alias{viewPlp}
 \title{viewPlp - Interactively view the performance and model settings}
 \usage{
-viewPlp(runPlp, validatePlp = NULL)
+viewPlp(runPlp, validatePlp = NULL, diagnosePlp = NULL)
 }
 \arguments{
 \item{runPlp}{The output of runPlp() (an object of class 'runPlp')}
 
 \item{validatePlp}{The output of externalValidatePlp (on object of class 'validatePlp')}
+
+\item{diagnosePlp}{The output of diagnosePlp()}
 }
 \value{
 Opens a shiny app for interactively viewing the results
diff --git a/tests/testthat/helper-expectations.R b/tests/testthat/helper-expectations.R
new file mode 100644
index 000000000..25210fdf6
--- /dev/null
+++ b/tests/testthat/helper-expectations.R
@@ -0,0 +1,29 @@
+# common tests that can be grouped together, such as testing the output from fitplp
+
+expect_correct_fitPlp <- function(plpModel, trainData) {
+  
+  # predictions are same amount as labels
+  multiplicativeFactor <- dplyr::n_distinct(plpModel$prediction %>% dplyr::pull(evaluationType))
+  expect_equal(NROW(trainData$labels)*multiplicativeFactor, NROW(plpModel$prediction))
+  
+  # predictions are all between 0 and 1
+  expect_true(all((plpModel$prediction$value >= 0) & 
+                    (plpModel$prediction$value <= 1)))
+
+  # model directory exists
+  expect_true(dir.exists(plpModel$model))
+  
+  expect_equal(plpModel$modelDesign$outcomeId, outcomeId)
+  expect_equal(plpModel$modelDesign$targetId, 1)
+  
+  # structure of plpModel is correct
+  expect_equal(names(plpModel), c("model", "preprocessing", "prediction",
+                                  "modelDesign", "trainDetails", "covariateImportance"))
+}
+
+expect_correct_predictions <- function(predictions, testData) {
+  
+  # predictions are all between 0 and 1
+  expect_true(all((predictions$value >= 0) & (predictions$value <= 1)))
+  expect_equal(NROW(testData$labels), NROW(predictions))
+}
diff --git a/tests/testthat/helper-functions.R b/tests/testthat/helper-functions.R
new file mode 100644
index 000000000..3a44091e8
--- /dev/null
+++ b/tests/testthat/helper-functions.R
@@ -0,0 +1,34 @@
+# helper functions for tests
+
+# copies trainData and makes sure andromeda object is copied correctly 
+copyTrainData <- function(trainData) {
+  newTrainData <- trainData
+  
+  # force andromeda to copy
+  newTrainData$covariateData <- Andromeda::copyAndromeda(trainData$covariateData)
+
+  class(newTrainData$covariateData) <- class(trainData$covariateData)
+  return(newTrainData)
+}
+
+# create tiny dataset with subset of covariates based on lasso fit
+createTinyPlpData <- function(plpData, plpResult, n= 20) {
+  
+  covariates <- plpResult$model$covariateImportance %>% 
+    dplyr::slice_max(order_by = abs(.data$covariateValue),
+                     n = n, with_ties = F) %>% 
+    dplyr::pull(.data$covariateId)
+  tinyPlpData <- plpData
+  tinyPlpData$covariateData <- Andromeda::copyAndromeda(plpData$covariateData)
+  
+  tinyPlpData$covariateData$covariates <- plpData$covariateData$covariates %>% 
+    dplyr::filter(covariateId %in% covariates)
+  tinyPlpData$covariateData$covariateRef <- plpData$covariateData$covariateRef %>% 
+    dplyr::filter(covariateId %in% covariates)
+  
+  attributes(tinyPlpData$covariateData)$metaData <- attributes(plpData$covariateData)$metaData
+  class(tinyPlpData$covariateData) <- class(plpData$covariateData)
+  attributes(tinyPlpData)$metaData <- attributes(plpData)$metaData
+  class(tinyPlpData) <- class(plpData)
+  return(tinyPlpData)
+}
\ No newline at end of file
diff --git a/tests/testthat/helper-objects.R b/tests/testthat/helper-objects.R
index 51fa98fdf..fccf9bf1d 100644
--- a/tests/testthat/helper-objects.R
+++ b/tests/testthat/helper-objects.R
@@ -1,50 +1,69 @@
 
-# Download the PostreSQL driver ---------------------------
-# If DATABASECONNECTOR_JAR_FOLDER exists, assume driver has been downloaded
-jarFolder <- Sys.getenv("DATABASECONNECTOR_JAR_FOLDER", unset = "")
-if (jarFolder == "") {
-  tempJarFolder <- tempfile("jdbcDrivers")
-  dir.create(tempJarFolder)
-  Sys.setenv("DATABASECONNECTOR_JAR_FOLDER" = tempJarFolder)
-  downloadJdbcDrivers("postgresql")
-  
-  withr::defer({
-    unlink(tempJarFolder, recursive = TRUE, force = TRUE)
-    Sys.unsetenv("DATABASECONNECTOR_JAR_FOLDER")
-  }, testthat::teardown_env())
-}
-
-
 # this files contains the objects used in the tests:
-travis <- T
-
-saveLoc <- tempfile("saveLoc")
-dir.create(saveLoc)
-
-
-if(ifelse(is.null(Sys.info()), T, Sys.info()['sysname'] != 'Windows')){
+if (Sys.getenv('GITHUB_ACTIONS') == 'true') {
+  # Download the PostreSQL driver ---------------------------
+  # If DATABASECONNECTOR_JAR_FOLDER exists, assume driver has been downloaded
+  jarFolder <- Sys.getenv("DATABASECONNECTOR_JAR_FOLDER", unset = "")
+  if (jarFolder == "") {
+    tempJarFolder <- tempfile("jdbcDrivers")
+    dir.create(tempJarFolder)
+    Sys.setenv("DATABASECONNECTOR_JAR_FOLDER" = tempJarFolder)
+    DatabaseConnector::downloadJdbcDrivers("postgresql")
+    
+    withr::defer({
+      unlink(tempJarFolder, recursive = TRUE, force = TRUE)
+      Sys.unsetenv("DATABASECONNECTOR_JAR_FOLDER")
+    }, testthat::teardown_env())
+  }
+  
   # configure and activate python
   PatientLevelPrediction::configurePython(envname = 'r-reticulate', envtype = "conda")
   PatientLevelPrediction::setPythonEnvironment(envname = 'r-reticulate', envtype = "conda")
   
   # if mac install nomkl -- trying to fix github actions
-  if(ifelse(is.null(Sys.info()), F, Sys.info()['sysname'] == 'Darwin')){
+  if (ifelse(is.null(Sys.info()), F, Sys.info()['sysname'] == 'Darwin')){
     reticulate::conda_install(envname = 'r-reticulate', packages = c('nomkl'), 
                               forge = TRUE, pip = FALSE, pip_ignore_installed = TRUE, 
                               conda = "auto")
   }
 }
 
+
+saveLoc <- tempfile("saveLoc")
+dir.create(saveLoc)
+
+
 #@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
 # simulated data Tests
 #@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
-
-data(plpDataSimulationProfile, envir = environment())
+data("plpDataSimulationProfile")
 
 # PLPDATA
-sampleSize <- 2500+sample(1000,1)
-plpData <- simulatePlpData(plpDataSimulationProfile, n = sampleSize)
-#plpData$metaData$cohortId <- plpData$metaData$cohortIds
+connectionDetails <- Eunomia::getEunomiaConnectionDetails()
+Eunomia::createCohorts(connectionDetails)
+outcomeId <- 3 # GIbleed
+
+databaseDetails <- createDatabaseDetails(
+  connectionDetails = connectionDetails, 
+  cdmDatabaseSchema = "main", 
+  cdmDatabaseName = "main",
+  cohortDatabaseSchema = "main", 
+  cohortTable = "cohort", 
+  outcomeDatabaseSchema = "main", 
+  outcomeTable =  "cohort",
+  targetId = 1, 
+  outcomeIds = outcomeId,
+  cdmVersion = 5)
+
+covariateSettings <- FeatureExtraction::createCovariateSettings(
+  useDemographicsAge = TRUE,
+  useDemographicsGender = TRUE,
+  useConditionOccurrenceAnyTimePrior = TRUE
+)
+
+plpData <- getPlpData(databaseDetails = databaseDetails,
+                      covariateSettings = covariateSettings,
+                      restrictPlpDataSettings = createRestrictPlpDataSettings()) 
 
 # POPULATION
 populationSettings <- createStudyPopulationSettings(
@@ -53,7 +72,7 @@ populationSettings <- createStudyPopulationSettings(
   removeSubjectsWithPriorOutcome = FALSE,
   priorOutcomeLookback = 99999,
   requireTimeAtRisk = T,
-  minTimeAtRisk=10,
+  minTimeAtRisk = 10,
   riskWindowStart = 0,
   startAnchor = 'cohort start',
   riskWindowEnd = 365,
@@ -62,66 +81,111 @@ populationSettings <- createStudyPopulationSettings(
 
 
 # MODEL SETTINGS
-lrSet <- setLassoLogisticRegression()
+lrSet <- setLassoLogisticRegression(seed = 42)
 
 # RUNPLP - LASSO LR
 plpResult <- runPlp(
   plpData = plpData, 
-  outcomeId = 2, 
+  outcomeId = outcomeId, 
   analysisId = 'Test', 
   analysisName = 'Testing analysis',
   populationSettings = populationSettings, 
-  splitSettings = createDefaultSplitSetting(),
+  splitSettings = createDefaultSplitSetting(splitSeed = 12),
   preprocessSettings = createPreprocessSettings(), 
   modelSettings = lrSet, 
-  logSettings = createLogSettings(verbosity = 'TRACE'),
+  logSettings = createLogSettings(verbosity = 'ERROR'),
   executeSettings = createDefaultExecuteSettings(), 
   saveDirectory = saveLoc
   )
 
+
+# now diagnose
+diagnoseResult <- diagnosePlp(
+  plpData = plpData,
+  outcomeId = outcomeId,
+  analysisId = 'Test', 
+  populationSettings = populationSettings,
+  splitSettings = createDefaultSplitSetting(splitSeed = 12),
+  saveDirectory = saveLoc,
+  modelSettings = lrSet,
+  logSettings =  createLogSettings(
+    verbosity = 'DEBUG',
+    timeStamp = T,
+    logName = 'diagnosePlp Log'
+  ),
+  preprocessSettings = createPreprocessSettings(), 
+  sampleSettings = NULL, 
+  featureEngineeringSettings = NULL
+)
+
 #
 
 
 population <- createStudyPopulation(
   plpData = plpData,
-  outcomeId = 2, 
+  outcomeId = outcomeId, 
   populationSettings = populationSettings
   )
 
 createTrainData <- function(plpData, population){
-  trainData <- list()
-  trainData$covariateData <- Andromeda::copyAndromeda(plpData$covariateData)
-  attr(trainData$covariateData, "metaData") <- attr(plpData$covariateData, "metaData")
-  trainData$labels <- population
-  trainData$folds <- data.frame(
-    rowId = population$rowId,
-    index = sample(3, nrow(population), replace = T)
-  )
-  
-  attr(trainData, "metaData")$outcomeId <- 2
-  attr(trainData, "metaData")$cohortId <- 1
-
-  class(trainData$covariateData) <- 'CovariateData'
-  
+  data <- PatientLevelPrediction::splitData(plpData = plpData, 
+                                            population = population,
+                                            splitSettings = PatientLevelPrediction::createDefaultSplitSetting(splitSeed = 12))
+  trainData <- data$Train
   return(trainData)
 }
 
+trainData <- createTrainData(plpData, population)
 
-sampleSize2 <- 1000+sample(1000,1)
-plpData2 <- simulatePlpData(plpDataSimulationProfile, n = sampleSize2)
-
-population2 <- createStudyPopulation(
-  plpData = plpData2,
-  outcomeId = 2, 
-  populationSettings = populationSettings
-)
+createTestData <- function(plpData, population){
+  data <- PatientLevelPrediction::splitData(plpData = plpData,
+                                            population = population,
+                                            splitSettings = PatientLevelPrediction::createDefaultSplitSetting(splitSeed = 12))
+  testData <- data$Test
+  return(testData)
+}
 
-sampleSizeBig <- 10000
-plpDataBig <- simulatePlpData(plpDataSimulationProfile, n = sampleSizeBig)
+testData <- createTestData(plpData, population)
 
-populationBig <- createStudyPopulation(
-  plpData = plpDataBig,
-  outcomeId = 2, 
-  populationSettings = populationSettings
-)
+# reduced trainData to only use n most important features (as decided by LR)
+reduceTrainData <- function(trainData, n=20) {
+  covariates <- plpResult$model$covariateImportance %>% 
+    dplyr::slice_max(order_by = abs(.data$covariateValue),n = n, with_ties = F) %>% 
+    dplyr::pull(.data$covariateId)
+  
+  reducedTrainData <- list(labels = trainData$labels,
+                           folds = trainData$folds,
+                           covariateData = Andromeda::andromeda(
+                             analysisRef = trainData$covariateData$analysisRef
+                           ))
+  
+  
+  reducedTrainData$covariateData$covariates <- trainData$covariateData$covariates %>% 
+    dplyr::filter(covariateId %in% covariates)
+  reducedTrainData$covariateData$covariateRef <- trainData$covariateData$covariateRef %>% 
+    dplyr::filter(covariateId %in% covariates)
+  
+  attributes(reducedTrainData$covariateData)$metaData <- attributes(trainData$covariateData)$metaData
+  class(reducedTrainData$covariateData) <- class(trainData$covariateData)
+  attributes(reducedTrainData)$metaData <- attributes(trainData)$metaData
+  return(reducedTrainData)  
+}
 
+tinyTrainData <- reduceTrainData(trainData)
+
+tinyPlpData <- createTinyPlpData(plpData, plpResult)
+
+nanoData <- createTinyPlpData(plpData, plpResult, n = 2)
+tinyResults <- runPlp(plpData = nanoData,
+                      populationSettings = populationSettings,
+                      outcomeId = outcomeId,
+                      analysisId = 'tinyFit',
+                      executeSettings = createExecuteSettings(
+                        runSplitData = T,
+                        runSampleData = F,
+                        runfeatureEngineering = F,
+                        runPreprocessData = T,
+                        runModelDevelopment = T,
+                        runCovariateSummary = F
+                      ),
+                      saveDirectory = file.path(saveLoc, 'tinyResults'))
diff --git a/tests/testthat/test-KNN.R b/tests/testthat/test-KNN.R
new file mode 100644
index 000000000..23cc8486b
--- /dev/null
+++ b/tests/testthat/test-KNN.R
@@ -0,0 +1,30 @@
+
+
+test_that('KNN fit works', {
+  skip_on_ci()
+  modelSettings = setKNN(k = 2)
+  nanoTrainData <- reduceTrainData(tinyTrainData, n = 2)
+  subjectToKeep <- nanoTrainData$labels[sample.int(nrow(nanoTrainData$labels), 50),"rowId"]
+  nanoTrainData$labels <- nanoTrainData$labels[nanoTrainData$labels$rowId %in% subjectToKeep,]
+  nanoTrainData$folds <- nanoTrainData$folds[nanoTrainData$folds$rowId %in% subjectToKeep,]
+  nanoTrainData$covariateData$covariates <- nanoTrainData$covariateData$covariates %>% dplyr::filter(.data$rowId %in% subjectToKeep)
+  plpModel <- fitPlp(
+    trainData = nanoTrainData, 
+    modelSettings = modelSettings,
+    analysisId = 'KNN',
+    analysisPath = tempdir()
+  )
+  
+  expect_correct_fitPlp(plpModel, nanoTrainData) 
+  
+})
+
+
+test_that("KNN settings", {
+skip_on_ci()
+model_set <- setKNN(k=5)
+testthat::expect_is(model_set, "modelSettings")
+testthat::expect_length(model_set,2)
+testthat::expect_error(setKNN(k = 0))
+testthat::expect_error(setKNN(indexFolder = 2372))
+})
diff --git a/tests/testthat/test-KNN_updated.R b/tests/testthat/test-KNN_updated.R
deleted file mode 100644
index 20bc9bda3..000000000
--- a/tests/testthat/test-KNN_updated.R
+++ /dev/null
@@ -1,44 +0,0 @@
-
-resultNames <- c('executionSummary','model','prediction', 'performanceEvaluation', 'covariateSummary', 'analysisRef')
-
-plpResultKNN <- runPlp(
-  plpData = plpData, 
-  outcomeId = 2, 
-  analysisId = 'knnTest', 
-  analysisName = 'Testing knn',
-  populationSettings = populationSettings, 
-  splitSettings = createDefaultSplitSetting(),
-  preprocessSettings = createPreprocessSettings(), 
-  modelSettings = setKNN(k=10), 
-  logSettings = createLogSettings(verbosity = 'TRACE'),
-  executeSettings = createDefaultExecuteSettings(), 
-  saveDirectory = file.path(saveLoc, 'knn')
-  )
-
-test_that("covRef is correct size", {
-  
-  testthat::expect_true(nrow(as.data.frame(plpData$covariateData$covariateRef)) >=  
-    nrow(plpResultKNN$model$covariateImportance))
-  
-})
-
-
-test_that("KNN results have correct structure", {
-  
-  
-  # same output names for LR, KNN and GBM
-  testthat::expect_equal(
-    names(plpResultKNN),
-    resultNames
-    )
-  
-})
-
-test_that("KNN settings", {
-
-model_set <- setKNN(k=5)
-testthat::expect_is(model_set, "modelSettings")
-testthat::expect_length(model_set,2)
-testthat::expect_error(setKNN(k = 0))
-testthat::expect_error(setKNN(indexFolder = 2372))
-})
diff --git a/tests/testthat/test-LightGBM.R b/tests/testthat/test-LightGBM.R
new file mode 100644
index 000000000..35e742cfb
--- /dev/null
+++ b/tests/testthat/test-LightGBM.R
@@ -0,0 +1,121 @@
+# Copyright 2023 Observational Health Data Sciences and Informatics
+#
+# This file is part of PatientLevelPrediction
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+library("testthat")
+
+context("LightGBM")
+
+
+test_that("LightGBM settings work", {
+  
+  seed <- sample(10000000,1)
+  #=====================================
+  # checking Light GBM
+  #=====================================
+  lgbmSet <- setLightGBM(
+    nthread = 5, 
+    earlyStopRound = 25,
+    numIterations = 10,
+    numLeaves = c(31, 20),
+    maxDepth = 5, 
+    minDataInLeaf = 10, 
+    learningRate = 0.1,
+    lambdaL1 = 0,
+    lambdaL2 =0,
+    scalePosWeight = 1,
+    isUnbalance = F,
+    seed = seed
+  )
+  
+  expect_is(lgbmSet, 'modelSettings')
+  expect_equal(lgbmSet$fitFunction, 'fitRclassifier')
+  expect_is(lgbmSet$param, 'list')
+  
+  expect_equal(attr(lgbmSet$param, 'settings')$modelType, 'LightGBM')
+  expect_equal(attr(lgbmSet$param, 'settings')$seed, seed)
+  expect_equal(attr(lgbmSet$param, 'settings')$modelName, "LightGBM")
+  
+  expect_equal(attr(lgbmSet$param, 'settings')$threads, 5)
+  expect_equal(attr(lgbmSet$param, 'settings')$varImpRFunction, 'varImpLightGBM')
+  expect_equal(attr(lgbmSet$param, 'settings')$trainRFunction, 'fitLightGBM')
+  expect_equal(attr(lgbmSet$param, 'settings')$predictRFunction, 'predictLightGBM')
+  
+  expect_equal(length(lgbmSet$param),2)
+  
+  expect_equal(length(unique(unlist(lapply(lgbmSet$param, function(x) x$numIterations)))), 1)
+  expect_equal(length(unique(unlist(lapply(lgbmSet$param, function(x) x$numLeaves)))), 2)
+  expect_equal(length(unique(unlist(lapply(lgbmSet$param, function(x) x$earlyStopRound)))), 1)
+  expect_equal(length(unique(unlist(lapply(lgbmSet$param, function(x) x$maxDepth)))), 1)
+  expect_equal(length(unique(unlist(lapply(lgbmSet$param, function(x) x$minDataInLeaf)))), 1)
+  expect_equal(length(unique(unlist(lapply(lgbmSet$param, function(x) x$learningRate)))), 1)
+  expect_equal(length(unique(unlist(lapply(lgbmSet$param, function(x) x$lambdaL1)))), 1)
+  expect_equal(length(unique(unlist(lapply(lgbmSet$param, function(x) x$lambdaL2)))), 1)
+  expect_equal(length(unique(unlist(lapply(lgbmSet$param, function(x) x$scalePosWeight)))), 1)
+  expect_equal(length(unique(unlist(lapply(lgbmSet$param, function(x) x$isUnbalance)))), 1)
+  
+})
+
+
+test_that("LightGBM settings expected errors", {
+  #=====================================
+  # checking Gradient Boosting Machine
+  #=====================================
+  
+  testthat::expect_error(setLightGBM(numIterations = -1))
+  testthat::expect_error(setLightGBM(numLeaves = -1))
+  testthat::expect_error(setLightGBM(numLeaves = 10000000))
+  testthat::expect_error(setLightGBM(learningRate = -2))
+  testthat::expect_error(setLightGBM(seed = 'F'))
+  testthat::expect_error(setLightGBM(lambdaL1 = -1))
+  testthat::expect_error(setLightGBM(lambdaL2 = -1))
+  testthat::expect_error(setLightGBM(scalePosWeight = -1))
+  testthat::expect_error(setLightGBM(isUnbalance = TRUE, scalePosWeight = 0.5))
+  
+})
+
+
+
+
+test_that("LightGBM working checks", {
+  
+  modelSettings <- setLightGBM(numIterations = 10, maxDepth = 3, learningRate = 0.1, numLeaves = 31, minDataInLeaf = 10, lambdaL1 = 0, lambdaL2 = 0)
+  
+  fitModel <- fitPlp(
+    trainData = trainData,   
+    modelSettings = modelSettings, 
+    analysisId = 'lgbmTest',
+    analysisPath = tempdir()
+  )
+  
+  expect_equal(nrow(fitModel$prediction), nrow(trainData$labels)*2)
+  expect_equal(length(unique(fitModel$prediction$evaluationType)),2)
+  
+  # check prediction between 0 and 1
+  expect_gte(min(fitModel$prediction$value), 0)
+  expect_lte(max(fitModel$prediction$value), 1)
+  
+  expect_equal(class(fitModel$model), c("lgb.Booster", "R6"))
+  
+  expect_lte(nrow(fitModel$covariateImportance), trainData$covariateData$covariateRef %>% dplyr::tally() %>% dplyr::pull())
+  
+  expect_equal(fitModel$modelDesign$outcomeId, outcomeId)
+  expect_equal(fitModel$modelDesign$targetId, 1)
+  # TODO check other model design values?
+  
+  # test that at least some features have importances that are not zero
+  expect_equal(sum(abs(fitModel$covariateImportance$covariateValue))>0, TRUE)
+  
+})
diff --git a/tests/testthat/test-PredictionDistribution_updated.R b/tests/testthat/test-PredictionDistribution.R
similarity index 100%
rename from tests/testthat/test-PredictionDistribution_updated.R
rename to tests/testthat/test-PredictionDistribution.R
diff --git a/tests/testthat/test-ThresholdSummary_updated.R b/tests/testthat/test-ThresholdSummary.R
similarity index 100%
rename from tests/testthat/test-ThresholdSummary_updated.R
rename to tests/testthat/test-ThresholdSummary.R
diff --git a/tests/testthat/test-UploadPlpDbResults_upadted.R b/tests/testthat/test-UploadPlpDbResults_upadted.R
deleted file mode 100644
index f1ed72629..000000000
--- a/tests/testthat/test-UploadPlpDbResults_upadted.R
+++ /dev/null
@@ -1,130 +0,0 @@
-# Copyright 2021 Observational Health Data Sciences and Informatics
-#
-# This file is part of PatientLevelPrediction
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-library("testthat")
-
-context("UploadPlpDbResults")
-
-cdmDatabaseSchema <- Sys.getenv("CDM5_POSTGRESQL_CDM_SCHEMA")
-ohdsiDatabaseSchema <- Sys.getenv("CDM5_POSTGRESQL_OHDSI_SCHEMA")
-connectionRedshift <- DatabaseConnector::createConnectionDetails(
-  dbms = "postgresql",
-  user = Sys.getenv("CDM5_POSTGRESQL_USER"),
-  password = URLdecode(Sys.getenv("CDM5_POSTGRESQL_PASSWORD")),
-  server = Sys.getenv("CDM5_POSTGRESQL_SERVER")
-  )
-conn <- DatabaseConnector::connect(connectionRedshift)
-
-
-randVar <- rawToChar(as.raw(sample(c(65:90,97:122), 5, replace=T)))
-appendRandom <- function(x, rand = randVar){
-  return(paste(rand, x, sep=''))
-}
-
-
-test_that("database creation", {
-
-  createPlpResultTables(conn = conn, 
-                        resultSchema = ohdsiDatabaseSchema, 
-                        targetDialect = 'postgresql',
-                        deleteExistingTables = T, 
-                        createTables = T,
-                        stringAppendToTables = appendRandom('test'))
-  
-  tableNames <- DatabaseConnector::getTableNames(connection = conn, databaseSchema = ohdsiDatabaseSchema)
-  # check the results table is created
-  testthat::expect_true(paste0(toupper(appendRandom('test')),'_RESULTS') %in% tableNames)
-
-})
-
-
-test_that("results uploaded to database", {
-  
-  resultsLoc <- file.path(saveLoc,'dbUp')
-  
-  savePlpResult(plpResult, file.path(resultsLoc, 'Analysis_1','plpResult'))
-  # save validation
-  if(!dir.exists(file.path(resultsLoc,'Validation','test', 'Analysis_1'))){
-    dir.create(file.path(resultsLoc,'Validation','test', 'Analysis_1'), recursive = T)
-  }
-  plpResult$model$validationDetails <- list(
-    cohortId = 1, 
-    outcomeId = 3,
-    populationSettings = plpResult$model$settings$populationSettings, 
-    plpDataSettings = plpResult$model$settings$plpDataSettings
-    )
-  savePlpResult(plpResult, file.path(resultsLoc,'Validation','test', 'Analysis_1', 'validationResult'))
-
-  # add results:
-  populatePlpResultTables(conn = conn, 
-                          resultSchema = ohdsiDatabaseSchema, 
-                          stringAppendToTables = appendRandom('test'),
-                          targetDialect = 'postgresql',
-                          studyJsonList = list(list(cohort_name = 'blank1', cohort_id = 1, cohort_json = 'bla'),
-                                               list(cohort_name = 'blank2', cohort_id = 2, cohort_json = 'bla'),
-                                               list(cohort_name = 'blank3', cohort_id = 3, cohort_json = 'bla')),
-                          studyName = 'test',
-                          studyDescription = 'testing',
-                          researcherName = 'jane doe',
-                          researcherEmail = 'none',
-                          researcherOrg = 'none',
-                          databaseName = 'test',
-                          databaseAcronym = 'test',
-                          databaseVersion = 1,
-                          databaseDescription = 'test',
-                          databaseType = 'claims',
-                          valDatabases = list(test = list(name = 'test', 
-                                                          description = 'test',
-                                                          version = 1,
-                                                          type = 'claims')),
-                          resultLocation = resultsLoc,
-                          resultPattern = 'Analysis',
-                          validationLocation = file.path(resultsLoc,'Validation'),
-                          addInternalValidation = T,
-                          addExternalValidation = T,
-                          gsubVal = NULL,
-                          removePattern = NULL
-  )
-  
-  
-  # check the results table is populated
-  sql <- 'select count(*) as N from @resultSchema.@appendresults;'
-  sql <- SqlRender::render(sql, resultSchema = ohdsiDatabaseSchema, append = appendRandom('test_'))
-  res <- DatabaseConnector::querySql(conn, sql)
-  testthat::expect_true(res$N[1]>0)
-
-  
-})
-
-
-test_that("database deletion", {
-
-  createPlpResultTables(conn = conn, 
-                        resultSchema = ohdsiDatabaseSchema, 
-                        targetDialect = 'postgresql',
-                        deleteExistingTables = T, 
-                        createTables = F,
-                        stringAppendToTables = appendRandom('test'))
-  
-  tableNames <- DatabaseConnector::getTableNames(connection = conn, databaseSchema = ohdsiDatabaseSchema)
-  # check the results table is then deleted
-  testthat::expect_false(paste0(toupper(appendRandom('test')),'_RESULTS') %in% tableNames)
-  
-  
-})
-
-# disconnect
-DatabaseConnector::disconnect(conn)
diff --git a/tests/testthat/test-UploadToDatabase.R b/tests/testthat/test-UploadToDatabase.R
new file mode 100644
index 000000000..908ab44d3
--- /dev/null
+++ b/tests/testthat/test-UploadToDatabase.R
@@ -0,0 +1,458 @@
+# Copyright 2021 Observational Health Data Sciences and Informatics
+#
+# This file is part of PatientLevelPrediction
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+library("testthat")
+
+context("UploadToDatabase")
+
+# only run this during CI
+if (Sys.getenv('CI') == 'true') {
+cdmDatabaseSchema <- Sys.getenv("CDM5_POSTGRESQL_CDM_SCHEMA")
+ohdsiDatabaseSchema <- Sys.getenv("CDM5_POSTGRESQL_OHDSI_SCHEMA")
+connectionRedshift <- DatabaseConnector::createConnectionDetails(
+  dbms = "postgresql",
+  user = Sys.getenv("CDM5_POSTGRESQL_USER"),
+  password = URLdecode(Sys.getenv("CDM5_POSTGRESQL_PASSWORD")),
+  server = Sys.getenv("CDM5_POSTGRESQL_SERVER"), 
+  )
+conn <- DatabaseConnector::connect(connectionRedshift)
+targetDialect <- 'postgresql'
+
+set.seed(NULL)
+randVar <- rawToChar(as.raw(sample(c(65:90,97:122), 5, replace=T)))
+
+appendRandom <- function(x, rand = randVar){
+  return(paste("plp", rand, x, sep=''))
+}
+
+}
+test_that("test createDatabaseSchemaSettings works", {
+  skip_if(Sys.getenv('CI') != 'true', 'not run locally')
+  databaseSchemaSettings <- createDatabaseSchemaSettings(
+    resultSchema = ohdsiDatabaseSchema, 
+    tablePrefix = '',
+    targetDialect = targetDialect
+  )
+  
+  # check inputs as expected
+  testthat::expect_true(databaseSchemaSettings$resultSchema == ohdsiDatabaseSchema)
+  testthat::expect_true(databaseSchemaSettings$tablePrefix == '')
+  testthat::expect_true(databaseSchemaSettings$targetDialect == targetDialect)
+  testthat::expect_true(databaseSchemaSettings$cohortDefinitionSchema == ohdsiDatabaseSchema)
+  testthat::expect_true(databaseSchemaSettings$databaseDefinitionSchema == ohdsiDatabaseSchema)
+  testthat::expect_true(databaseSchemaSettings$tablePrefixCohortDefinitionTables == '')
+  testthat::expect_true(databaseSchemaSettings$tablePrefixDatabaseDefinitionTables == '')
+  
+  databaseSchemaSettings <- createDatabaseSchemaSettings(
+    resultSchema = ohdsiDatabaseSchema, 
+    tablePrefix = '',
+    targetDialect = targetDialect,
+    cohortDefinitionSchema = 'test 123',
+    tablePrefixCohortDefinitionTables = 'a',
+    databaseDefinitionSchema = 'test234',
+    tablePrefixDatabaseDefinitionTables = 'b'
+  )
+  
+  testthat::expect_true(databaseSchemaSettings$cohortDefinitionSchema == 'test 123')
+  testthat::expect_true(databaseSchemaSettings$databaseDefinitionSchema == 'test234')
+  testthat::expect_true(databaseSchemaSettings$tablePrefixCohortDefinitionTables == 'A_')
+  testthat::expect_true(databaseSchemaSettings$tablePrefixDatabaseDefinitionTables == 'B_')
+  
+  
+  testthat::expect_true(class(databaseSchemaSettings) == 'plpDatabaseResultSchema')
+  
+}
+)
+
+
+test_that("test createDatabaseDetails works", {
+  
+  databaseList <- createDatabaseList(
+    cdmDatabaseSchemas = paste0('database', 1:5)
+  )
+  
+  testthat::expect_true(length(databaseList) == length(paste0('database', 1:5)))
+  testthat::expect_true(class(databaseList) == 'list')
+  testthat::expect_true(!is.null(databaseList$database1$databaseDetails))
+  testthat::expect_true(!is.null(databaseList$database1$databaseMetaData))
+  
+  testthat::expect_equal(
+    databaseList$database1$databaseDetails$databaseMetaDataId,
+    databaseList$database1$databaseMetaData$databaseId
+  )
+  
+}
+)
+
+
+test_that("database creation", {
+  skip_if(Sys.getenv('CI') != 'true', 'not run locally')
+  createPlpResultTables(
+    connectionDetails = connectionRedshift, 
+    resultSchema = ohdsiDatabaseSchema, 
+    targetDialect = targetDialect,
+    deleteTables = T, 
+    createTables = T,
+    tablePrefix = appendRandom('test')
+  )
+  
+  # check the results table is created
+  testthat::expect_true(DatabaseConnector::existsTable(
+    connection = conn, 
+    databaseSchema = ohdsiDatabaseSchema,
+    tableName = paste0(appendRandom('test'),'_PERFORMANCES')
+  ))
+
+})
+
+
+test_that("results uploaded to database", {
+  skip_if(Sys.getenv('CI') != 'true', 'not run locally')
+  resultsLoc <- file.path(saveLoc,'dbUp')
+  
+  plpResult$model$trainDetails$developmentDatabase <- 'test' 
+  savePlpResult(plpResult, file.path(resultsLoc, 'Analysis_1','plpResult'))
+  # save validation
+  if(!dir.exists(file.path(resultsLoc,'Validation','test', 'Analysis_1'))){
+    dir.create(file.path(resultsLoc,'Validation','test', 'Analysis_1'), recursive = T)
+  }
+  plpResult$model$validationDetails <- list(
+    targetId = 1, 
+    outcomeId = outcomeId,
+    developmentDatabase = 'test',
+    validationDatabase = 'test',
+    populationSettings = plpResult$model$modelDesign$populationSettings, 
+    restrictPlpDataSettings = plpResult$model$modelDesign$restrictPlpDataSettings
+    )
+  savePlpResult(plpResult, file.path(resultsLoc,'Validation','test', 'Analysis_1', 'validationResult'))
+  
+  # add results:
+  addMultipleRunPlpToDatabase(
+    connectionDetails  = connectionRedshift, 
+    databaseSchemaSettings = createDatabaseSchemaSettings(
+      resultSchema = ohdsiDatabaseSchema, 
+      tablePrefix = appendRandom('test'),
+      targetDialect = targetDialect
+    ), 
+    cohortDefinitions = data.frame(
+      cohortName = c('blank1','blank2','blank3'), 
+      cohortId = c(1,2,3), 
+      json = rep('bla',3)
+      ),
+    databaseList = createDatabaseList(
+      cdmDatabaseSchemas = c('test')
+    ),
+    resultLocation = resultsLoc,
+    modelSaveLocation = file.path(saveLoc,'modelLocation') # new
+  )
+  
+  # check the results table is populated
+  sql <- 'select count(*) as N from @resultSchema.@appendperformances;'
+  sql <- SqlRender::render(sql, resultSchema = ohdsiDatabaseSchema, append = appendRandom('test_'))
+  res <- DatabaseConnector::querySql(conn, sql)
+  testthat::expect_true(res$N[1]>0)
+  
+  # add test: check model location has result?
+
+})
+
+test_that("database deletion", {
+  skip_if(Sys.getenv('CI') != 'true', 'not run locally')
+  createPlpResultTables(
+    connectionDetails = connectionRedshift, 
+    resultSchema = ohdsiDatabaseSchema, 
+    targetDialect = targetDialect,
+    deleteTables = T, 
+    createTables = F,
+    tablePrefix = appendRandom('test')
+  )
+  
+  # check the results table is then deleted
+  testthat::expect_false(DatabaseConnector::existsTable(
+    connection = conn, 
+    databaseSchema = ohdsiDatabaseSchema,
+    tableName = paste0(appendRandom('test'),'_PERFORMANCES')
+  ))
+
+})
+
+# disconnect
+if (Sys.getenv('CI') == 'true') {
+  DatabaseConnector::disconnect(conn)
+}
+
+# code to test sqlite creation, result and diagnostic upload all in one
+test_that("temporary sqlite with results works", {
+  
+  resultsLoc <- file.path(saveLoc,'sqliteTest')
+  
+  savePlpResult(plpResult, file.path(resultsLoc, 'Analysis_1','plpResult'))
+  # save diagnostic
+  saveRDS(diagnoseResult, file.path(resultsLoc,'Analysis_1','diagnosePlp.rds'))
+  
+  sqliteLocation <- insertResultsToSqlite(
+    resultLocation = resultsLoc, 
+    cohortDefinitions = data.frame(
+      cohortName = c('blank1','blank2','blank3'), 
+      cohortId = c(1,2,3), 
+      json = rep('bla',3)
+    ),
+    databaseList = createDatabaseList(
+      cdmDatabaseSchemas = c('test')
+    ),
+    sqliteLocation = file.path(resultsLoc, 'sqlite')
+  )
+  
+  # expect the database to exist
+  testthat::expect_true(file.exists(sqliteLocation))
+  
+  cdmDatabaseSchema <- 'main'
+  ohdsiDatabaseSchema <- 'main'
+  connectionDetails <- DatabaseConnector::createConnectionDetails(
+    dbms = 'sqlite',
+    server = sqliteLocation
+  )
+  conn <- DatabaseConnector::connect(connectionDetails = connectionDetails)
+  targetDialect <- 'sqlite'
+  
+  # check the results table is populated
+  sql <- 'select count(*) as N from main.performances;'
+  res <- DatabaseConnector::querySql(conn, sql)
+  testthat::expect_true(res$N[1]>0)
+  
+  # check the diagnostic table is populated
+  sql <- 'select count(*) as N from main.diagnostics;'
+  res <- DatabaseConnector::querySql(conn, sql)
+  testthat::expect_true(res$N[1]>0)
+  
+  # disconnect
+  DatabaseConnector::disconnect(conn)
+
+})
+
+# SQL lite test
+test_that("temporary sqlite with results works", {
+  
+  externalVal <- plpResult
+  externalVal$model$model <- 'none'
+  externalVal$model$trainDetails <- NULL
+  externalVal$model$validationDetails <- list(
+    targetId = 1, 
+    outcomeId = 3,
+    developmentDatabase = 'test',
+    validationDatabase = 'test',
+    populationSettings = plpResult$model$modelDesign$populationSettings, 
+    restrictPlpDataSettings = plpResult$model$modelDesign$restrictPlpDataSettings
+  )
+  
+sqliteLocation <- insertRunPlpToSqlite(
+  runPlp = plpResult, 
+  externalValidatePlp = NULL
+  )
+
+# expect the database to exist
+testthat::expect_true(file.exists(sqliteLocation))
+
+cdmDatabaseSchema <- 'main'
+ohdsiDatabaseSchema <- 'main'
+connectionDetails <- DatabaseConnector::createConnectionDetails(
+  dbms = 'sqlite',
+  server = sqliteLocation
+)
+conn <- DatabaseConnector::connect(connectionDetails = connectionDetails)
+targetDialect <- 'sqlite'
+
+# check the results table is populated
+sql <- 'select count(*) as N from main.performances;'
+res <- DatabaseConnector::querySql(conn, sql)
+testthat::expect_true(res$N[1]>0)
+
+
+# check export to csv
+extractDatabaseToCsv(
+  connectionDetails = connectionDetails,
+  databaseSchemaSettings = createDatabaseSchemaSettings(resultSchema = 'main'),
+  csvFolder = file.path(saveLoc, 'csvFolder')
+)
+
+testthat::expect_true(dir.exists(file.path(saveLoc, 'csvFolder')))
+testthat::expect_true(length(dir(file.path(saveLoc, 'csvFolder'))) > 0 )
+testthat::expect_true(dir.exists(file.path(saveLoc, 'csvFolder', 'models'))) # new
+testthat::expect_true(length(dir(file.path(saveLoc, 'csvFolder', 'models'))) > 0 ) # new
+# disconnect
+DatabaseConnector::disconnect(conn)
+
+
+})
+
+# importFromCsv test here as can use previous csv saving
+test_that("import from csv", {
+  
+  cohortDef <- extractCohortDefinitionsCSV(
+    csvFolder = file.path(saveLoc, 'csvFolder')
+  )
+  testthat::expect_true(inherits(cohortDef, 'data.frame'))
+  testthat::expect_true(ncol(cohortDef) == 4)
+  
+  databaseList <- extractDatabaseListCSV(
+    csvFolder = file.path(saveLoc, 'csvFolder')
+  )
+  testthat::expect_true(inherits(databaseList, 'list'))
+  testthat::expect_true(!is.null(databaseList[[1]]$databaseDetails))
+  testthat::expect_true(!is.null(databaseList[[1]]$databaseMetaData))
+  
+  # model designs work
+  modeldesignsRow <- data.frame(
+    target_id = 1, outcome_id = 2, population_setting_id = 1, 
+    plp_data_setting_id = 1, model_setting_id = 1, 
+    covariate_setting_id = 1, sample_setting_id = 1,
+    split_setting_id = 1, feature_engineering_setting_id =1	, 
+    tidy_covariates_setting_id = 1
+    )
+  res <- getModelDesignSettingTable(modeldesignsRow)
+  # expect res to be a data.frame, check values?
+  testthat::expect_true(inherits(res, 'data.frame'))
+  
+  modelDesign <- getModelDesignCsv(
+    modelDesignSettingTable = res, 
+    csvFolder = file.path(saveLoc, 'csvFolder')
+  ) 
+  testthat::expect_true(inherits(modelDesign, 'modelDesign'))
+  
+  # performance works
+  res <- getPerformanceEvaluationCsv(
+    performanceId = 1, 
+    csvFolder = file.path(saveLoc, 'csvFolder')
+  )
+  testthat::expect_true(inherits(res, 'list'))
+  testthat::expect_true(
+    sum(names(res) %in% 
+      c('evaluationStatistics', 'thresholdSummary',
+        'calibrationSummary', 'demographicSummary',
+        'predictionDistribution'
+      )
+    ) == 5
+  )
+  
+  
+  # test object extracts  
+  obj <- extractObjectFromCsv(
+    performanceId = 1, 
+    csvFolder = file.path(saveLoc, 'csvFolder')
+  )
+  testthat::expect_true(inherits(obj, 'externalValidatePlp') | inherits(obj, 'runPlp'))
+  
+  # test diagnostic extracted
+  diag <- extractDiagnosticFromCsv(
+    diagnosticId = 1, 
+    csvFolder = file.path(saveLoc, 'csvFolder')
+  )
+  testthat::expect_true(inherits(diag, 'diagnosePlp') | is.null(diag))
+  
+  
+  
+  # Testing everything together
+  csvServerLoc <- file.path(tempdir(), 'newCsvDatabase')
+  if(!dir.exists(file.path(tempdir(), 'newCsvDatabase'))){
+    dir.create(file.path(tempdir(), 'newCsvDatabase'), recursive = T)
+  }
+  newResultConnDetails <- DatabaseConnector::createConnectionDetails(
+    dbms = 'sqlite', 
+    server = file.path(csvServerLoc,'newCsv.sqlite')
+    )
+  newResultConn <- DatabaseConnector::connect(newResultConnDetails)
+  csvDatabaseSchemaSettings <-  PatientLevelPrediction::createDatabaseSchemaSettings(
+    resultSchema = 'main', 
+    tablePrefix = '', 
+    targetDialect = 'sqlite', 
+    tempEmulationSchema = NULL
+    )
+  
+  # create empty tables to insert csv into
+  PatientLevelPrediction::createPlpResultTables(
+    connectionDetails = newResultConnDetails, 
+    targetDialect = 'sqlite', 
+    resultSchema = 'main', 
+    createTables = T, 
+    deleteTables = T, 
+    tablePrefix = '', 
+    tempEmulationSchema = NULL
+    )
+    
+  res <- insertCsvToDatabase(
+    csvFolder = file.path(saveLoc, 'csvFolder'),
+    connectionDetails = newResultConnDetails,
+    databaseSchemaSettings = csvDatabaseSchemaSettings,
+    modelSaveLocation = file.path(csvServerLoc,'models'),
+    csvTableAppend = ''
+  )
+  testthat::expect_true(res)
+  
+  # check some of the tables
+  
+  
+})
+
+
+# new - check null model just reports message
+test_that("message if model is null", {
+  
+  model2 <- list(noModel = T)
+  attr(model2, "predictionFunction") <- 'noModel'
+  attr(model2, "saveType") <- 'RtoJson'
+  class(model2) <- 'plpModel'
+  
+  plpResult2 <- plpResult
+  plpResult2$model <- model2
+  
+  savePlpResult(plpResult2, file.path(tempdir(), 'null_model', 'Analysis_1', 'plpResult'))
+  
+  nullModelServerLoc <- file.path(tempdir(), 'nullModelDatabase')
+  if(!dir.exists(file.path(tempdir(), 'nullModelDatabase'))){
+    dir.create(file.path(tempdir(), 'nullModelDatabase'), recursive = T)
+  }
+  nullModelResultConnDetails <- DatabaseConnector::createConnectionDetails(
+    dbms = 'sqlite', 
+    server = file.path(nullModelServerLoc,'sqlite.sqlite')
+  )
+  nullModelDatabaseSchemaSettings <-  createDatabaseSchemaSettings(
+    resultSchema = 'main', 
+    tablePrefix = '', 
+    targetDialect = 'sqlite', 
+    tempEmulationSchema = NULL
+  )
+  
+  createPlpResultTables(
+    connectionDetails = nullModelResultConnDetails,
+    targetDialect = 'sqlite',
+    resultSchema = 'main', 
+    deleteTables = T, 
+    createTables = T,
+    tablePrefix = ''
+  )
+
+  testthat::expect_message(
+    addMultipleRunPlpToDatabase(
+      connectionDetails = nullModelResultConnDetails, 
+      databaseSchemaSettings = nullModelDatabaseSchemaSettings,
+      resultLocation = file.path(tempdir(), 'null_model'), 
+      modelSaveLocation = file.path(tempdir(), 'null_model', 'models')
+    )
+  )
+  
+})
+
+
diff --git a/tests/testthat/test-andromedahelperfunctions_updated.R b/tests/testthat/test-andromedahelperfunctions.R
similarity index 90%
rename from tests/testthat/test-andromedahelperfunctions_updated.R
rename to tests/testthat/test-andromedahelperfunctions.R
index c11072f75..d63bb397d 100644
--- a/tests/testthat/test-andromedahelperfunctions_updated.R
+++ b/tests/testthat/test-andromedahelperfunctions.R
@@ -23,8 +23,8 @@ context("AndromedaHelperFunctions")
 # batcheRestrict test 
 test_that("batchRestrict", {
   
-  metaData <- attr(plpDataBig$covariateData, 'metaData')
-  covariateData <- PatientLevelPrediction:::batchRestrict(plpDataBig$covariateData, populationBig, sizeN = 10000000)
+  metaData <- attr(plpData$covariateData, 'metaData')
+  covariateData <- PatientLevelPrediction:::batchRestrict(plpData$covariateData, population, sizeN = 1000000)
   testthat::expect_is(covariateData, 'CovariateData')
   
   expect_equal(names(metaData), names(attr(covariateData, 'metaData')))
diff --git a/tests/testthat/test-covariateExtras.R b/tests/testthat/test-covariateExtras.R
index fa2f960a5..17c9e04ec 100644
--- a/tests/testthat/test-covariateExtras.R
+++ b/tests/testthat/test-covariateExtras.R
@@ -18,10 +18,6 @@ library("testthat")
 
 context("CovariateExtras")
 
-connectionDetails <- Eunomia::getEunomiaConnectionDetails()
-Eunomia::createCohorts(connectionDetails)
-
-
 test_that("settings creation", {
   
 covSet <- createCohortCovariateSettings(
@@ -113,14 +109,14 @@ covs <- FeatureExtraction::getDbCovariateData(
   cohortTable = "cohort", 
   cohortDatabaseSchema = "main", 
   cohortTableIsTemp = F, 
-  cohortId = 1, 
+  cohortIds = c(1), 
   rowIdField = 'rowId', 
   covariateSettings = covSet, 
   aggregated = F
   )
 
 expect_equal(1, covs$covariateRef %>% dplyr::tally() %>% dplyr::pull())
-expect_equal(as.double(covs$covariateRef %>% dplyr::select(.data$covariateId) %>% dplyr::collect()), covSet$covariateId)
+expect_equal(as.double(covs$covariateRef %>% dplyr::select("covariateId") %>% dplyr::collect()), covSet$covariateId)
 expect_true(covs$covariates %>% dplyr::tally() %>% dplyr::pull() > 0)
 
 })
diff --git a/tests/testthat/test-cyclopsModels_updated.R b/tests/testthat/test-cyclopsModels.R
similarity index 97%
rename from tests/testthat/test-cyclopsModels_updated.R
rename to tests/testthat/test-cyclopsModels.R
index f452f7ce4..0bbf6d779 100644
--- a/tests/testthat/test-cyclopsModels_updated.R
+++ b/tests/testthat/test-cyclopsModels.R
@@ -239,18 +239,20 @@ testthat::expect_error(setIterativeHardThresholding(seed = 'F'))
 test_that("test logistic regression runs", {
 
 modelSettings <- setLassoLogisticRegression()
-trainData <- createTrainData(plpData, population)
 
 fitModel <- fitPlp(
   trainData = trainData,   
   modelSettings = modelSettings,
   search = "grid", 
-  analysisId = 'lrTest'
+  analysisId = 'lrTest',
+  analysisPath = tempdir()
   )
 
 expect_equal(length(unique(fitModel$prediction$evaluationType)),2)
 expect_equal(nrow(fitModel$prediction), nrow(trainData$labels)*2)
 expect_true(length(fitModel$model$coefficients) < trainData$covariateData$covariateRef %>% dplyr::tally() %>% dplyr::pull()+1)
+
+
 expect_true(!is.null(fitModel$trainDetails$trainingTime))
 expect_equal(fitModel$trainDetails$trainingDate,Sys.Date())
 
@@ -261,7 +263,7 @@ expect_equal(
 
 expect_true('covariateValue' %in% colnames(fitModel$covariateImportance))
 
-expect_equal(fitModel$trainDetails$outcomeId, attr(trainData, 'metaData')$outcomeId)
-expect_equal(fitModel$trainDetails$cohortId, attr(trainData, 'metaData')$cohortId)
+expect_equal(fitModel$modelDesign$outcomeId, attr(trainData, 'metaData')$outcomeId)
+expect_equal(fitModel$modelDesign$targetId, attr(trainData, 'metaData')$targetId)
 
 })
diff --git a/tests/testthat/test-dataSplitting_updated.R b/tests/testthat/test-dataSplitting.R
similarity index 97%
rename from tests/testthat/test-dataSplitting_updated.R
rename to tests/testthat/test-dataSplitting.R
index 158c6ca25..b8ce628bb 100644
--- a/tests/testthat/test-dataSplitting_updated.R
+++ b/tests/testthat/test-dataSplitting.R
@@ -22,9 +22,9 @@ context("Data splitting")
 
 populationT <- plpData$cohorts
 populationT$outcomeCount <- sample(c(0,1), nrow(populationT), replace = T)
-attr(populationT, "metaData")$outcomeId <- 2
+attr(populationT, "metaData")$outcomeId <- outcomeId
 attr(populationT, "metaData")$populationSettings <- list(madeup = T)
-attr(populationT, "metaData")$plpDataSettings <- list(madeup = T)
+attr(populationT, "metaData")$restrictPlpDataSettings <- list(madeup = T)
 attr(populationT, "metaData")$attrition <- c(1,2,3)
 
 # check correct inputs
@@ -161,13 +161,13 @@ test_that("Main split function: splitData", {
   
   # check attributes for Train
   expect_equal(attr(splitData$Train, "metaData")$outcomeId, attr(populationT, "metaData")$outcomeId)
-  expect_equal(attr(splitData$Train, "metaData")$cohortId, plpData$metaData$databaseDetails$cohortId)
+  expect_equal(attr(splitData$Train, "metaData")$targetId, plpData$metaData$databaseDetails$targetId)
   expect_equal(
     attr(splitData$Train, "metaData")$cdmDatabaseSchema, 
     plpData$metaData$databaseDetails$cdmDatabaseSchema
     )
   
-  expect_is(attr(splitData$Train, "metaData")$plpDataSettings, 'list')
+  expect_is(attr(splitData$Train, "metaData")$restrictPlpDataSettings, 'list')
   expect_equal(
     attr(splitData$Train, "metaData")$covariateSettings, 
     plpData$metaData$covariateSettings
diff --git a/tests/testthat/test-demographicSummary_updated.R b/tests/testthat/test-demographicSummary.R
similarity index 100%
rename from tests/testthat/test-demographicSummary_updated.R
rename to tests/testthat/test-demographicSummary.R
diff --git a/tests/testthat/test-diagnostic.R b/tests/testthat/test-diagnostic.R
new file mode 100644
index 000000000..2c261dd16
--- /dev/null
+++ b/tests/testthat/test-diagnostic.R
@@ -0,0 +1,166 @@
+# Copyright 2021 Observational Health Data Sciences and Informatics
+#
+# This file is part of PatientLevelPrediction
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+context("Diagnostic")
+
+
+test_that("getMaxEndDaysFromCovariates works", {
+  
+  covariateSettings <- FeatureExtraction::createCovariateSettings(
+    useDemographicsGender = T,
+    endDays = -1
+  )
+  testthat::expect_equal(getMaxEndDaysFromCovariates(covariateSettings), -1)
+  
+  covariateSettings <- list(
+    FeatureExtraction::createCovariateSettings(
+    useDemographicsGender = T,
+    endDays = -1
+  ),
+  FeatureExtraction::createCovariateSettings(
+    useDemographicsGender = T,
+    endDays = 2
+  )
+  )
+  testthat::expect_equal(getMaxEndDaysFromCovariates(covariateSettings), 2)
+  
+  covariateSettings <- list(
+    FeatureExtraction::createCovariateSettings(
+      useDemographicsGender = T,
+      endDays = -1,
+    ),
+    PatientLevelPrediction::createCohortCovariateSettings(
+      endDay = 5, 
+      settingId = 1, 
+      cohortName = 'test', 
+      cohortId = 1, 
+      analysisId = 111, 
+      cohortDatabaseSchema = '', cohortTable = '')
+  )
+  testthat::expect_equal(getMaxEndDaysFromCovariates(covariateSettings), 5)
+  
+  # if no covariate setting has endDays return 0
+  testthat::expect_equal(
+    getMaxEndDaysFromCovariates(list(empty = list(gfg=2), empty2 = list(ff=1))), 
+    0
+    )
+  
+})
+
+test_that("test diagnosePlp works", {
+  test <- diagnosePlp(
+    plpData = tinyPlpData,
+    outcomeId = outcomeId,
+    analysisId = 'diagnoseTest',
+    populationSettings = createStudyPopulationSettings(
+      riskWindowStart = 1, 
+      startAnchor = 'cohort start', 
+      riskWindowEnd = 365,
+      endAnchor = 'cohort start'
+    ),
+    splitSettings = createDefaultSplitSetting(),
+    sampleSettings = createSampleSettings(), # default none
+    saveDirectory = file.path(saveLoc, 'diagnostics'),
+    featureEngineeringSettings = createFeatureEngineeringSettings(), # default none
+    modelSettings = setLassoLogisticRegression(), # default to logistic regression
+    preprocessSettings = createPreprocessSettings()
+  )
+  #check results are a list
+  testthat::expect_is(test, 'diagnosePlp')
+                         
+  # check list names
+  testthat::expect_equal(
+    sum( names(test) %in% 
+           c('summary','participants','predictors',
+             'outcomes', 'designs', 'modelDesign',
+             'databaseSchema')
+         )
+    , 7)
+  
+  # check the results are saved into the databaseName directory
+  testthat::expect_equal(T, dir.exists(file.path(saveLoc, 'diagnostics')))
+  testthat::expect_equal(T, file.exists(file.path(saveLoc, 'diagnostics','diagnoseTest','diagnosePlp.rds')))
+  
+  testthat::expect_is(test$summary, 'data.frame')
+  testthat::expect_is(test$participants, 'data.frame')
+  testthat::expect_is(test$predictors, 'data.frame') # rename this outcome survival?
+  testthat::expect_is(test$outcomes, 'data.frame') 
+  testthat::expect_is(test$databaseSchema, 'character') 
+  
+  testthat::expect_true(!is.null(test$modelDesign$targetId))
+  testthat::expect_true(!is.null(test$modelDesign$outcomeId))
+  testthat::expect_true(!is.null(test$modelDesign$restrictPlpDataSettings))
+  testthat::expect_true(!is.null(test$modelDesign$covariateSettings))
+  testthat::expect_true(!is.null(test$modelDesign$populationSettings))
+
+
+})
+
+
+test_that("test diagnoseMultiplePlp works", {
+  
+  analysis1 <- createModelDesign(
+    targetId = 1,
+    outcomeId = outcomeId,
+    restrictPlpDataSettings = createRestrictPlpDataSettings(firstExposureOnly = F, 
+                                                            washoutPeriod = 0,
+                                                            sampleSize = 100),
+    populationSettings = createStudyPopulationSettings(),
+    covariateSettings = covariateSettings,
+    featureEngineeringSettings = NULL,
+    sampleSettings = NULL,
+    splitSettings = createDefaultSplitSetting(),
+    preprocessSettings = createPreprocessSettings(),
+    modelSettings = setLassoLogisticRegression(seed = 12)
+  )
+  
+  analysis2 <- createModelDesign(
+    targetId = 1,
+    outcomeId = outcomeId,
+    restrictPlpDataSettings = createRestrictPlpDataSettings(firstExposureOnly = F, 
+                                                            washoutPeriod = 0,
+                                                            sampleSize = 100),
+    populationSettings = createStudyPopulationSettings(washoutPeriod = 400),
+    covariateSettings = covariateSettings,
+    featureEngineeringSettings = NULL,
+    sampleSettings = NULL,
+    splitSettings = createDefaultSplitSetting(),
+    preprocessSettings = createPreprocessSettings(),
+    modelSettings = setLassoLogisticRegression(seed = 12)
+  )
+  
+  diagnoseMultiplePlp(
+    databaseDetails = databaseDetails,
+    modelDesignList = list(
+      analysis1,
+      analysis2
+      ),
+    cohortDefinitions = data.frame(
+      cohortId = c(1, outcomeId),
+      cohortName = c('target', 'outcome')
+    ), 
+    saveDirectory = file.path(saveLoc, 'diagnosticsMultiple')
+  )
+  
+  # file.path(saveDirectory,'settings.csv') exits
+  testthat::expect_true(file.exists(file.path(saveLoc, 'diagnosticsMultiple', 'settings.csv')))
+  
+  # file.path(saveDirectory, settings$analysisId, 'diagnosePlp.rds') exists
+  testthat::expect_true(length(dir(file.path(saveLoc, 'diagnosticsMultiple'), pattern = 'Analysis_')) == 2)
+  
+  testthat::expect_true(file.exists(file.path(saveLoc, 'diagnosticsMultiple', 'Analysis_1', 'diagnosePlp.rds')))
+  
+})
diff --git a/tests/testthat/test-diagnostic_updated.R b/tests/testthat/test-diagnostic_updated.R
deleted file mode 100644
index 3a12a7b7a..000000000
--- a/tests/testthat/test-diagnostic_updated.R
+++ /dev/null
@@ -1,49 +0,0 @@
-# Copyright 2021 Observational Health Data Sciences and Informatics
-#
-# This file is part of PatientLevelPrediction
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-context("Diagnostic")
-
-test_that("test code works when using plpData", {
-  test <- diagnostic(
-    plpData = plpData, 
-    cdmDatabaseName = 'madeup', 
-    cohortName = 'made up target',
-    outcomeNames = paste0('made up outcome', 1:2),
-    databaseDetails,
-    restrictPlpDataSettings,
-    populationSettings = createStudyPopulationSettings(
-      riskWindowStart = 1, 
-      startAnchor = 'cohort start', 
-      riskWindowEnd = 365,
-      endAnchor = 'cohort start'
-      ),
-    minCellCount = 5,
-    outputFolder = file.path(saveLoc, 'diagnostics')
-  )
-  #check results are a list
-  testthat::expect_equal(class(test), 'list')
-                         
-  # check list names
-  testthat::expect_equal(sum(names(test)%in%c('distribution','proportion','characterization')), 3)
-  
-  # check the results are saved into the databaseName directory
-  testthat::expect_equal(T, dir.exists(file.path(saveLoc, 'diagnostics')))
-  
-  #check tar
-
-  testthat::expect_equal(unique(test$proportion$TAR)[1], paste0('cohort start', ' + ', 1, ' days - ',
-                                                        'cohort start', ' + ', 365, ' days'))
-})
diff --git a/tests/testthat/test-ensemble.R b/tests/testthat/test-ensemble.R
deleted file mode 100644
index 5a092f332..000000000
--- a/tests/testthat/test-ensemble.R
+++ /dev/null
@@ -1,100 +0,0 @@
-# Copyright 2021 Observational Health Data Sciences and Informatics
-#
-# This file is part of PatientLevelPrediction
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-context("Ensemble")
-
-if(F){
-ensemble <- runEnsembleModel(population = population,
-                             dataList = list(plpData, plpData),
-                             modelList = list(lrSet, gbmSet), # change to get rid of warning?
-                             testSplit = "subject",
-                             testFraction = 0.2, 
-                             stackerUseCV = T,
-                             splitSeed = 1,
-                             nfold = 3,
-                             saveDirectory= saveLoc,
-                             saveEnsemble = F,
-                             savePlpData=F, 
-                             savePlpResult=F, 
-                             savePlpPlots = F, 
-                             saveEvaluation = F,
-                             analysisId = 'ensemble',
-                             verbosity = "INFO",
-                             ensembleStrategy = "stacked")
-
-test_that("run ensemble model", {
-  
-  testthat::expect_s3_class(ensemble, 'ensemblePlp')
-  })
-
-test_that("combine mean ensemble model works", {
-  comEn <- createEnsemble(runPlpList = list(plpResult,plpDataReal))
-  testthat::expect_s3_class(comEn, 'ensemblePlp')
-})
-
-test_that("combine AUC ensemble model works", {
-  comEn <- createEnsemble(runPlpList = list(plpResult,plpDataReal), weighted = T)
-  testthat::expect_s3_class(comEn, 'ensemblePlp')
-})
-
-test_that("combine manual weights ensemble model works", {
-  comEn <- createEnsemble(runPlpList = list(plpResult,plpDataReal), weighted = T, weights = runif(2))
-  testthat::expect_s3_class(comEn, 'ensemblePlp')
-})
-
-test_that("combine ensemble model fails when weights too long", {
-  testthat::expect_error(createEnsemble(runPlpList = list(plpResult,plpDataReal), weighted = T, weights = runif(3)))
-})
-
-
-test_that("apply ensemble model", {
-  ensemblePerf <- applyEnsembleModel(population = population,
-                                     dataList = list(plpData,plpData),
-                                     ensembleModel = ensemble, 
-                                     calculatePerformance = T)
-  
-  testthat::expect_equal(class(ensemblePerf), 'list')
-  testthat::expect_equal(sum(names(ensemblePerf)%in%c('prediction','performanceEvaluation')), 2)
-  testthat::expect_s3_class(ensemblePerf$prediction, 'data.frame')
-  testthat::expect_equal(class(ensemblePerf$performanceEvaluation), 'plpEvaluation')
-
-})
-
-
-test_that("save/load ensemble model", {
-  saveEnsemblePlpModel(ensembleModel = ensemble$model, dirPath = file.path(saveLoc, 'ensembleSave'))
-  
-  testthat::expect_equal(dir.exists(file.path(saveLoc,'ensembleSave/level1')), T)
-  testthat::expect_equal(dir.exists(file.path(saveLoc,'ensembleSave/level2')), T)
-  
-  ensembleModelLoad <- loadEnsemblePlpModel(file.path(saveLoc,'ensembleSave'))
-  testthat::expect_equal(names(ensemble$model), names(ensembleModelLoad))
-  
-})
-
-
-test_that("save/load ensemble result", {
-  saveEnsemblePlpResult(ensembleResult = ensemble, dirPath = file.path(saveLoc,'ensembleResult'))
-  
-  testthat::expect_equal(file.exists(file.path(saveLoc,'ensembleResult/performanceEvaluation.rds')), T)
-  testthat::expect_equal(file.exists(file.path(saveLoc,'ensembleResult/covariateSummary.rds')), T)
-  
-  ensembleLoad <- loadEnsemblePlpResult(file.path(saveLoc,'ensembleResult'))
-  testthat::expect_equal(names(ensemble), names(ensembleLoad))
-  
-})
-
-}
diff --git a/tests/testthat/test-evaluation_updated.R b/tests/testthat/test-evaluation.R
similarity index 89%
rename from tests/testthat/test-evaluation_updated.R
rename to tests/testthat/test-evaluation.R
index d7effdd51..6669aa488 100644
--- a/tests/testthat/test-evaluation_updated.R
+++ b/tests/testthat/test-evaluation.R
@@ -38,15 +38,15 @@ test_that("modelBasedConcordance", {
 })
 
 test_that("evaluatePlp_survival", {
-  
+  N <- 100
   plpResultSurvivalPred <- data.frame(
-    rowId = 1:300, 
-    ageYear = sample(100, 300, replace = T),
-    gender = sample(c('8507','8532'), 300, replace = T),
-    outcomeCount = c(rep(1,40), rep(0,260)),
-    value = runif(300),
-    evaluationType = rep('Train', 300),
-    survivalTime = sample(2000, 300, replace = T)
+    rowId = 1:N, 
+    ageYear = sample(100, N, replace = T),
+    gender = sample(c('8507','8532'), N, replace = T),
+    outcomeCount = c(rep(1,N*0.1), rep(0,N*0.9)),
+    value = runif(N, max=0.1),
+    evaluationType = rep('Train', N),
+    survivalTime = sample(2000, N, replace = T)
   )
   attr(plpResultSurvivalPred, "metaData")$modelType <- 'survival'
   attr(plpResultSurvivalPred, 'metaData')$timepoint <- 365
@@ -139,6 +139,18 @@ test_that("Calibration metrics", {
   
   })
 
+test_that("E statistics binary", {
+  prediction <- data.frame(
+    value = c(seq(.1, .5, length.out = 5), NA, .2),
+    outcomeCount = c(0, 0, 0, 1, 1, 0, NA)
+  )
+  EStatsBinary <- PatientLevelPrediction:::calculateEStatisticsBinary(prediction)
+  expect_equal(
+    EStatsBinary,
+    c(Eavg = .34, E90 = .56, Emax = .6)
+  )
+})
+
   # TODO: test pref scores 
   # test computePreferenceScore(prediction)
  
diff --git a/tests/testthat/test-extractData_updated.R b/tests/testthat/test-extractData.R
similarity index 61%
rename from tests/testthat/test-extractData_updated.R
rename to tests/testthat/test-extractData.R
index 13470af44..0e15eb37a 100644
--- a/tests/testthat/test-extractData_updated.R
+++ b/tests/testthat/test-extractData.R
@@ -17,15 +17,27 @@
 context("extractPlp")
 
 test_that("summary.plpData", {
-  attr(plpData$outcomes, "metaData")$outcomeIds <- c(2,3)
+  attr(plpData$outcomes, "metaData")$outcomeIds <- c(outcomeId)
   sum <- summary.plpData(plpData)
   testthat::expect_equal(class(sum),'summary.plpData')
 })
 
 test_that("getPlpData errors", {
-  testthat::expect_error(getPlpData(cohortId = NULL))
-  testthat::expect_error(getPlpData(cohortId = c(1,2)))
-  testthat::expect_error(getPlpData(cohortId = 1, outcomeIds = NULL))
+  testthat::expect_error(
+    getPlpData(
+      databaseDetails = list(targetId = NULL)
+      )
+  )
+  testthat::expect_error(
+    getPlpData(
+      databaseDetails = list(targetId = c(1,2))
+      )
+  )
+  testthat::expect_error(
+    getPlpData(
+      databaseDetails = list(targetId = 1, outcomeIds = NULL)
+      )
+  )
 })
 
 
@@ -37,3 +49,13 @@ test_that("getCovariateData", {
   testthat::expect_error(getCovariateData())
 })
 
+test_that("createDatabaseDetails with NULL cdmDatabaseId errors", {
+  testthat::expect_error(createDatabaseDetails(
+    connectionDetails = list(), 
+    cdmDatabaseSchema = 'main', 
+    cdmDatabaseId = NULL, 
+    targetId = 1, 
+    outcomeIds = outcomeId
+  ))
+})
+
diff --git a/tests/testthat/test-featureEngineering.R b/tests/testthat/test-featureEngineering.R
new file mode 100644
index 000000000..8ae88f5ce
--- /dev/null
+++ b/tests/testthat/test-featureEngineering.R
@@ -0,0 +1,292 @@
+# Copyright 2021 Observational Health Data Sciences and Informatics
+#
+# This file is part of PatientLevelPrediction
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+library("testthat")
+context("FeatureEngineering")
+
+
+testFEFun <- function(type = 'none'){
+  
+  result <- createFeatureEngineeringSettings(type = type)
+  
+  return(result)
+}
+
+  
+test_that("createFeatureEngineeringSettings correct class", {
+  
+  featureEngineeringSettings <- testFEFun()
+  
+  expect_is(featureEngineeringSettings, 'featureEngineeringSettings')
+  
+  checkFun <- 'sameData'  # this is the only option at the moment, edit this when more are added
+  expect_equal(attr(featureEngineeringSettings, "fun"), checkFun)
+  
+})
+
+
+testUniFun <- function(k = 100){
+  
+  result <- createUnivariateFeatureSelection(k = k)
+  
+  return(result)
+}
+
+
+
+test_that("createUnivariateFeatureSelection correct class", {
+  k <- sample(1000,1)
+  featureEngineeringSettings <- testUniFun(k = k)
+  
+  expect_is(featureEngineeringSettings, 'featureEngineeringSettings')
+  expect_equal(featureEngineeringSettings$k, k)
+  expect_equal(attr(featureEngineeringSettings, "fun"), 'univariateFeatureSelection')
+  
+  expect_error(testUniFun(k = 'ffdff'))
+  expect_error(testUniFun(k = NULL))
+  expect_error(testUniFun(k = -1))
+})
+
+
+test_that("univariateFeatureSelection", {
+  
+  k <- 20+sample(10,1)
+  featureEngineeringSettings <- testUniFun(k = k)
+  newTrainData <- copyTrainData(trainData)
+  
+  trainDataCovariateSize <- newTrainData$covariateData$covariates %>% dplyr::tally() %>% dplyr::pull()
+  
+  reducedTrainData <- univariateFeatureSelection(
+    trainData = newTrainData, 
+    featureEngineeringSettings = featureEngineeringSettings,
+    covariateIdsInclude = NULL
+    )
+  
+  newDataCovariateSize <- reducedTrainData$covariateData$covariates %>% dplyr::tally() %>% dplyr::pull()
+  expect_true(newDataCovariateSize <= trainDataCovariateSize)
+  
+  # expect k many covariates left
+  expect_equal(k,reducedTrainData$covariateData$covariateRef %>% dplyr::tally() %>% dplyr::pull())
+  
+})
+
+
+test_that("createRandomForestFeatureSelection correct class", {
+  ntreesTest <- sample(1000,1)
+  maxDepthTest <- sample(20,1)
+  featureEngineeringSettings <- createRandomForestFeatureSelection(
+    ntrees = ntreesTest, 
+    maxDepth = maxDepthTest
+    )
+  
+  expect_is(featureEngineeringSettings, 'featureEngineeringSettings')
+  expect_equal(featureEngineeringSettings$ntrees, ntreesTest)
+  expect_equal(featureEngineeringSettings$max_depth, maxDepthTest)
+  expect_equal(attr(featureEngineeringSettings, "fun"), 'randomForestFeatureSelection')
+  
+  # error due to params
+  expect_error(
+    createRandomForestFeatureSelection(
+      ntrees = -1, 
+      maxDepth = maxDepthTest
+    )
+  )
+  
+  expect_error(
+    createRandomForestFeatureSelection(
+      ntrees = 'dfdfd', 
+      maxDepth = maxDepthTest
+    )
+  )
+  
+  expect_error(
+    createRandomForestFeatureSelection(
+      ntrees = 50, 
+      maxDepth = 'maxDepthTest'
+    )
+  )
+  
+  expect_error(
+    createRandomForestFeatureSelection(
+      ntrees = 50, 
+      maxDepth = -1
+    )
+  )
+  
+})
+
+
+test_that("randomForestFeatureSelection", {
+  
+  ntreesTest <- sample(1000,1)
+  maxDepthTest <- sample(20,1)
+  featureEngineeringSettings <- createRandomForestFeatureSelection(
+    ntrees = ntreesTest, 
+    maxDepth = maxDepthTest
+  )
+  
+  newTrainData <- copyTrainData(trainData)
+  trainDataCovariateSize <- newTrainData$covariateData$covariates %>% dplyr::tally() %>% dplyr::pull()
+  
+  reducedTrainData <- randomForestFeatureSelection(
+    trainData = newTrainData, 
+    featureEngineeringSettings = featureEngineeringSettings,
+    covariateIdsInclude = NULL
+  )
+  
+  newDataCovariateSize <- reducedTrainData$covariateData$covariates %>% dplyr::tally() %>% dplyr::pull()
+  expect_true(newDataCovariateSize < trainDataCovariateSize)
+
+})
+
+test_that("featureSelection is applied on test_data", {
+  k <- 20
+  featureEngineeringSettings <- testUniFun(k = k)
+  newTrainData <- copyTrainData(trainData)
+  newTrainData <- univariateFeatureSelection(
+    trainData = newTrainData, 
+    featureEngineeringSettings = featureEngineeringSettings,
+    covariateIdsInclude = NULL
+  )
+  
+  modelSettings <- setLassoLogisticRegression()
+  
+  # added try catch due to model sometimes not fitting
+  plpModel <- tryCatch(
+    {fitPlp(newTrainData, modelSettings, analysisId='FE')}, 
+    error = function(e){return(NULL)}
+  )
+  
+  if(!is.null(plpModel)){ # if the model fit then check this
+    prediction <- predictPlp(plpModel, testData, population)
+    expect_true(attr(prediction, 'metaData')$featureEngineering) 
+  }
+})
+
+test_that("createSplineSettings correct class", {
+ 
+  featureEngineeringSettings <- createSplineSettings(
+    continousCovariateId = 12, 
+    knots = 4
+    )
+  
+  expect_is(featureEngineeringSettings, 'featureEngineeringSettings')
+  expect_equal(featureEngineeringSettings$knots, 4)
+  expect_equal(featureEngineeringSettings$continousCovariateId, 12)
+  expect_equal(attr(featureEngineeringSettings, "fun"), 'splineCovariates')
+  
+  expect_error(createSplineSettings(knots = 'ffdff'))
+  expect_error(createSplineSettings(knots = NULL))
+})
+
+test_that("createSplineSettings correct class", {
+  
+  knots <- 4
+  featureEngineeringSettings <- createSplineSettings(
+    continousCovariateId = 12101, 
+    knots = knots
+  )
+  
+  trainData <- simulatePlpData(plpDataSimulationProfile, n = 200)
+  
+  N <- 50
+  trainData$covariateData$covariates <- data.frame(
+    rowId = sample(trainData$cohorts$rowId, N),
+    covariateId = rep(12101, N),
+    covariateValue = sample(10, N, replace = T)
+  )
+  
+  trainData$covariateData$analysisRef <- data.frame(
+    analysisId = 101,
+    analysisName = 'cond',
+    domainId = 'madeup',
+    startDay = 0,
+    endDay = 0,
+    isBinary = 'N',
+    missingMeansZero = 'N'
+  )
+  
+  trainData$covariateData$covariateRef <- data.frame(
+    covariateId = 12101,
+    covariateName = 'test',
+    analysisId = 101,
+    conceptId = 1
+  )
+  
+newData <- splineCovariates(
+    trainData = trainData, 
+    featureEngineeringSettings = featureEngineeringSettings
+)
+
+testthat::expect_true(1 < nrow(as.data.frame(newData$covariateData$analysisRef)))
+testthat::expect_true((knots+1) == nrow(as.data.frame(newData$covariateData$covariateRef)))
+testthat::expect_true((knots+1) == length(table(as.data.frame(newData$covariateData$covariates)$covariateId)))
+  
+})
+
+
+test_that("createStratifiedImputationSettings correct class", {
+  
+  featureEngineeringSettings <- createStratifiedImputationSettings(
+    covariateId = 12101, 
+    ageSplits = c(20,50,70)
+  )
+  
+  trainData <- simulatePlpData(plpDataSimulationProfile, n = 200)
+  
+  N <- 50
+  trainData$covariateData$covariates <- data.frame(
+    rowId = sample(trainData$cohorts$rowId, N),
+    covariateId = rep(12101, N),
+    covariateValue = sample(10, N, replace = T)
+  )
+  
+  trainData$covariateData$analysisRef <- data.frame(
+    analysisId = 101,
+    analysisName = 'cond',
+    domainId = 'madeup',
+    startDay = 0,
+    endDay = 0,
+    isBinary = 'N',
+    missingMeansZero = 'N'
+  )
+  
+  trainData$covariateData$covariateRef <- data.frame(
+    covariateId = 12101,
+    covariateName = 'test',
+    analysisId = 101,
+    conceptId = 1
+  )
+  
+  stratifiedMeans <- calculateStratifiedMeans(
+    trainData = trainData,
+    featureEngineeringSettings = featureEngineeringSettings
+  )
+  
+  testthat::expect_true(nrow(stratifiedMeans)  == 8)
+  
+imputedData <- imputeMissingMeans(
+    trainData = trainData, 
+    covariateId = 12101,
+    ageSplits = c(20,50,70),
+    stratifiedMeans = stratifiedMeans
+)
+
+testthat::expect_true(
+  nrow(as.data.frame(imputedData$covariateData$covariates)) == 200
+)
+
+})
\ No newline at end of file
diff --git a/tests/testthat/test-featureEngineering_updated.R b/tests/testthat/test-featureEngineering_updated.R
deleted file mode 100644
index 8d05d1650..000000000
--- a/tests/testthat/test-featureEngineering_updated.R
+++ /dev/null
@@ -1,155 +0,0 @@
-# Copyright 2021 Observational Health Data Sciences and Informatics
-#
-# This file is part of PatientLevelPrediction
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-library("testthat")
-context("FeatureEngineering")
-
-
-trainData <- createTrainData(plpData, population)
-
-testFEFun <- function(type = 'none'){
-  
-  result <- createFeatureEngineeringSettings(type = type)
-  
-  return(result)
-}
-
-  
-test_that("createFeatureEngineeringSettings correct class", {
-  
-  featureEngineeringSettings <- testFEFun()
-  
-  expect_is(featureEngineeringSettings, 'featureEngineeringSettings')
-  
-  checkFun <- 'sameData'  # this is the only option at the moment, edit this when more are added
-  expect_equal(attr(featureEngineeringSettings, "fun"), checkFun)
-  
-})
-
-
-testUniFun <- function(k = 100){
-  
-  result <- createUnivariateFeatureSelection(k = k)
-  
-  return(result)
-}
-
-
-
-test_that("createUnivariateFeatureSelection correct class", {
-  k <- sample(1000,1)
-  featureEngineeringSettings <- testUniFun(k = k)
-  
-  expect_is(featureEngineeringSettings, 'featureEngineeringSettings')
-  expect_equal(featureEngineeringSettings$k, k)
-  expect_equal(attr(featureEngineeringSettings, "fun"), 'univariateFeatureSelection')
-  
-  expect_error(testUniFun(k = 'ffdff'))
-  expect_error(testUniFun(k = NULL))
-  expect_error(testUniFun(k = -1))
-})
-
-
-test_that("univariateFeatureSelection", {
-  
-  k <- 20+sample(100,1)
-  featureEngineeringSettings <- testUniFun(k = k)
-  
-  trainDataCovariateSize <- trainData$covariateData$covariates %>% dplyr::tally() %>% dplyr::pull()
-  
-  reducedTrainData <- univariateFeatureSelection(
-    trainData = trainData, 
-    featureEngineeringSettings = featureEngineeringSettings,
-    covariateIdsInclude = NULL
-    )
-  
-  newDataCovariateSize <- reducedTrainData$covariateData$covariates %>% dplyr::tally() %>% dplyr::pull()
-  expect_true(newDataCovariateSize <= trainDataCovariateSize)
-  
-  # expect k many covariates left - REMOVED AS TIES MAKES THIS FAIL OCCASIONALLY
-  ##expect_true(abs(k - reducedTrainData$covariateData$covariateRef %>% dplyr::tally() %>% dplyr::pull()) <= 5)
-  
-})
-
-# refresh the training data
-trainData <- createTrainData(plpData, population)
-
-test_that("createRandomForestFeatureSelection correct class", {
-  ntreesTest <- sample(1000,1)
-  maxDepthTest <- sample(20,1)
-  featureEngineeringSettings <- createRandomForestFeatureSelection(
-    ntrees = ntreesTest, 
-    maxDepth = maxDepthTest
-    )
-  
-  expect_is(featureEngineeringSettings, 'featureEngineeringSettings')
-  expect_equal(featureEngineeringSettings$ntrees, ntreesTest)
-  expect_equal(featureEngineeringSettings$max_depth, maxDepthTest)
-  expect_equal(attr(featureEngineeringSettings, "fun"), 'randomForestFeatureSelection')
-  
-  # error due to params
-  expect_error(
-    createRandomForestFeatureSelection(
-      ntrees = -1, 
-      maxDepth = maxDepthTest
-    )
-  )
-  
-  expect_error(
-    createRandomForestFeatureSelection(
-      ntrees = 'dfdfd', 
-      maxDepth = maxDepthTest
-    )
-  )
-  
-  expect_error(
-    createRandomForestFeatureSelection(
-      ntrees = 50, 
-      maxDepth = 'maxDepthTest'
-    )
-  )
-  
-  expect_error(
-    createRandomForestFeatureSelection(
-      ntrees = 50, 
-      maxDepth = -1
-    )
-  )
-  
-})
-
-
-test_that("randomForestFeatureSelection", {
-  
-  ntreesTest <- sample(1000,1)
-  maxDepthTest <- sample(20,1)
-  featureEngineeringSettings <- createRandomForestFeatureSelection(
-    ntrees = ntreesTest, 
-    maxDepth = maxDepthTest
-  )
-  
-  trainDataCovariateSize <- trainData$covariateData$covariates %>% dplyr::tally() %>% dplyr::pull()
-  
-  reducedTrainData <- randomForestFeatureSelection(
-    trainData = trainData, 
-    featureEngineeringSettings = featureEngineeringSettings,
-    covariateIdsInclude = NULL
-  )
-  
-  newDataCovariateSize <- reducedTrainData$covariateData$covariates %>% dplyr::tally() %>% dplyr::pull()
-  expect_true(newDataCovariateSize < trainDataCovariateSize)
-
-})
diff --git a/tests/testthat/test-featureImportance.R b/tests/testthat/test-featureImportance.R
index 6c964e2c3..d0140adda 100644
--- a/tests/testthat/test-featureImportance.R
+++ b/tests/testthat/test-featureImportance.R
@@ -21,12 +21,38 @@ context("FeatureImportance")
 
 test_that("pfi feature importance returns data.frame", {
   
-  pfiTest <- pfi(plpResult, population, plpData, repeats = 1,
-                  covariates = NULL, cores = NULL, log = NULL,
-                  logthreshold = "INFO")
+  # limit to a sample of 2 covariates for faster test
+  covariates <- plpResult$model$covariateImportance %>% 
+    dplyr::filter("covariateValue" != 0) %>% 
+    dplyr::select("covariateId") %>% 
+    dplyr::arrange(desc("covariateValue")) %>%
+    dplyr::pull()
   
-  testthat::expect_equal(class(pfiTest), 'data.frame')
-  testthat::expect_equal(sum(names(pfiTest)%in%c("covariateId", "pfi")), 2)
+  # if the model had non-zero covariates
+  if(length(covariates) > 0){
+    if (length(covariates) > 2) {
+      covariates <- covariates[1:2]
+    }
+    pfiTest <- pfi(plpResult, population, plpData, repeats = 1,
+                   covariates = covariates, cores = 1, log = NULL,
+                   logthreshold = "INFO")
+    
+    expect_equal(class(pfiTest), 'data.frame')
+    expect_equal(sum(names(pfiTest) %in% c("covariateId", "pfi")), 2)
+    expect_true(all(!is.nan(pfiTest$pfi)))
+    
+  }
   
 })
 
+test_that('pfi feature importance works with logger or without covariates', {
+
+  pfiTest <- pfi(tinyResults, population, nanoData, cores = 1, 
+                 covariates = NULL, log = file.path(tempdir(), 'pfiLog'))
+  
+  expect_equal(class(pfiTest), 'data.frame')
+  expect_equal(sum(names(pfiTest) %in% c("covariateId", "pfi")), 2)
+  expect_true(all(!is.nan(pfiTest$pfi)))
+
+})
+
diff --git a/tests/testthat/test-fitting_updated.R b/tests/testthat/test-fitting.R
similarity index 78%
rename from tests/testthat/test-fitting_updated.R
rename to tests/testthat/test-fitting.R
index 8f1fb5639..ad22e92bf 100644
--- a/tests/testthat/test-fitting_updated.R
+++ b/tests/testthat/test-fitting.R
@@ -18,7 +18,6 @@ library("testthat")
 
 context("Fitting")
 
-trainData <- createTrainData(plpData, population)
 modelSettings <- setLassoLogisticRegression()
 
 test_that("fitPlp", {
@@ -27,7 +26,8 @@ plpModel <- fitPlp(
   trainData = trainData,   
   modelSettings = modelSettings,
   search = "grid", 
-  analysisId = 'fitting'
+  analysisId = 'fitting',
+  analysisPath = tempdir()
   )
 
 expect_is(plpModel, 'plpModel')
@@ -39,7 +39,8 @@ test_that("fitPlp input errors", {
   expect_error(
     fitPlp(
       trainData = trainData,   
-      modelSettings = modelSettings
+      modelSettings = modelSettings,
+      analysisPath = tempDir()
     )
   )
   
@@ -47,7 +48,8 @@ test_that("fitPlp input errors", {
     fitPlp(
       trainData = list(covariateData = NULL),   
       modelSettings = modelSettings,
-      analysisId = 'fitting'
+      analysisId = 'fitting',
+      analysisPath = tempDir()
     )
   )
   
@@ -55,6 +57,15 @@ test_that("fitPlp input errors", {
     fitPlp(
       trainData = trainData,   
       modelSettings = NULL,
+      analysisId = 'fitting',
+      analysisPath = tempDir()
+    )
+  )
+  
+  expect_error(
+    fitPlp(
+      trainData = trainData,   
+      modelSettings = modelSettings,
       analysisId = 'fitting'
     )
   )
diff --git a/tests/testthat/test-formatting_updated.R b/tests/testthat/test-formatting.R
similarity index 93%
rename from tests/testthat/test-formatting_updated.R
rename to tests/testthat/test-formatting.R
index 62c843b95..0e3ac0121 100644
--- a/tests/testthat/test-formatting_updated.R
+++ b/tests/testthat/test-formatting.R
@@ -58,7 +58,7 @@ test_that("MapIds with no cohort", {
   # some covariates not in data 5,6,7 so should be removed from covRef
   expect_equal(mappings$covariateRef %>% dplyr::tally() %>% dplyr::pull(), 7)
   
-  correctCov <- mappings$covariateRef %>% dplyr::select(.data$covariateId) %>% dplyr::pull() %in% c(123,2002,10,3,4,9,8)
+  correctCov <- mappings$covariateRef %>% dplyr::select("covariateId") %>% dplyr::pull() %in% c(123,2002,10,3,4,9,8)
   expect_equal(sum(correctCov), length(correctCov))
   
 })
@@ -92,7 +92,7 @@ test_that("MapIds with a cohort", {
   # some covariates not in data 5,6,7 so should be removed from covRef
   expect_equal(mappings$covariateRef %>% dplyr::tally() %>% dplyr::pull(), 3)
   
-  correctCov <- mappings$covariateRef %>% dplyr::select(.data$covariateId) %>% dplyr::pull() %in% c(123,9,8)
+  correctCov <- mappings$covariateRef %>% dplyr::select("covariateId") %>% dplyr::pull() %in% c(123,9,8)
   expect_equal(sum(correctCov), length(correctCov))
   
 })
@@ -102,7 +102,7 @@ test_that("toSparseM", {
   
   cohorts <- data.frame(rowId=1:6,   
                         subjectId=1:6, 
-                        cohortId=rep(1,6),
+                        targetId=rep(1,6),
                         cohortStartDate= rep('2007-12-28 00:00:00.0',6),
                         daysFromObsStart= c(500,50,500,500,500,500),
                         daysToCohortEnd= rep(200,6),
@@ -115,7 +115,7 @@ test_that("toSparseM", {
                                                          outcomes=2))
   
   outcomes <- data.frame(rowId=c(1,2), 
-                         outcomeId=rep(2,2), 
+                         outcomeId=rep(outcomeId,2), 
                          daysToEvent=c(150,40))
   
   FplpData <- list(cohorts=cohorts,
@@ -157,7 +157,7 @@ test_that("toSparseM", {
   compTest <- as.matrix(test$dataMatrix)
   testthat::expect_equal(test$labels %>% dplyr::tally() %>% dplyr::pull(), length(population$rowId))
   testthat::expect_equal(nrow(compTest), length(population$rowId))
-  testthat::expect_true(ncol(compTest) <= nrow(plpData$covariateData$covariateRef))
+  testthat::expect_true(ncol(compTest) <= plpData$covariateData$covariateRef %>% dplyr::tally() %>% dplyr::pull())
   testthat::expect_equal(ncol(compTest), test$covariateRef %>% dplyr::tally() %>% dplyr::pull())
   testthat::expect_equal(ncol(compTest), test$covariateMap %>% dplyr::tally() %>% dplyr::pull())
   
@@ -166,7 +166,7 @@ test_that("toSparseM", {
 
 test_that("checkRam", {
   
-  ramCheck <- checkRam(createCovariateData(), 0.8)
+  ramCheck <- checkRam(createCovariateData())
   expect_true(ramCheck)
   
 })
diff --git a/tests/testthat/test-getCalibration_updated.R b/tests/testthat/test-getCalibration.R
similarity index 97%
rename from tests/testthat/test-getCalibration_updated.R
rename to tests/testthat/test-getCalibration.R
index 831e2ebb9..522fb6b98 100644
--- a/tests/testthat/test-getCalibration_updated.R
+++ b/tests/testthat/test-getCalibration.R
@@ -37,7 +37,7 @@ test_that("getCalibration binary", {
   expect_true('evaluation' %in% colnames(calib))
   
   
-  calibBinary <-getCalibrationSummary_binary(
+  calibBinary <- getCalibrationSummary_binary(
     prediction = Eprediction,
     evalColumn = 'evaluation',
     numberOfStrata = 100,
diff --git a/tests/testthat/test-helperfunctions_updated.R b/tests/testthat/test-helperfunctions.R
similarity index 88%
rename from tests/testthat/test-helperfunctions_updated.R
rename to tests/testthat/test-helperfunctions.R
index 413b82a8d..526d6466b 100644
--- a/tests/testthat/test-helperfunctions_updated.R
+++ b/tests/testthat/test-helperfunctions.R
@@ -46,5 +46,12 @@ test_that("setPythonEnvironment", {
   testthat::expect_equal(class(setPythonEnvironment(envname='madeup34343', envtype = 'conda')), "character")
 })
 
+test_that("Borrowed cut2", {
+   x <- c(1, rep(2, 2), rep(4, 4), rep(5, 5), rep(6, 6)) 
+   groups <- PatientLevelPrediction:::cut2(x, g = 3)
+   expect_true(
+     all(levels(groups) == c("[1,5)", "5", "6"))
+   )
+})
 
 # getOs test?
diff --git a/tests/testthat/test-learningCurves_updated.R b/tests/testthat/test-learningCurves.R
similarity index 64%
rename from tests/testthat/test-learningCurves_updated.R
rename to tests/testthat/test-learningCurves.R
index e8b553274..15da60bcf 100644
--- a/tests/testthat/test-learningCurves_updated.R
+++ b/tests/testthat/test-learningCurves.R
@@ -19,11 +19,11 @@ context("LearningCurves")
 # learningCurve 
 learningCurve <- PatientLevelPrediction::createLearningCurve(
   plpData = plpData,
-  outcomeId = 2, parallel = T, cores = 3,
+  outcomeId = outcomeId, parallel = F, cores = -1,
   modelSettings = setLassoLogisticRegression(),
   saveDirectory =  file.path(saveLoc, 'lcc'),
-  splitSettings = createDefaultSplitSetting(testFraction = 0.2), 
-  trainFractions = c(0.6,0.7,0.8),
+  splitSettings = createDefaultSplitSetting(testFraction = 0.2, nfold=2), 
+  trainFractions = c(0.6,0.7),
   trainEvents = NULL,
   preprocessSettings = createPreprocessSettings(
     minFraction = 0.001,
@@ -42,7 +42,7 @@ test_that("learningCurve output correct", {
     "Train_populationSize",
     "Train_outcomeCount") ),5)
   
-  testthat::expect_equal(learningCurve$trainFraction,  c(0.6,0.7,0.8)*100)
+  testthat::expect_equal(learningCurve$trainFraction,  c(0.6,0.7)*100)
   
 })
 
@@ -64,4 +64,28 @@ test_that("plotLearningCurve", {
   
 })
 
+test_that("getTrainFractions works", {
+  
+  learningCurve <- PatientLevelPrediction::createLearningCurve(
+    plpData = tinyPlpData,
+    outcomeId = outcomeId, parallel = F, cores = -1,
+    modelSettings = setLassoLogisticRegression(seed = 42),
+    saveDirectory =  file.path(saveLoc, 'lcc'),
+    splitSettings = createDefaultSplitSetting(testFraction = 0.33, nfold = 2,
+                                              splitSeed = 42), 
+    trainEvents = c(150,200),
+    preprocessSettings = createPreprocessSettings(
+      minFraction = 0.001,
+      normalize = T
+    )
+  )
+  testthat::expect_true(is.data.frame(learningCurve))
+  testthat::expect_equal(sum(colnames(learningCurve) %in% c(
+    "trainFraction",
+    "Train_AUROC",
+    "nPredictors",
+    "Train_populationSize",
+    "Train_outcomeCount") ),5)
+  
+})
 
diff --git a/tests/testthat/test-multiplePlp_updated.R b/tests/testthat/test-multiplePlp.R
similarity index 59%
rename from tests/testthat/test-multiplePlp_updated.R
rename to tests/testthat/test-multiplePlp.R
index 09aa74adc..14cf651dc 100644
--- a/tests/testthat/test-multiplePlp_updated.R
+++ b/tests/testthat/test-multiplePlp.R
@@ -17,43 +17,29 @@
 library("testthat")
 context("MultiplePlp")
 
-connectionDetails <- Eunomia::getEunomiaConnectionDetails()
-Eunomia::createCohorts(connectionDetails)
-
-databaseDetails <- createDatabaseDetails(
-  connectionDetails = connectionDetails, 
-  cdmDatabaseSchema = "main", 
-  cdmDatabaseName = "main",
-  cohortDatabaseSchema = "main", 
-  cohortTable = "cohort", 
-  outcomeDatabaseSchema = "main", 
-  outcomeTable =  "cohort",
-  cohortId = 1, 
-  outcomeIds = 3, #make this ids
-  cdmVersion = 5)
-
-
 analysis1 <- createModelDesign(
   targetId = 1,
-  outcomeId = 3,
+  outcomeId = outcomeId,
   restrictPlpDataSettings = createRestrictPlpDataSettings(firstExposureOnly = F, washoutPeriod = 0),
   populationSettings = createStudyPopulationSettings(),
-  covariateSettings = FeatureExtraction::createDefaultCovariateSettings(),
+  covariateSettings = covariateSettings,
   featureEngineeringSettings = NULL,
   sampleSettings = NULL,
+  splitSettings = createDefaultSplitSetting(splitSeed = 1),
   preprocessSettings = createPreprocessSettings(),
   modelSettings = setLassoLogisticRegression(seed = 12)
 )
 
 test_that("createModelDesign - test working", {
   
-  
   expect_equal(analysis1$targetId, 1)
-  expect_equal(analysis1$outcomeId, 3)
+  expect_equal(analysis1$outcomeId, outcomeId)
   expect_equal(analysis1$restrictPlpDataSettings, createRestrictPlpDataSettings(firstExposureOnly = F, washoutPeriod = 0))
-  expect_equal(analysis1$covariateSettings, FeatureExtraction::createDefaultCovariateSettings())
-  expect_equal(analysis1$featureEngineeringSettings, createFeatureEngineeringSettings())
+  expect_equal(analysis1$covariateSettings, covariateSettings)
+  expect_equal(analysis1$featureEngineeringSettings, list(createFeatureEngineeringSettings(type= "none")))
+  expect_equal(analysis1$sampleSettings, list(createSampleSettings(type = 'none')))
   expect_equal(analysis1$preprocessSettings, createPreprocessSettings())
+  expect_equal(analysis1$splitSettings, createDefaultSplitSetting(splitSeed = 1))
   expect_equal(analysis1$modelSettings, setLassoLogisticRegression(seed = 12))
   expect_equal(
     analysis1$executeSettings, 
@@ -88,133 +74,47 @@ test_that("loading analyses settings", {
   expect_equal(analysis1$targetId, analysisSetting$analyses[[1]]$targetId)
   expect_equal(analysis1$outcomeId, analysisSetting$analyses[[1]]$outcomeId)
   expect_equal(analysis1$restrictPlpDataSettings, analysisSetting$analyses[[1]]$restrictPlpDataSettings)
-  expect_equal(attr(analysis1$covariateSettings, 'fun'), attr(analysisSetting$analyses[[1]]$covariateSettings[[1]],'fun') ) 
+  expect_equal(attr(analysis1$covariateSettings, 'fun'), attr(analysisSetting$analyses[[1]]$covariateSettings,'fun') ) 
   expect_equal(analysis1$populationSettings, analysisSetting$analyses[[1]]$populationSettings)
-  expect_equal(analysis1$sampleSettings, analysisSetting$analyses[[1]]$sampleSettings[[1]])
-  expect_equal(attr(analysis1$featureEngineeringSettings,'class'), attr(analysisSetting$analyses[[1]]$featureEngineeringSettings[[1]],'class'))
-  expect_equal(attr(analysis1$featureEngineeringSettings,'fun'), attr(analysisSetting$analyses[[1]]$featureEngineeringSettings[[1]],'fun'))
+  expect_equal(analysis1$sampleSettings, analysisSetting$analyses[[1]]$sampleSettings)
+  expect_equal(attr(analysis1$featureEngineeringSettings,'class'), attr(analysisSetting$analyses[[1]]$featureEngineeringSettings,'class'))
+  expect_equal(attr(analysis1$featureEngineeringSettings,'fun'), attr(analysisSetting$analyses[[1]]$featureEngineeringSettings,'fun'))
   expect_equal(analysis1$preprocessSettings, analysisSetting$analyses[[1]]$preprocessSettings)
   expect_equal(analysis1$modelSettings, analysisSetting$analyses[[1]]$modelSettings)
+  expect_equal(analysis1$splitSettings, analysisSetting$analyses[[1]]$splitSettings)
   expect_equal(analysis1$executeSettings, analysisSetting$analyses[[1]]$executeSettings)
-  
 }
 )
 
-analysis2 <- createModelDesign(
-  targetId = 10,
-  outcomeId = 2,
-  restrictPlpDataSettings = createRestrictPlpDataSettings(firstExposureOnly = F, washoutPeriod = 9999),
-  populationSettings = createStudyPopulationSettings(),
-  covariateSettings = FeatureExtraction::createCovariateSettings(useDemographicsAge = T),
-  featureEngineeringSettings = NULL,
-  sampleSettings = NULL,
-  preprocessSettings = createPreprocessSettings(),
-  modelSettings = setLassoLogisticRegression(seed = 12)
-)
-
-test_that("getSettingValues works", {
-  
-  # works for single setting:
-  result <- getSettingValues(
-    modelDesignList = list(analysis1), 
-    type = 'targetId' 
-    )
-  
-  expect_equal(nrow(result), 1)
-  expect_equal(result$value, 1)
-  
-  
-  # works for multiple setting:
-  result <- getSettingValues(
-    modelDesignList = list(analysis1, analysis2), 
-    type = 'targetId' 
-  )
-  
-  expect_equal(nrow(result), 2)
-  expect_equal(result$value, c(1,10))
-  
-  
-}
-  )
-
-
-test_that("getSettingValues works", {
-  
-  result <- getidList(modelDesignList = list(analysis1))
-  expect_is(result, 'list')
-  result <- getidList(modelDesignList = list(analysis1, analysis2))
-  expect_is(result, 'list')
-  expect_equal(nrow(result$targetId), 2)
-  expect_equal(nrow(result$covariateSettings), 2)
-}
-)
-
-test_that("getSettingFromId works", {
-  
-  result <- getidList(modelDesignList = list(analysis1, analysis2))
-
-  cov <- getSettingFromId(idList = result, type = 'covariateSettings', id = 1)
-  expect_equal(names(cov), names(FeatureExtraction::createDefaultCovariateSettings()))
-  
-  id <- getSettingFromId(idList = result, type = 'targetId', id = 10)
-  expect_equal(id, 10)
-
-})
-
-
-test_that("getSettingsTable", {
-  
-  result <- getidList(modelDesignList = list(analysis1, analysis2))
-
-  settingsTable <- getSettingsTable(
-    modelDesignList = list(analysis1, analysis2), 
-    idList = result
-    )
-  expect_is(settingsTable, 'data.frame')
-  
-})
-
-test_that("getDataSettings", {
-  
-  result <- getidList(modelDesignList = list(analysis1, analysis2))
-  
-  settingsTable <- getSettingsTable(
-    modelDesignList = list(analysis1, analysis2), 
-    idList = result
-  )
-  dataSettings <- getDataSettings(settingsTable)
-  expect_is(dataSettings, 'list')
-  expect_equal(length(dataSettings), 2)
-  
-})
-
 test_that("test run multiple", {
   
   analysis3 <- createModelDesign(
     targetId = 1,
-    outcomeId = 3,
+    outcomeId = outcomeId,
     restrictPlpDataSettings = createRestrictPlpDataSettings(firstExposureOnly = F, washoutPeriod = 0),
     populationSettings = createStudyPopulationSettings(),
-    covariateSettings = FeatureExtraction::createDefaultCovariateSettings(),
+    covariateSettings = covariateSettings,
     featureEngineeringSettings = createFeatureEngineeringSettings(),
     sampleSettings = createSampleSettings(),
     preprocessSettings = createPreprocessSettings(),
-    modelSettings = setLassoLogisticRegression(seed = 12)
+    modelSettings = setLassoLogisticRegression(seed = 12),
+    splitSettings = createDefaultSplitSetting(
+      type = "stratified", 
+      testFraction = 0.25,
+      trainFraction = 0.75, 
+      splitSeed = 123, 
+      nfold = 3
+    ),
+    runCovariateSummary = FALSE
   )
   
   runMultiplePlp(
     databaseDetails = databaseDetails,
     modelDesignList = list(
+      # add this twice to make sure no issue with overlapping ids?
       analysis3
     ),
     onlyFetchData = F,
-    splitSettings = createDefaultSplitSetting(
-      type = "stratified", 
-      testFraction = 0.25,
-      trainFraction = 0.75, 
-      splitSeed = 123, 
-      nfold = 3
-    ),
     logSettings = createLogSettings(
       verbosity = "DEBUG", 
       timeStamp = T, 
@@ -223,6 +123,7 @@ test_that("test run multiple", {
     saveDirectory = file.path(saveLoc, 'multiple')
   )
   
+  expect_true(file.exists(file.path(saveLoc, 'multiple', 'settings.csv')))
   expect_true(dir.exists(file.path(saveLoc, 'multiple', 'Analysis_1')))
   expect_true(file.exists(file.path(saveLoc, 'multiple', 'Analysis_1','plpResult', 'runPlp.rds')))
   
diff --git a/tests/testthat/test-paramchecks_updated.R b/tests/testthat/test-paramchecks.R
similarity index 100%
rename from tests/testthat/test-paramchecks_updated.R
rename to tests/testthat/test-paramchecks.R
diff --git a/tests/testthat/test-plotting_updated.R b/tests/testthat/test-plotting.R
similarity index 94%
rename from tests/testthat/test-plotting_updated.R
rename to tests/testthat/test-plotting.R
index a3989c9b6..17039166e 100644
--- a/tests/testthat/test-plotting_updated.R
+++ b/tests/testthat/test-plotting.R
@@ -38,7 +38,7 @@ test_that("plots", {
   test <- plotF1Measure(plpResult, typeColumn = 'evaluation')
   testthat::expect_s3_class(test, 'arrangelist')
   
-  if(!is.null(plpResult$performanceEvaluation$demographicSummary)){
+  if (!is.null(plpResult$performanceEvaluation$demographicSummary)) {
     test <- plotDemographicSummary(plpResult, typeColumn = 'evaluation')
     testthat::expect_s3_class(test, 'arrangelist')
   }
@@ -52,7 +52,7 @@ test_that("plots", {
   test <- plotVariableScatterplot(plpResult$covariateSummary)
   testthat::expect_s3_class(test, 'ggplot')
   
-  test <- plotGeneralizability(plpResult$covariateSummary, fileName=NULL)
+  test <- plotGeneralizability(plpResult$covariateSummary, fileName = NULL)
   testthat::expect_s3_class(test, 'grob')
 
 })
@@ -61,7 +61,7 @@ test_that("plots", {
 test_that("outcomeSurvivalPlot", {
   
   # test the plot works
-  test <- outcomeSurvivalPlot(plpData = plpData, outcomeId = 2)
+  test <- outcomeSurvivalPlot(plpData = plpData, outcomeId = outcomeId)
   testthat::expect_s3_class(test, 'ggsurvplot')
   
   testthat::expect_error(outcomeSurvivalPlot())
@@ -82,7 +82,7 @@ test_that("plotPlp", {
   testthat::expect_equal(dir.exists(file.path(saveLoc,'plots')), T)
   
   # expect plots to be there
-  expect_true(length(dir(file.path(saveLoc,'plots')))>0)
+  expect_true(length(dir(file.path(saveLoc,'plots'))) > 0)
   
 })
 
@@ -134,10 +134,4 @@ test_that("plotSmoothCalibration", {
     )
   )
   
-})
-
-
-
-
-
-
+})
\ No newline at end of file
diff --git a/tests/testthat/test-population_updated.R b/tests/testthat/test-population.R
similarity index 90%
rename from tests/testthat/test-population_updated.R
rename to tests/testthat/test-population.R
index 124b2d17e..6add83fee 100644
--- a/tests/testthat/test-population_updated.R
+++ b/tests/testthat/test-population.R
@@ -293,7 +293,7 @@ test_that("population creation parameters", {
   
   studyPopulation <- createStudyPopulation(
     plpData = plpData,
-    outcomeId = 3,
+    outcomeId = outcomeId,
     populationSettings = defaultSettings()
   )
   
@@ -306,7 +306,7 @@ test_that("population creation parameters", {
   #firstExposureOnly test (should have no effect on simulated data)
   studyPopulation <- createStudyPopulation(
     plpData = plpData,
-    outcomeId = 3,
+    outcomeId = outcomeId,
     populationSettings = defaultSettings(firstExposureOnly = T)
     )
   
@@ -317,7 +317,7 @@ test_that("population creation parameters", {
   #requireTimeAtRisk
   studyPopulation <- createStudyPopulation(
     plpData = plpData,
-    outcomeId = 3, 
+    outcomeId = outcomeId, 
     populationSettings = defaultSettings(requireTimeAtRisk = T)
     )
                                            
@@ -330,7 +330,7 @@ test_that("population creation parameters", {
   expect_warning(
     createStudyPopulation(
       plpData = plpData,
-      outcomeId = 3, 
+      outcomeId = outcomeId, 
       populationSettings = defaultSettings(requireTimeAtRisk = T, minTimeAtRisk = 99999)
     )
   )
@@ -338,7 +338,7 @@ test_that("population creation parameters", {
   #washoutPeriod = 365,
   studyPopulation <- createStudyPopulation(
     plpData = plpData,
-    outcomeId = 3, 
+    outcomeId = outcomeId, 
     populationSettings = defaultSettings(washoutPeriod = 365)
   )
   nrOutcomes4 <- sum(studyPopulation$outcomeCount)
@@ -349,7 +349,7 @@ test_that("population creation parameters", {
   expect_error(
     createStudyPopulation(
       plpData = plpData,
-      outcomeId = 3, 
+      outcomeId = outcomeId, 
       populationSettings = defaultSettings(washoutPeriod = -1)
     )
   )
@@ -358,7 +358,7 @@ test_that("population creation parameters", {
   expect_error(
     createStudyPopulation(
       plpData = plpData,
-      outcomeId = 3, 
+      outcomeId = outcomeId, 
       populationSettings = defaultSettings(
         priorOutcomeLookback = -1, 
         removeSubjectsWithPriorOutcome = T
@@ -370,7 +370,7 @@ test_that("population creation parameters", {
   expect_error(
     createStudyPopulation(
       plpData = plpData,
-      outcomeId = 3, 
+      outcomeId = outcomeId, 
       populationSettings = defaultSettings(
         minTimeAtRisk  = -1, 
         requireTimeAtRisk = T
@@ -382,7 +382,7 @@ test_that("population creation parameters", {
   expect_error(
     createStudyPopulation(
       plpData = plpData,
-      outcomeId = 3, 
+      outcomeId = outcomeId, 
       populationSettings = defaultSettings(
         startAnchor = 'cohort stard'
       )
@@ -394,7 +394,7 @@ test_that("population creation parameters", {
   expect_error(
     createStudyPopulation(
       plpData = plpData,
-      outcomeId = 3, 
+      outcomeId = outcomeId, 
       populationSettings = defaultSettings(
         endAnchor = 'cohort ent'
       )
@@ -411,7 +411,7 @@ test_that("population creation parameters", {
   )
   cohorts <- data.frame(rowId=1:20, 
                         subjectId=1:20, 
-                        cohortId=rep(2,20),
+                        targetId=rep(2,20),
                         time=rep(365,20),
                         ageYear = rep(18,20),
                         gender = rep(8507,20),
@@ -495,4 +495,31 @@ test_that("population creation parameters", {
   
 })
 
+testthat::test_that("Providing an existing population and skipping population creation works", {
+  popSize <- 400
+  newPopulation <- population[sample.int(nrow.default(population), popSize), ]
+  
+  tinyPlpData$population <- newPopulation
+  
+  plpResults <- runPlp(
+    plpData = tinyPlpData,
+    outcomeId = 2,
+    analysisId = "1",
+    analysisName = "existing population",
+    populationSettings = createStudyPopulationSettings(),
+    splitSettings = createDefaultSplitSetting(),
+    modelSettings = setLassoLogisticRegression(),
+    executeSettings = createExecuteSettings(
+      runSplitData = TRUE,
+      runPreprocessData = FALSE,
+      runModelDevelopment = TRUE
+    )
+  )
+  
+  trainPredictions <- plpResults$prediction %>%
+    dplyr::filter(.data$evaluationType == "Train") %>% nrow.default()
+  testPredictions <- plpResults$prediction %>%
+    dplyr::filter(.data$evaluationType == "Test") %>% nrow.default()
+  expect_equal(popSize, trainPredictions + testPredictions)
+})
 
diff --git a/tests/testthat/test-prediction_updated.R b/tests/testthat/test-prediction.R
similarity index 94%
rename from tests/testthat/test-prediction_updated.R
rename to tests/testthat/test-prediction.R
index dbf5f65d6..85aecf052 100644
--- a/tests/testthat/test-prediction_updated.R
+++ b/tests/testthat/test-prediction.R
@@ -57,7 +57,7 @@ test_that("prediction works", {
     )
   
   # check metaData
-  expect_equal(length(names(attr(pred, "metaData"))), 4)  # 6 if survivial
+  expect_equal(length(names(attr(pred, "metaData"))), 6)  # 8 if survivial
   
   # add single person pred and compare with manual cal
   
@@ -70,7 +70,7 @@ test_that("prediction works", {
 
 test_that("applyTidyCovariateData", {
   
-  covariateIds <- plpData$covariateData$covariateRef %>% dplyr::select(.data$covariateId) %>% dplyr::pull()
+  covariateIds <- plpData$covariateData$covariateRef %>% dplyr::select("covariateId") %>% dplyr::pull()
   remove <- sample(covariateIds, 10)
   deletedRedundantCovariateIds = remove[1:5]
   deletedInfrequentCovariateIds = remove[6:10]
@@ -95,7 +95,7 @@ test_that("applyTidyCovariateData", {
   # some covariates removed
   expect_true(newCovariateData$covariates %>% dplyr::tally() %>% dplyr::pull() < covariateCount)
   
-  newCovs <- newCovariateData$covariateRef %>% dplyr::select(.data$covariateId) %>% dplyr::pull()
+  newCovs <- newCovariateData$covariateRef %>% dplyr::select("covariateId") %>% dplyr::pull()
   
   expect_equal(sum(covariateIds[!covariateIds %in% newCovs] %in% remove),10)
   
diff --git a/tests/testthat/test-preprocessingData_updated.R b/tests/testthat/test-preprocessingData.R
similarity index 86%
rename from tests/testthat/test-preprocessingData_updated.R
rename to tests/testthat/test-preprocessingData.R
index ca8259ecc..9b3755b72 100644
--- a/tests/testthat/test-preprocessingData_updated.R
+++ b/tests/testthat/test-preprocessingData.R
@@ -58,11 +58,10 @@ test_that("createPreprocessSettings", {
 })
 
 test_that("createPreprocessSettings", {
-  trainData <- createTrainData(plpData, population)
-  
+  attr(trainData$covariateData, "metaData")$preprocessSettings <- NULL # removing for test
   metaData <- attr(trainData$covariateData, "metaData")
-  covSize <- trainData$covariateData$covariates %>% dplyr::tally() %>% dplyr::pull()
   metaLength <- length(metaData)
+  covSize <- trainData$covariateData$covariates %>% dplyr::tally() %>% dplyr::pull()
   oldFeatureCount <- trainData$covariateData$covariateRef %>% dplyr::tally() %>% dplyr::pull()
   
   preprocessSettings <- createDefaultSettings(
@@ -73,11 +72,10 @@ test_that("createPreprocessSettings", {
   newData <- preprocessData(trainData$covariateData, preprocessSettings)
   
   expect_is(newData, 'CovariateData')
-  expect_equal(length(attr(newData, "metaData")), 1+length(metaData))
   expect_true(newData$covariates %>% dplyr::tally() %>% dplyr::pull() < covSize)
   
-  # metaData should have tidyCovariateDataSettings (so 1 bigger)
-  expect_equal(length(attr(newData, "metaData")), metaLength+1)
+  # metaData should have tidyCovariateDataSettings + preprocessSettings (so 2 bigger)
+  expect_equal(length(attr(newData, "metaData")), metaLength+2)
   
   expect_true(length(attr(newData, "metaData")$tidyCovariateDataSettings$deletedInfrequentCovariateIds)>=0)
   expect_equal(attr(newData, "metaData")$tidyCovariateDataSettings$deletedRedundantCovariateIds, NULL)
@@ -87,7 +85,8 @@ test_that("createPreprocessSettings", {
   
   expect_equal(newFeatureCount, oldFeatureCount)
   
-  trainData <- createTrainData(plpData, population)
+  oldFeatureCount <- trainData$covariateData$covariateRef %>% dplyr::tally() %>% dplyr::pull()
+  
   metaData <- attr(trainData$covariateData, "metaData")
   preprocessSettings <- createDefaultSettings(
     minCovariateFraction = 0,
@@ -102,6 +101,13 @@ test_that("createPreprocessSettings", {
   newFeatureCount <- newData$covariateRef %>% dplyr::tally() %>% dplyr::pull() + 
     length(attr(newData, "metaData")$tidyCovariateDataSettings$deletedRedundantCovariateIds) 
   
-  expect_equal(newFeatureCount, oldFeatureCount)
+  expect_equal(newFeatureCount, oldFeatureCount) # sometimes differ?
+  
+  # check settings are saved
+  expect_equal(attr(newData, "metaData")$preprocessSettings, preprocessSettings)
   
 })
+
+test_that('Did tidy on test', {
+  expect_true(attr(plpResult$prediction, 'metaData')$tidyCovariates)
+})
diff --git a/tests/testthat/test-rclassifier_updated.R b/tests/testthat/test-rclassifier.R
similarity index 93%
rename from tests/testthat/test-rclassifier_updated.R
rename to tests/testthat/test-rclassifier.R
index 0781a8301..5a0dadc31 100644
--- a/tests/testthat/test-rclassifier_updated.R
+++ b/tests/testthat/test-rclassifier.R
@@ -41,7 +41,7 @@ test_that("GBM settings work", {
   expect_equal(gbmSet$fitFunction, 'fitRclassifier')
   expect_is(gbmSet$param, 'list')
   
-  expect_equal(attr(gbmSet$param, 'settings')$modeType, 'Xgboost')
+  expect_equal(attr(gbmSet$param, 'settings')$modelType, 'Xgboost')
   expect_equal(attr(gbmSet$param, 'settings')$seed, seed)
   expect_equal(attr(gbmSet$param, 'settings')$modelName, "Gradient Boosting Machine")
   
@@ -88,13 +88,13 @@ testthat::expect_error(setGradientBoostingMachine(scalePosWeight = -1))
 
 test_that("GBM working checks", {
   
-  trainData <- createTrainData(plpData = plpData, population = population)
   modelSettings <- setGradientBoostingMachine(ntrees = 10, maxDepth = 3, learnRate = 0.1)
   
   fitModel <- fitPlp(
     trainData = trainData,   
     modelSettings = modelSettings, 
-    analysisId = 'gbmTest'
+    analysisId = 'gbmTest',
+    analysisPath = tempdir()
   )
   
   expect_equal(nrow(fitModel$prediction), nrow(trainData$labels)*2)
@@ -108,8 +108,9 @@ test_that("GBM working checks", {
   
   expect_lte(nrow(fitModel$covariateImportance), trainData$covariateData$covariateRef %>% dplyr::tally() %>% dplyr::pull())
   
-  expect_equal(fitModel$trainDetails$outcomeId, 2)
-  expect_equal(fitModel$trainDetails$cohortId, 1)
+  expect_equal(fitModel$modelDesign$outcomeId, outcomeId)
+  expect_equal(fitModel$modelDesign$targetId, 1)
+  # TODO check other model design values?
   
   # test that at least some features have importances that are not zero
   expect_equal(sum(abs(fitModel$covariateImportance$covariateValue))>0, TRUE)
diff --git a/tests/testthat/test-recalibration_updated.R b/tests/testthat/test-recalibration.R
similarity index 76%
rename from tests/testthat/test-recalibration_updated.R
rename to tests/testthat/test-recalibration.R
index ba992bd76..33e717197 100644
--- a/tests/testthat/test-recalibration_updated.R
+++ b/tests/testthat/test-recalibration.R
@@ -31,8 +31,8 @@ prediction <- data.frame(
 
 metaData <- list(
   modelType = "binary", 
-  cohortId = 1,
-  outcomeId = 2,
+  targetId = 1,
+  outcomeId = outcomeId,
   timepoint = 365
 )
 
@@ -62,17 +62,28 @@ testthat::expect_true(sum(test$evaluationType == 'weakRecalibration') == 100)
 
 
 test_that("recalibratePlpRefit", {
+  
+  newPop <- plpResult$prediction %>% dplyr::select(-"value") %>% dplyr::filter(.data$evaluationType %in% c('Test','Train'))
+  attr(newPop, 'metaData') <- list(
+    targetId = 1, 
+    outcomeId = outcomeId,
+    restrictPlpDataSettings = PatientLevelPrediction::createRestrictPlpDataSettings(),
+    populationSettings = PatientLevelPrediction::createStudyPopulationSettings()
+  )
+  
   testRecal <- recalibratePlpRefit(
     plpModel = plpResult$model, 
-    newPopulation = plpResult$prediction %>% dplyr::select(-.data$value) %>% dplyr::filter(.data$evaluationType %in% c('Test','Train')), 
+    newPopulation = newPop, 
     newData = plpData
   )
   
-  testthat::expect_true(
-    sum(testRecal$evaluationType == 'recalibrationRefit')>0
-  )
-  
-  testthat::expect_s3_class(testRecal, 'data.frame')
+  if(!is.null(testRecal)){
+    testthat::expect_true(
+      sum(testRecal$evaluationType == 'recalibrationRefit')>0
+    )
+    
+    testthat::expect_s3_class(testRecal, 'data.frame')
+  }
   
   # add more test...
 })
@@ -83,8 +94,8 @@ test_that("survival", {
 # survival
 metaData <- list(
   modelType = "survival", 
-  cohortId = 1,
-  outcomeId = 2,
+  targetId = 1,
+  outcomeId = outcomeId,
   timepoint = 365
 )
 
diff --git a/tests/testthat/test-runPlpHelpers_updated.R b/tests/testthat/test-runPlpHelpers.R
similarity index 97%
rename from tests/testthat/test-runPlpHelpers_updated.R
rename to tests/testthat/test-runPlpHelpers.R
index 38e36b423..4d361410d 100644
--- a/tests/testthat/test-runPlpHelpers_updated.R
+++ b/tests/testthat/test-runPlpHelpers.R
@@ -21,8 +21,8 @@ test_that("check printHeader runs", {
   
   header <- printHeader(
     plpData = plpData, 
-    cohortId = 1, 
-    outcomeId = 2, 
+    targetId = 1, 
+    outcomeId = outcomeId, 
     analysisId = 123, 
     analysisName = 'test', 
     ExecutionDateTime = Sys.time()
@@ -36,7 +36,7 @@ test_that("checkInputs", {
   check <- checkInputs(
     list(
       plpData = plpData, 
-      outcomeId = 2, 
+      outcomeId = outcomeId, 
       populationSettings = populationSettings
     )
   )
@@ -49,7 +49,7 @@ test_that("checkInputs", {
     checkInputs(
       list(
         plpData = NULL, 
-        outcomeId = 2, 
+        outcomeId = outcomeId, 
         populationSettings = populationSettings
       )
     )
@@ -71,7 +71,7 @@ test_that("checkInputs", {
     checkInputs(
       list(
         plpData = plpData, 
-        outcomeId = 2, 
+        outcomeId = outcomeId, 
         populationSettings = 'populationSettings'
       )
     )
diff --git a/tests/testthat/test-sampling_updated.R b/tests/testthat/test-sampling.R
similarity index 63%
rename from tests/testthat/test-sampling_updated.R
rename to tests/testthat/test-sampling.R
index b6603a633..967dd6dc6 100644
--- a/tests/testthat/test-sampling_updated.R
+++ b/tests/testthat/test-sampling.R
@@ -62,8 +62,12 @@ test_that("createSampleSettings works", {
     testNumberOutcomestoNonOutcomes
     )
   
+  # the seed is ignored if sameData 
+  if(testType == 'none'){
+    testSampleSeed <- 1
+  }
   expect_equal(
-    sampleSettings$sampleSeed , 
+    sampleSettings$sampleSeed, 
     testSampleSeed 
   )
   
@@ -95,37 +99,38 @@ test_that("createSampleSettings expected errors", {
 
 test_that("sampleData outputs are correct", {
   
-  trainData <- createTrainData(plpData, population)
+  newTrainData <- trainData
+  attr(newTrainData, "metaData")$sampleSettings <- NULL # remove for test
   
   sampleSettings <- sampleSettingFunc(type = 'none') 
   
-  sampleData <- sampleData(trainData, sampleSettings)
+  sampleData <- sampleData(newTrainData, sampleSettings)
   
   # make sure metaData captures
   expect_equal(
     length(attr(sampleData, "metaData")),
-    length(attr(trainData, "metaData"))+1
+    length(attr(newTrainData, "metaData"))+1
   )
   
   expect_equal(
-    attr(sampleData, "metaData")$sampleSettings,
+    attr(sampleData, "metaData")$sampleSettings[[1]],
     sampleSettings
   )
   
   # check the data is the same:
   expect_equal(
     nrow(sampleData$labels), 
-    nrow(trainData$labels)
+    nrow(newTrainData$labels)
   )
   
   expect_equal(
     nrow(sampleData$folds), 
-    nrow(trainData$folds)
+    nrow(newTrainData$folds)
   )
   
   expect_equal(
     sampleData$covariateData$covariates %>% dplyr::tally() %>% dplyr::pull(), 
-    trainData$covariateData$covariates  %>% dplyr::tally() %>% dplyr::pull()
+    newTrainData$covariateData$covariates  %>% dplyr::tally() %>% dplyr::pull()
   )
   
   
@@ -136,7 +141,7 @@ test_that("sampleData outputs are correct", {
  
 test_that("underSampleData works", {
   
-  trainData <- createTrainData(plpData, population)
+  newTrainData <- trainData
   
   sampleSettings <- list(
     sampleSeed = 1,
@@ -145,13 +150,18 @@ test_that("underSampleData works", {
   
   underSampleData <- underSampleData(trainData, sampleSettings)
   
+  expect_true(inherits(underSampleData, 'plpData')) # add test based on github issue
+  
   # the sampled data should be smaller...
-  expect_true(nrow(underSampleData$labels) <= nrow(trainData$labels))
+  expect_true(nrow(underSampleData$labels) <= nrow(newTrainData$labels))
   
-  expect_true(nrow(underSampleData$folds) <= nrow(trainData$folds))
+  expect_true(nrow(underSampleData$folds) <= nrow(newTrainData$folds))
   
   expect_true(
-    underSampleData$covariateData$covariates %>% dplyr::tally() %>% dplyr::pull() <= trainData$covariateData$covariates  %>% dplyr::tally() %>% dplyr::pull()
+    underSampleData$covariateData$covariates %>% 
+      dplyr::tally() %>% 
+      dplyr::pull() <= newTrainData$covariateData$covariates  %>% 
+      dplyr::tally() %>% dplyr::pull()
   )
   
   # perhaps add manual data test
@@ -159,3 +169,32 @@ test_that("underSampleData works", {
   
 })
 
+
+test_that("overSampleData works", {
+  
+  newTrainData <- trainData
+  
+  sampleSettings <- list(
+    sampleSeed = 1,
+    numberOutcomestoNonOutcomes = 0.5
+  )
+  
+  overSampleData <- overSampleData(newTrainData, sampleSettings)
+  
+  expect_true(inherits(overSampleData, 'plpData')) # add test based on github issue
+  
+  # the sampled data should be smaller...
+  expect_true(nrow(overSampleData$labels) >= nrow(newTrainData$labels))
+  
+  expect_true(nrow(overSampleData$folds) >= nrow(newTrainData$folds))
+  
+  expect_true(
+    overSampleData$covariateData$covariates %>% dplyr::tally() %>% 
+      dplyr::pull() >= newTrainData$covariateData$covariates  %>% 
+      dplyr::tally() %>% dplyr::pull()
+  )
+  
+  # perhaps add manual data test
+  
+  
+})
diff --git a/tests/testthat/test-saveloadplp_updated.R b/tests/testthat/test-saveloadplp.R
similarity index 75%
rename from tests/testthat/test-saveloadplp_updated.R
rename to tests/testthat/test-saveloadplp.R
index 5f4fdc794..ed71009ce 100644
--- a/tests/testthat/test-saveloadplp_updated.R
+++ b/tests/testthat/test-saveloadplp.R
@@ -41,12 +41,12 @@ test_that("loadPlpDataError", {
 })
 
 test_that("loadPlpData", {
-  plpData <- loadPlpData(file = file.path(saveLoc,"saveDataTest"))
-  expect_identical(plpData$cohorts, oldCohorts)
-  expect_identical(plpData$outcomes, oldOutcomes)
-  expect_equal(as.data.frame(plpData$covariateData$covariates), 
+  loadedData <- loadPlpData(file = file.path(saveLoc,"saveDataTest"))
+  expect_identical(loadedData$cohorts, oldCohorts)
+  expect_identical(loadedData$outcomes, oldOutcomes)
+  expect_equal(as.data.frame(loadedData$covariateData$covariates), 
                          oldCovariates)
-  expect_equal(as.data.frame(plpData$covariateData$covariateRef), 
+  expect_equal(as.data.frame(loadedData$covariateData$covariateRef), 
                          oldCovariateRef)
 })
 
@@ -186,3 +186,53 @@ test_that("savePlpShareable works", {
   )
   
 })
+
+# Note: saving from database to csv is in the database upload test file
+
+
+
+test_that("applyMinCellCount works", {
+  
+  result <- data.frame(
+    performance_id = 1:2,
+    covariate_id = 1:2,
+    covariate_name = paste0('name', 1:2),
+    concept_id = 1:2,
+    covariate_value = runif(2),
+    covariate_count = c(100,50),
+    covariate_mean = runif(2),
+    covariate_st_dev = runif(2),
+    with_no_outcome_covariate_count = c(10,5),
+    with_no_outcome_covariate_mean = runif(2),
+    with_no_outcome_covariate_st_dev = runif(2),
+    with_outcome_covariate_count = c(90,45),
+    with_outcome_covariate_mean = runif(2),
+    with_outcome_covariate_st_dev = runif(2),
+    standardized_mean_diff = runif(2)
+  )
+  
+  minCellResult <- applyMinCellCount(
+    tableName = "covariate_summary",
+    sensitiveColumns = getPlpSensitiveColumns(),
+    result = result,
+    minCellCount = 5
+  )
+   # check nothing removed 
+  testthat::expect_equal(2,sum(minCellResult$covariate_count != -1))
+  testthat::expect_equal(2,sum(minCellResult$with_no_outcome_covariate_count != -1))
+  testthat::expect_equal(2,sum(minCellResult$with_outcome_covariate_count != -1))
+  
+  # now check values are removed
+  minCellResult <- applyMinCellCount(
+    tableName = "covariate_summary",
+    sensitiveColumns = getPlpSensitiveColumns(),
+    result = result,
+    minCellCount = 10
+  )
+  testthat::expect_equal(0,sum(minCellResult$covariate_count == -1))
+  testthat::expect_equal(minCellResult$with_no_outcome_covariate_count[2],-1)
+  testthat::expect_equal(1,sum(minCellResult$with_no_outcome_covariate_count == -1))
+  testthat::expect_equal(1,sum(minCellResult$with_outcome_covariate_count == -1))
+  
+  
+})
diff --git a/tests/testthat/test-sklearnClassifier.R b/tests/testthat/test-sklearnClassifier.R
new file mode 100644
index 000000000..c87a1937b
--- /dev/null
+++ b/tests/testthat/test-sklearnClassifier.R
@@ -0,0 +1,205 @@
+
+
+test_that("DecisionTree settings work checks", {
+  
+dtset <- setDecisionTree(
+  criterion = list('gini'),
+  splitter = list('best'),
+  maxDepth = list(4, 10, NULL),
+  minSamplesSplit = list(2, 10),
+  minSamplesLeaf = list(10, 50),
+  minWeightFractionLeaf = list(0),
+  maxFeatures = list(100,'sqrt', NULL),
+  maxLeafNodes = list(NULL),
+  minImpurityDecrease = list(10^-7),
+  classWeight = list(NULL),
+  seed = sample(1000000,1)
+)
+
+expect_equal(dtset$fitFunction, "fitSklearn")
+
+expect_equal(length(dtset$param), 3*2*2*3*1)
+
+expect_equal(unique(unlist(lapply(dtset$param, function(x) x[[1]]))), 'gini')
+expect_equal(unique(unlist(lapply(dtset$param, function(x) x[[2]]))), 'best')
+expect_equal(length(unique(lapply(dtset$param, function(x) x[[3]]))), 3)
+
+expect_false(attr(dtset$param, 'settings')$requiresDenseMatrix)
+expect_equal(attr(dtset$param, 'settings')$name, 'Decision Tree')
+expect_equal(attr(dtset$param, 'settings')$pythonModule, 'sklearn.tree')
+expect_equal(attr(dtset$param, 'settings')$pythonClass, "DecisionTreeClassifier")
+
+
+})
+
+
+test_that("DecisionTree errors as expected", {
+  
+  expect_error(setDecisionTree(criterion = list('madeup')))
+  
+  expect_error(setDecisionTree(maxDepth =  list(-1)))
+  expect_error(setDecisionTree(minSamplesSplit =  list(-1)))
+  expect_error(setDecisionTree(minSamplesLeaf  =  list(-1)))
+  
+})
+
+
+test_that("check fit of DecisionTree", {
+  
+  modelSettings <- setDecisionTree(
+    criterion = list('gini'),
+    splitter = list('best'),
+    maxDepth = list(as.integer(4)),
+    minSamplesSplit = list(2),
+    minSamplesLeaf = list(10),
+    minWeightFractionLeaf = list(0),
+    maxFeatures = list('sqrt'),
+    maxLeafNodes = list(NULL),
+    minImpurityDecrease = list(10^-7),
+    classWeight = list(NULL),
+    seed = sample(1000000,1)
+  )
+  
+  plpModel <- fitPlp(
+    trainData = tinyTrainData, 
+    modelSettings = modelSettings,
+    analysisId = 'DecisionTree',
+    analysisPath = tempdir()
+    )
+  
+  expect_correct_fitPlp(plpModel, trainData)
+  # add check for other model design settings
+  
+})
+
+test_that('fitSklearn errors with wrong covariateData', {
+  
+  newTrainData <- copyTrainData(trainData)
+  class(newTrainData$covariateData) <- 'notCovariateData'
+  modelSettings <- setAdaBoost()
+  analysisId <- 42
+  
+  expect_error(fitSklearn(newTrainData,
+                          modelSettings,
+                          search='grid',
+                          analysisId))
+})
+
+
+test_that('AdaBoost fit works', {
+  
+  modelSettings <- setAdaBoost(nEstimators = list(10),
+                               learningRate = list(0.1),
+                               )
+  
+  plpModel <- fitPlp(
+    trainData = tinyTrainData, 
+    modelSettings = modelSettings,
+    analysisId = 'Adaboost',
+    analysisPath = tempdir()
+  )
+  
+  expect_correct_fitPlp(plpModel, trainData)
+  expect_equal(dir(plpModel$model),"model.json")
+  
+})
+
+
+test_that('RandomForest fit works', {
+  
+  modelSettings <- setRandomForest(ntrees=list(10),
+                                   maxDepth=list(4),
+                                   minSamplesSplit = list(2),
+                                   minSamplesLeaf = list(10),
+                                   mtries = list("sqrt"),
+                                   maxSamples = list(0.9),
+                                   classWeight = list(NULL))
+  
+  plpModel <- fitPlp(
+    trainData = tinyTrainData, 
+    modelSettings = modelSettings,
+    analysisId = 'RandomForest',
+    analysisPath = tempdir()
+  )
+  
+  expect_correct_fitPlp(plpModel, trainData)
+  expect_equal(dir(plpModel$model),"model.json")
+  
+})
+
+
+test_that('MLP fit works', {
+  modelSettings <- setMLP(
+    hiddenLayerSizes = list(c(20)),
+    alpha = list(1e-6),
+    maxIter = list(50),
+    epsilon = list(1e-08),
+    learningRateInit = list(0.01),
+    tol = list(1e-2) # reduce tol so I don't get convergence warnings
+  )
+  
+  plpModel <- fitPlp(
+    trainData = tinyTrainData, 
+    modelSettings = modelSettings,
+    analysisId = 'MLP',
+    analysisPath = tempdir()
+  )
+  
+  expect_correct_fitPlp(plpModel, trainData)
+  expect_equal(dir(plpModel$model),"model.json")
+  
+})
+
+
+test_that('Naive bayes fit works', {
+  modelSettings <- setNaiveBayes()
+  
+  plpModel <- fitPlp(
+    trainData = tinyTrainData, 
+    modelSettings = modelSettings,
+    analysisId = 'Naive bayes',
+    analysisPath = tempdir()
+  )
+  
+  expect_correct_fitPlp(plpModel, trainData) 
+  expect_equal(dir(plpModel$model),"model.json")
+  
+})
+
+
+test_that('Support vector machine fit works', {
+  modelSettings <- setSVM(C = list(1),
+                          degree = list(1),
+                          gamma = list('scale'),
+                          classWeight = list(NULL))
+  
+  plpModel <- fitPlp(
+    trainData = tinyTrainData, 
+    modelSettings = modelSettings,
+    analysisId = 'SVM',
+    analysisPath = tempdir()
+  )
+  
+  expect_correct_fitPlp(plpModel, trainData) 
+  expect_equal(dir(plpModel$model),"model.json")
+  
+})
+
+test_that('Sklearn predict works', {
+  
+  modelSettings <- setAdaBoost(nEstimators = list(10),
+                               learningRate = list(0.1),
+  )
+  
+  plpModel <- fitPlp(
+    trainData = tinyTrainData, 
+    modelSettings = modelSettings,
+    analysisId = 'Adaboost',
+    analysisPath = tempdir()
+  )
+  
+  predictions <- predictPythonSklearn(plpModel,
+                                      testData,
+                                      population)
+  expect_correct_predictions(predictions, testData)
+})
diff --git a/tests/testthat/test-sklearnClassifierHelpers_updated.R b/tests/testthat/test-sklearnClassifierHelpers.R
similarity index 100%
rename from tests/testthat/test-sklearnClassifierHelpers_updated.R
rename to tests/testthat/test-sklearnClassifierHelpers.R
diff --git a/tests/testthat/test-sklearnClassifierSettings.R b/tests/testthat/test-sklearnClassifierSettings.R
index 9b380f595..29c3f7f3d 100644
--- a/tests/testthat/test-sklearnClassifierSettings.R
+++ b/tests/testthat/test-sklearnClassifierSettings.R
@@ -11,22 +11,20 @@ test_that("setAdaBoost settings work checks", {
   
   expect_equal(length(adset$param), 3*3*1)
   
-  expect_equal(unique(unlist(lapply(adset$param, function(x) x[[1]]))), NULL)
-  expect_equal(unique(unlist(lapply(adset$param, function(x) x[[2]]))), c(10,50, 200))
-  expect_equal(unique(unlist(lapply(adset$param, function(x) x[[3]]))), c(1, 0.5, 0.1))
-  expect_equal(unique(lapply(adset$param, function(x) x[[4]])), list('SAMME.R'))
+  expect_equal(unique(unlist(lapply(adset$param, function(x) x[[1]]))), c(10,50, 200))
+  expect_equal(unique(unlist(lapply(adset$param, function(x) x[[2]]))), c(1, 0.5, 0.1))
+  expect_equal(unique(lapply(adset$param, function(x) x[[3]])), list('SAMME.R'))
   
   expect_false(attr(adset$param, 'settings')$requiresDenseMatrix)
   expect_equal(attr(adset$param, 'settings')$name, 'AdaBoost')
-  expect_equal(attr(adset$param, 'settings')$pythonImport, 'sklearn')
-  expect_equal(attr(adset$param, 'settings')$pythonImportSecond, 'ensemble')
-  expect_equal(attr(adset$param, 'settings')$pythonClassifier, "AdaBoostClassifier")
+  expect_equal(attr(adset$param, 'settings')$pythonModule, 'sklearn.ensemble')
+  expect_equal(attr(adset$param, 'settings')$pythonClass, "AdaBoostClassifier")
   
   
   inputs <- AdaBoostClassifierInputs(list, adset$param[[1]])
   expect_equal(
     names(inputs), 
-    c("base_estimator","n_estimators","learning_rate","algorithm","random_state" )
+    c("n_estimators","learning_rate","algorithm","random_state" )
     )
   
 })
@@ -42,10 +40,6 @@ test_that("setAdaBoost errors as expected", {
 })
 
 
-
-
-
-
 test_that("setMLP settings work checks", {
   
   mlpset <- setMLP(
@@ -83,9 +77,8 @@ test_that("setMLP settings work checks", {
   
   expect_false(attr(mlpset$param, 'settings')$requiresDenseMatrix)
   expect_equal(attr(mlpset$param, 'settings')$name, 'Neural Network')
-  expect_equal(attr(mlpset$param, 'settings')$pythonImport, 'sklearn')
-  expect_equal(attr(mlpset$param, 'settings')$pythonImportSecond, 'neural_network')
-  expect_equal(attr(mlpset$param, 'settings')$pythonClassifier, "MLPClassifier")
+  expect_equal(attr(mlpset$param, 'settings')$pythonModule, 'sklearn.neural_network')
+  expect_equal(attr(mlpset$param, 'settings')$pythonClass, "MLPClassifier")
   
   inputs <- MLPClassifierInputs(list, mlpset$param[[1]])
   expect_equal(
@@ -99,12 +92,6 @@ test_that("setMLP settings work checks", {
 })
 
 
-
-
-
-
-
-
 test_that("setNaiveBayes settings work checks", {
   
   nbset <- setNaiveBayes(
@@ -116,9 +103,8 @@ test_that("setNaiveBayes settings work checks", {
   
   expect_true(attr(nbset$param, 'settings')$requiresDenseMatrix)
   expect_equal(attr(nbset$param, 'settings')$name, 'Naive Bayes')
-  expect_equal(attr(nbset$param, 'settings')$pythonImport, 'sklearn')
-  expect_equal(attr(nbset$param, 'settings')$pythonImportSecond, 'naive_bayes')
-  expect_equal(attr(nbset$param, 'settings')$pythonClassifier, "GaussianNB")
+  expect_equal(attr(nbset$param, 'settings')$pythonModule, 'sklearn.naive_bayes')
+  expect_equal(attr(nbset$param, 'settings')$pythonClass, "GaussianNB")
   
   inputs <- GaussianNBInputs(list, nbset$param[[1]])
   expect_equal(names(inputs),NULL)
@@ -126,11 +112,6 @@ test_that("setNaiveBayes settings work checks", {
 })
 
 
-
-
-
-
-
 test_that("setRandomForest settings work checks", {
   
   rfset <- setRandomForest(
@@ -140,29 +121,28 @@ test_that("setRandomForest settings work checks", {
     minSamplesSplit = list(2,5),
     minSamplesLeaf = list(1,10),
     minWeightFractionLeaf = list(0),
-    mtries = list('auto', 'log2'),
+    mtries = list('sqrt', 'log2'),
     maxLeafNodes = list(NULL),
     minImpurityDecrease = list(0),
     bootstrap = list(TRUE),
     maxSamples = list(NULL, 0.9),
     oobScore = list(FALSE),
     nJobs = list(NULL),
-    classWeight = list('balanced_subsample', NULL),
+    classWeight = list(NULL),
     seed = sample(100000,1)
   )
   
   expect_equal(rfset$fitFunction, "fitSklearn")
   
-  expect_equal(length(rfset$param), 2*3*2*2*2*2*2)
+  expect_equal(length(rfset$param), 2*3*2*2*2*2*1)
   
   expect_equal(unique(lapply(rfset$param, function(x) x[[1]])), list(100,500))
   expect_equal(unique(unlist(lapply(rfset$param, function(x) x[[3]]))), c(4,10,17))
   
   expect_false(attr(rfset$param, 'settings')$requiresDenseMatrix)
   expect_equal(attr(rfset$param, 'settings')$name, 'Random forest')
-  expect_equal(attr(rfset$param, 'settings')$pythonImport, 'sklearn')
-  expect_equal(attr(rfset$param, 'settings')$pythonImportSecond, 'ensemble')
-  expect_equal(attr(rfset$param, 'settings')$pythonClassifier, "RandomForestClassifier")
+  expect_equal(attr(rfset$param, 'settings')$pythonModule, 'sklearn.ensemble')
+  expect_equal(attr(rfset$param, 'settings')$pythonClass, "RandomForestClassifier")
   
   inputs <- RandomForestClassifierInputs(list,  rfset$param[[1]])
   expect_equal(
@@ -175,8 +155,6 @@ test_that("setRandomForest settings work checks", {
 })
 
 
-
-
 test_that("setSVM  settings work checks", {
   
   svmset <- setSVM (
@@ -187,23 +165,22 @@ test_that("setSVM  settings work checks", {
     coef0 = list(0.0),
     shrinking = list(TRUE), 
     tol = list(0.001),
-    classWeight = list('balanced', NULL), 
+    classWeight = list(NULL), 
     cacheSize  = 500,
     seed = sample(100000,1)
   )
   
   expect_equal(svmset$fitFunction, "fitSklearn")
   
-  expect_equal(length(svmset$param), 4*3*6*2)
+  expect_equal(length(svmset$param), 4*3*6*1)
   
   expect_equal(unique(lapply(svmset$param, function(x) x[[4]])), list('scale', 1e-04, 3e-05, 0.001, 0.01, 0.25))
   expect_equal(unique(unlist(lapply(svmset$param, function(x) x[[1]]))), c(1,0.9,2,0.1))
   
   expect_false(attr(svmset$param, 'settings')$requiresDenseMatrix)
   expect_equal(attr(svmset$param, 'settings')$name, 'Support Vector Machine')
-  expect_equal(attr(svmset$param, 'settings')$pythonImport, 'sklearn')
-  expect_equal(attr(svmset$param, 'settings')$pythonImportSecond, 'svm')
-  expect_equal(attr(svmset$param, 'settings')$pythonClassifier, "SVC")
+  expect_equal(attr(svmset$param, 'settings')$pythonModule, 'sklearn.svm')
+  expect_equal(attr(svmset$param, 'settings')$pythonClass, "SVC")
   
   inputs <- SVCInputs(list,  svmset$param[[1]])
   expect_equal(
diff --git a/tests/testthat/test-sklearnClassifier_updated.R b/tests/testthat/test-sklearnClassifier_updated.R
deleted file mode 100644
index 761a9850e..000000000
--- a/tests/testthat/test-sklearnClassifier_updated.R
+++ /dev/null
@@ -1,89 +0,0 @@
-
-
-test_that("DecisionTree settings work checks", {
-  
-dtset <- setDecisionTree(
-  criterion = list('gini'),
-  splitter = list('best'),
-  maxDepth = list(4, 10, NULL),
-  minSamplesSplit = list(2, 10),
-  minSamplesLeaf = list(10, 50),
-  minWeightFractionLeaf = list(0),
-  maxFeatures = list(100,'auto', NULL),
-  maxLeafNodes = list(NULL),
-  minImpurityDecrease = list(10^-7),
-  classWeight = list(NULL, 'balanced'),
-  seed = sample(1000000,1)
-)
-
-expect_equal(dtset$fitFunction, "fitSklearn")
-
-expect_equal(length(dtset$param), 3*2*2*3*2)
-
-expect_equal(unique(unlist(lapply(dtset$param, function(x) x[[1]]))), 'gini')
-expect_equal(unique(unlist(lapply(dtset$param, function(x) x[[2]]))), 'best')
-expect_equal(length(unique(lapply(dtset$param, function(x) x[[3]]))), 3)
-
-expect_false(attr(dtset$param, 'settings')$requiresDenseMatrix)
-expect_equal(attr(dtset$param, 'settings')$name, 'Decision Tree')
-expect_equal(attr(dtset$param, 'settings')$pythonImport, 'sklearn')
-expect_equal(attr(dtset$param, 'settings')$pythonImportSecond, 'tree')
-expect_equal(attr(dtset$param, 'settings')$pythonClassifier, "DecisionTreeClassifier")
-
-
-})
-
-
-test_that("DecisionTree errors as expected", {
-  
-  expect_error(setDecisionTree(criterion = list('madeup')))
-  
-  expect_error(setDecisionTree(maxDepth =  list(-1)))
-  expect_error(setDecisionTree(minSamplesSplit =  list(-1)))
-  expect_error(setDecisionTree(minSamplesLeaf  =  list(-1)))
-  
-})
-
-
-test_that("check fit of DecisionTree", {
-  
-  
-  modelSettings <- setDecisionTree(
-    criterion = list('gini'),
-    splitter = list('best'),
-    maxDepth = list(as.integer(4)),
-    minSamplesSplit = list(2),
-    minSamplesLeaf = list(10),
-    minWeightFractionLeaf = list(0),
-    maxFeatures = list(as.integer(100),'auto'),
-    maxLeafNodes = list(NULL),
-    minImpurityDecrease = list(10^-7),
-    classWeight = list(NULL, 'balanced'),
-    seed = sample(1000000,1)
-  )
-  trainData <- createTrainData(
-    plpData = plpData, 
-    population = population
-    )
-  
-  plpModel <- fitPlp(
-    trainData = trainData, 
-    modelSettings = modelSettings,
-    analysisId = 'DecisionTree'
-    )
-  
-  expect_equal(nrow(trainData$labels)*2, nrow(plpModel$prediction))
-  expect_equal(length(unique(plpModel$prediction$evaluationType)), 2)
-
-  expect_true(nrow(plpModel$covariateImportance) < trainData$covariateData$covariateRef %>% dplyr::tally() %>% dplyr::pull())
-  
-  expect_true(dir.exists(plpModel$model))
-  expect_equal(dir(plpModel$model),"model.json")
-  
-  expect_equal(plpModel$trainDetails$outcomeId,2)
-  expect_equal(plpModel$trainDetails$cohortId,1)
-  
-})
-
-# add tests for other classifiers
-
diff --git a/tests/testthat/test-sklearnJson.R b/tests/testthat/test-sklearnJson.R
new file mode 100644
index 000000000..6d36b5f07
--- /dev/null
+++ b/tests/testthat/test-sklearnJson.R
@@ -0,0 +1,131 @@
+
+
+sklearn <- reticulate::import('sklearn', convert=FALSE)
+np <- reticulate::import('numpy', convert=FALSE)
+
+data <- sklearn$datasets$make_classification(n_samples=500L, n_features=3L, 
+                                            n_classes=2L, n_informative=3L, 
+                                            n_redundant=0L, random_state=0L, 
+                                            shuffle=FALSE)
+
+X_unseen <- sklearn$datasets$make_classification(n_samples=100L, n_features=3L, 
+                                                 n_classes=2L, n_informative=3L, 
+                                                 n_redundant=0L, random_state=42L, 
+                                                 shuffle=FALSE)[[0]]
+X <- data[[0]]
+y <- data[[1]]
+
+test_that("Decision tree to json is correct", {
+  classifier <- sklearn$tree$DecisionTreeClassifier(max_depth=3L)
+  
+  model <- classifier$fit(X,y)
+  predictions <- reticulate::py_to_r(model$predict_proba(X_unseen))
+  path <- file.path(tempdir(),"model.json")
+  
+  sklearnToJson(model, path)
+  
+  loadedModel <- sklearnFromJson(path)
+  
+  loadedPredictions  <- reticulate::py_to_r(loadedModel$predict_proba(X_unseen))
+  
+  expect_true(all.equal(predictions, loadedPredictions))
+})
+
+test_that("Random forest to json is correct", {
+  classifier <- sklearn$ensemble$RandomForestClassifier(n_estimators=10L)
+  
+  model <- classifier$fit(X,y)
+  predictions <- reticulate::py_to_r(model$predict_proba(X_unseen))
+  path <- file.path(tempdir(),"model.json")
+  
+  sklearnToJson(model, path)
+  
+  loadedModel <- sklearnFromJson(path)
+  
+  loadedPredictions  <- reticulate::py_to_r(loadedModel$predict_proba(X_unseen))
+  
+  expect_true(all.equal(predictions, loadedPredictions))
+})
+
+test_that("Adaboost to json is correct", {
+  classifier <- sklearn$ensemble$AdaBoostClassifier(n_estimators=10L)
+  
+  model <- classifier$fit(X,y)
+  predictions <- reticulate::py_to_r(model$predict_proba(X_unseen))
+  path <- file.path(tempdir(), "model.json")
+  
+  sklearnToJson(model, path)
+  
+  loadedModel <- sklearnFromJson(path)
+  
+  loadedPredictions  <- reticulate::py_to_r(loadedModel$predict_proba(X_unseen))
+  
+  expect_true(all.equal(predictions, loadedPredictions))
+})
+
+test_that("Naive Bayes to json is correct", {
+  classifier <- sklearn$naive_bayes$GaussianNB()
+  
+  model <- classifier$fit(X,y)
+  predictions <- reticulate::py_to_r(model$predict_proba(X_unseen))
+  path <- file.path(tempdir(), "model.json")
+  
+  sklearnToJson(model, path)
+  
+  loadedModel <- sklearnFromJson(path)
+  
+  loadedPredictions  <- reticulate::py_to_r(loadedModel$predict_proba(X_unseen))
+  
+  expect_true(all.equal(predictions, loadedPredictions))
+})
+
+test_that("MLP to json is correct", {
+  # lower tolerance to not get convergence warning
+  classifier <- sklearn$neural_network$MLPClassifier(tol=1e-2)
+  
+  model <- classifier$fit(X,y)
+  predictions <- reticulate::py_to_r(model$predict_proba(X_unseen))
+  path <- file.path(tempdir(), "model.json")
+  
+  sklearnToJson(model, path)
+  
+  loadedModel <- sklearnFromJson(path)
+  
+  loadedPredictions  <- reticulate::py_to_r(loadedModel$predict_proba(X_unseen))
+  
+  expect_true(all.equal(predictions, loadedPredictions))
+})
+
+test_that("SVM to json is correct", {
+  classifier <- sklearn$svm$SVC(probability=TRUE)
+  
+  # create sparse data because then some of the internal fields in the 
+  # SVM will be sparse
+  feature_hasher <- sklearn$feature_extraction$FeatureHasher(n_features=3L)
+  random <- reticulate::import("random", convert=FALSE)
+  features <- list()
+  y_sparse <- np$empty(100L)
+  for (i in 1:100) {
+    row <- reticulate::dict(a=random$randint(0,2),
+                            b=random$randint(3,5),
+                            c=random$randint(6,8))
+    features <- c(features, row)
+    reticulate::py_set_item(y_sparse, i - 1L, random$randint(0, 2))
+  } 
+  X_sparse <- feature_hasher$transform(features)
+  
+  model <- classifier$fit(X_sparse,y_sparse)
+  predictions <- reticulate::py_to_r(model$predict_proba(X_unseen))
+  path <- file.path(tempdir(), "model.json")
+  
+  sklearnToJson(model, path)
+  
+  loadedModel <- sklearnFromJson(path)
+  
+  loadedPredictions  <- reticulate::py_to_r(loadedModel$predict_proba(X_unseen))
+  
+  expect_true(all.equal(predictions, loadedPredictions))
+})
+
+
+
diff --git a/tests/testthat/test-validation_updated.R b/tests/testthat/test-validation.R
similarity index 82%
rename from tests/testthat/test-validation_updated.R
rename to tests/testthat/test-validation.R
index 74a49bd91..84ee78516 100644
--- a/tests/testthat/test-validation_updated.R
+++ b/tests/testthat/test-validation.R
@@ -17,23 +17,7 @@
 context("Validation")
 
 # Test unit for the creation of the study externalValidatePlp
-
-connectionDetails <- Eunomia::getEunomiaConnectionDetails()
-Eunomia::createCohorts(connectionDetails)
-
-databaseDetails <- createDatabaseDetails(
-  connectionDetails = connectionDetails, 
-  cdmDatabaseSchema = "main", 
-  cdmDatabaseName = "main",
-  cohortDatabaseSchema = "main", 
-  cohortTable = "cohort", 
-  outcomeDatabaseSchema = "main", 
-  outcomeTable =  "cohort",
-  cohortId = 1, 
-  outcomeIds = 3, #make this ids
-  cdmVersion = 5)
-
-modelVal <- plpResult$model
+modelVal <- loadPlpModel(file.path(saveLoc, 'Test', 'plpResult', 'model'))
 validationDatabaseDetailsVal <- databaseDetails  # from run multiple tests
 validationRestrictPlpDataSettingsVal <- createRestrictPlpDataSettings(washoutPeriod = 0, sampleSize = NULL)
 recalSet <- createValidationSettings(recalibrate = 'weakRecalibration')
diff --git a/vignettes/.gitignore b/vignettes/.gitignore
new file mode 100644
index 000000000..097b24163
--- /dev/null
+++ b/vignettes/.gitignore
@@ -0,0 +1,2 @@
+*.html
+*.R
diff --git a/vignettes/AddingCustomFeatureEngineering.Rmd b/vignettes/AddingCustomFeatureEngineering.Rmd
index ebef0e25b..fab612c7f 100644
--- a/vignettes/AddingCustomFeatureEngineering.Rmd
+++ b/vignettes/AddingCustomFeatureEngineering.Rmd
@@ -1,6 +1,6 @@
 ---
 title: "Adding Custom Feature Engineering Functions"
-author: "Jenna Reps"
+author: "Jenna Reps, Egill Fridgeirsson"
 date: "`r Sys.Date()`"
 header-includes:
     - \usepackage{fancyhdr}
@@ -19,61 +19,63 @@ output:
     number_sections: yes
     toc: yes
 ---
+
+```{=html}
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Adding Custom Feature Engineering}
 -->
-
+```
 ```{r, echo = FALSE, message = FALSE, warning = FALSE}
 library(PatientLevelPrediction)
 ```
+
 # Introduction
 
-This vignette describes how you can add your own custom function for feature engineering in the Observational Health Data Sciencs and Informatics (OHDSI) [`PatientLevelPrediction`](http://github.com/OHDSI/PatientLevelPrediction) package. This vignette assumes you have read and are comfortable with building single patient level prediction models as described in the  [`BuildingPredictiveModels` vignette](https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf).
+This vignette describes how you can add your own custom function for feature engineering in the Observational Health Data Sciences and Informatics (OHDSI) [`PatientLevelPrediction`](http://github.com/OHDSI/PatientLevelPrediction) package. This vignette assumes you have read and are comfortable with building single patient level prediction models as described in the [`BuildingPredictiveModels` vignette](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingPredictiveModels.pdf).
 
 **We invite you to share your new feature engineering functions with the OHDSI community through our [GitHub repository](http://github.com/OHDSI/PatientLevelPrediction).**
 
-# Feature Engineering Function Code Structure 
+# Feature Engineering Function Code Structure
 
-To make a custom feature engineering function that can be used within PatientLevelPrediction you need to write two different functions.  The 'create' function and the 'implement' function.  
+To make a custom feature engineering function that can be used within PatientLevelPrediction you need to write two different functions. The 'create' function and the 'implement' function.
 
 The 'create' function, e.g., create\<FeatureEngineeringFunctionName\>, takes the parameters of the feature engineering 'implement' function as input, checks these are valid and outputs these as a list of class 'featureEngineeringSettings' with the 'fun' attribute specifying the 'implement' function to call.
 
-The 'implement' function, e.g., implement\<FeatureEngineeringFunctionName\>,  must take as input:
- * trainData - a list containing:
-   - covariateData: the plpData$covariateData restricted to the training patients
-   - labels: a data frame that contain rowId (patient identifier) and outcomeCount (the class labels)
-   - folds: a data.frame that contains rowId (patient identifier) and index (the cross validation fold)
- * featureEngineeringSettings - the output of your create\<FeatureEngineeringFunctionName\>
- 
-The 'implement' function can then do any manipulation of the trainData (adding new features or removing features) but must output a trainData object containing the new covariateData, labels and folds for the training data patients.
+The 'implement' function, e.g., implement\<FeatureEngineeringFunctionName\>, must take as input:
+
+-   `trainData` - a list containing:
+
+    -   `covariateData`: the `plpData$covariateData`restricted to the training patients
+
+    -   `labels`: a data frame that contain `rowId`(patient identifier) and `outcomeCount` (the class labels)
+
+    -   `folds`: a data.frame that contains `rowId` (patient identifier) and `index` (the cross validation fold)
+
+-   `featureEngineeringSettings` - the output of your create\<FeatureEngineeringFunctionName\>
+
+The 'implement' function can then do any manipulation of the `trainData` (adding new features or removing features) but must output a `trainData` object containing the new `covariateData`, `labels` and `folds` for the training data patients.
 
 # Example
 
-Let's consider the situation where we wish to create an age spline feature.  To make this custom feature engineering function we need to write the 'create' and 'implement' R functions.
+Let's consider the situation where we wish to create an age spline feature. To make this custom feature engineering function we need to write the 'create' and 'implement' R functions.
 
 ## Create function
 
-Our age spline feature function will create a new feature using the plpData$cohorts ageYear column.  We will implement a restricted cubic spline that requires specifying the number of knots.
-.  Therefore, the inputs for this are: 
- * `knots` an integer/double specifying the number of knots
- 
+Our age spline feature function will create a new feature using the `plpData$cohorts$ageYear` column. We will implement a restricted cubic spline that requires specifying the number of knots. Therefore, the inputs for this are: `knots` - an integer/double specifying the number of knots.
+
 ```{r, echo = TRUE, eval=FALSE}
-createAgeSpine <- function(
+createAgeSpline <- function(
                      knots = 5
                      ){
   
-  # add input checks
-  checkIsClass(knots, c('numeric','integer'))
-  checkHigher(knots,0)
-  
   # create list of inputs to implement function
   featureEngineeringSettings <- list(
     knots = knots
     )
   
   # specify the function that will implement the sampling
-  attr(featureEngineeringSettings, "fun") <- "implementAgeSpine"
+  attr(featureEngineeringSettings, "fun") <- "implementAgeSplines"
 
   # make sure the object returned is of class "sampleSettings"
   class(featureEngineeringSettings) <- "featureEngineeringSettings"
@@ -82,64 +84,76 @@ createAgeSpine <- function(
 }
 ```
 
-We now need to create the 'implement' function `implementAgeSpine()`
+We now need to create the 'implement' function `implementAgeSplines()`
 
 ## Implement function
 
-All 'implement' functions must take as input the trainData and the featureEngineeringSettings (this is the output of the 'create' function).  They must return a trainData object containing the new covariateData, labels and folds.
+All 'implement' functions must take as input the `trainData` and the `featureEngineeringSettings` (this is the output of the 'create' function). They must return a `trainData` object containing the new `covariateData`, `labels` and `folds`.
 
-In our example, the `createAgeSpine()` will return a list with 'knots'.  The featureEngineeringSettings therefore contains this.
+In our example, the `createAgeSpline()` will return a list with 'knots'. The `featureEngineeringSettings` therefore contains this.
 
 ```{r tidy=FALSE,eval=FALSE}
-implementAgeSpine <- function(trainData, featureEngineeringSettings){
-
-  # currently not used
-  knots <- featureEngineeringSettings$knots
+implementAgeSplines <- function(trainData, featureEngineeringSettings, model=NULL) {
+  # if there is a model, it means this function is called through applyFeatureengineering, meaning it   # should apply the model fitten on training data to the test data
+  if (is.null(model)) {
+    knots <- featureEngineeringSettings$knots
+    ageData <- trainData$labels
+    y <- ageData$outcomeCount
+    X <- ageData$ageYear
+    model <- mgcv::gam(
+      y ~ s(X, bs='cr', k=knots, m=2)
+    )
+    newData <- data.frame(
+      rowId = ageData$rowId,
+      covariateId = 2002,
+      covariateValue = model$fitted.values
+    )
+  }
+  else {
+    ageData <- trainData$labels
+    X <- trainData$labels$ageYear
+    y <- ageData$outcomeCount
+    newData <- data.frame(y=y, X=X)
+    yHat <- predict(model, newData)
+    newData <- data.frame(
+      rowId = trainData$labels$rowId,
+      covariateId = 2002,
+      covariateValue = yHat
+    )
+  }
   
+  # remove existing age if in covariates 
+  trainData$covariateData$covariates <- trainData$covariateData$covariates |> 
+    dplyr::filter(!covariateId %in% c(1002))
   
-  # age in in trainData$labels as ageYear
-  ageData <- trainData$labels
+  # update covRef
+  Andromeda::appendToTable(trainData$covariateData$covariateRef, 
+                           data.frame(covariateId=2002,
+                                      covariateName='Cubic restricted age splines',
+                                      analysisId=2,
+                                      conceptId=2002))
   
-  # now implement the code to do your desired feature engineering
+  # update covariates
+  Andromeda::appendToTable(trainData$covariateData$covariates, newData)
   
-  data <- Matrix::sparseMatrix(
-    i = 1:length(ageData$rowId),
-    j = rep(1, length(ageData$rowId)),
-    x = ageData$ageYear,
-    dims=c(length(ageData$rowId),1)
+  featureEngineering <- list(
+    funct = 'implementAgeSplines',
+    settings = list(
+      featureEngineeringSettings = featureEngineeringSettings,
+      model = model
+    )
   )
   
-  data <- as.matrix(data)
-  x <- data[,1]
-  y <- ageData$outcomeCount
-  
-mRCS <- rms::ols(
-  y~rms::rcs(x, 
-             stats::quantile(
-               x, 
-               c(0, .05, .275, .5, .775, .95, 1),
-               include.lowest = TRUE
-               )
-             )
-  )
-
-newData <- data.frame(
-  rowId = ageData$rowId,
-  covariateId = 2002,
-  covariateValue = mRCS$fitted.values
+  attr(trainData$covariateData, 'metaData')$featureEngineering = listAppend(
+    attr(trainData$covariateData, 'metaData')$featureEngineering,
+    featureEngineering
   )
-
-# add new data
-Andromeda::appendToTable(tbl = trainData$covariateData$covariates, 
-                         data = newData)
   
-  # return the updated trainData
   return(trainData)
 }
 
 ```
 
-
 # Acknowledgments
 
 Considerable work has been dedicated to provide the `PatientLevelPrediction` package.
@@ -152,8 +166,4 @@ citation("PatientLevelPrediction")
 
 [Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.](http://dx.doi.org/10.1093/jamia/ocy032)
 
-
 This work is supported in part through the National Science Foundation grant IIS 1251151.
-
-
-
diff --git a/vignettes/AddingCustomModels.Rmd b/vignettes/AddingCustomModels.Rmd
index 801734dba..84b21e7b4 100644
--- a/vignettes/AddingCustomModels.Rmd
+++ b/vignettes/AddingCustomModels.Rmd
@@ -19,30 +19,35 @@ output:
     number_sections: yes
     toc: yes
 ---
+
+```{=html}
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Adding Custom Patient-Level Prediction Algorithms}
 -->
-
+```
 ```{r, echo = FALSE, message = FALSE, warning = FALSE}
 library(PatientLevelPrediction)
 ```
+
 # Introduction
 
-This vignette describes how you can add your own custom algorithms in the Observational Health Data Sciencs and Informatics (OHDSI) [`PatientLevelPrediction`](http://github.com/OHDSI/PatientLevelPrediction) package. This allows you to fully leverage the OHDSI PatientLevelPrediction framework for model development and validation. This vignette assumes you have read and are comfortable with building single patient level prediction models as described in the  [`BuildingPredictiveModels` vignette](https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf).
+This vignette describes how you can add your own custom algorithms in the Observational Health Data Sciencs and Informatics (OHDSI) [`PatientLevelPrediction`](http://github.com/OHDSI/PatientLevelPrediction) package. This allows you to fully leverage the OHDSI PatientLevelPrediction framework for model development and validation. This vignette assumes you have read and are comfortable with building single patient level prediction models as described in the [`BuildingPredictiveModels` vignette](https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf).
 
 **We invite you to share your new algorithms with the OHDSI community through our [GitHub repository](http://github.com/OHDSI/PatientLevelPrediction).**
 
-# Algorithm  Code Structure 
-Each algorithm in the package should be implemented in its own \<Name\>.R file, e.g. KNN.R, containing a set\<Name\> function, a fit\<Name\> function and a  predict\<Name\> function. Occasionally the fit and prediction functions may be reused (if using an R classifier see RClassifier.R or if using a scikit-learn classifier see SklearnClassifier.R).  We will now describe each of these functions in more detail below. 
+# Algorithm Code Structure
+
+Each algorithm in the package should be implemented in its own \<Name\>.R file, e.g. KNN.R, containing a set\<Name\> function, a fit\<Name\> function and a predict\<Name\> function. Occasionally the fit and prediction functions may be reused (if using an R classifier see RClassifier.R or if using a scikit-learn classifier see SklearnClassifier.R). We will now describe each of these functions in more detail below.
 
 ## Set
+
 The set\<Name\> is a function that takes as input the different hyper-parameter values to do a grid search when training. The output of the functions needs to be a list as class `modelSettings` containing:
 
-+ param \- all the combinations of the hyper-parameter values input
-+ fitFunction \- a string specifying what function to call to fit the model
+-   param - all the combinations of the hyper-parameter values input
+-   fitFunction - a string specifying what function to call to fit the model
 
-The param object can have a setttings attribute containing any extra settings.  For example to specify the model name and the seed used for reproducibility:
+The param object can have a setttings attribute containing any extra settings. For example to specify the model name and the seed used for reproducibility:
 
 ```{r, echo = TRUE, eval=FALSE}
 attr(param, 'settings') <- list(
@@ -84,53 +89,60 @@ setMadeUp <- function(a=c(1,4,10), b=2, seed=NULL){
 ```
 
 ## Fit
-This function should train your custom model for each parameter entry, pick the best parameters and train a final model for that setting. 
+
+This function should train your custom model for each parameter entry, pick the best parameters and train a final model for that setting.
 
 The fit\<Model\> should have as inputs:
 
- * trainData \- a list containing the covariateData, labels and folds for the training population
- * param \- the hyper-parameters as a list of all combinations 
- * search \- the type of hyper-parameter search 
- * analysisId \- an identifier for the analysis
+-   trainData - a list containing the covariateData, labels and folds for the training population
+-   param - the hyper-parameters as a list of all combinations
+-   search - the type of hyper-parameter search
+-   analysisId - an identifier for the analysis
 
 The fit function should return a list of class `plpModel` with the following objects:
 
- * model \- a trained model (or location of the model if it is not an R object)
- * prediction \- a data.frame object with the trainData$labels plus an extra column with the name 'value' corresponding to the predicted risk of having the outcome during the time-at-risk. 
- * settings \- a list containing:
-   + plpDataSettings \- the plpData settings e.g., attr(trainData, "metaData")$plpDataSettings
-   + covariateSettings \- the covariate settings e.g.,  attr(trainData, "metaData")$covariateSettings
-    + populationSettings \- the population settings e.g., attr(trainData, "metaData")$populationSettings,
-    + featureEngineering \- the feature engineering settings e.g., attr(trainData$covariateData, "metaData")$featureEngineering,
-    + tidyCovariates \- the preprocessing settings e.g., attr(trainData$covariateData, "metaData")$tidyCovariateDataSettings, 
-    + requireDenseMatrix \- does the model require a dense matrix? e.g., attr(param, 'settings')$requiresDenseMatrix,
-    + modelSettings = a list containing: model (model name), param (the hyper-parameter search list), finalModelParameters (the final model hyper-parameters), extraSettings (any extra settings)
-    + splitSettings \- the split settings e.g., attr(trainData, "metaData")$splitSettings,
-    + sampleSettings \- the sample settings e.g., attr(trainData, "metaData")$sampleSettings
- * trainDetails \- a list containing:
-   + analysisId \- the identifier for the analysis
-   + cdmDatabaseSchema \- the database used to develop the model
-   + outcomeId \- the outcome id
-   + cohortId \- the target population id
-   + attrition \- the attrition
-   + trainingTime \- how long it took to train the model
-   + trainingDate \- date of model training
-   + hyperParamSearch \- the hyper-parameter search used to train the model
- * covariateImportance \- a data.frame containing the columns 'covariateId', 'covariateValue' (the variable importance) and 'columnId' (the column number that the variable need to be mapped to when implementing the model)
- 
+-   model - a trained model (or location of the model if it is not an R object)
+-   prediction - a data.frame object with the trainData\$labels plus an extra column with the name 'value' corresponding to the predicted risk of having the outcome during the time-at-risk.
+-   preprocessing - the settings required to preprocess the data when applying the model
+    -   featureEngineering - the feature engineering settings e.g., attr(trainData$covariateData, "metaData")$featureEngineering,
+    -   tidyCovariates - the preprocessing settings e.g., attr(trainData$covariateData, "metaData")$tidyCovariateDataSettings,
+    -   requireDenseMatrix - does the model require a dense matrix? e.g., attr(param, 'settings')\$requiresDenseMatrix,
+-   modelDesign - a list containing:
+    -   targetId - the id of the target cohort
+    -   outcomeId - the id of the outcome cohort
+    -   plpDataSettings - the plpData settings e.g., attr(trainData, "metaData")\$plpDataSettings
+    -   covariateSettings - the covariate settings e.g., attr(trainData, "metaData")\$covariateSettings
+    -   populationSettings - the population settings e.g., attr(trainData, "metaData")\$populationSettings,
+    -   featureEngineeringSettings - the feature engineering settings e.g., attr(trainData$covariateData, "metaData")$featureEngineeringSettings,
+    -   preprocessSettings - the preprocessing settings e.g., attr(trainData$covariateData, "metaData")$preprocessSettings,
+    -   modelSettings = a list containing: model (model name), param (the hyper-parameter search list), finalModelParameters (the final model hyper-parameters), extraSettings (any extra settings)
+    -   splitSettings - the split settings e.g., attr(trainData, "metaData")\$splitSettings,
+    -   sampleSettings - the sample settings e.g., attr(trainData, "metaData")\$sampleSettings
+-   trainDetails - a list containing:
+    -   analysisId - the identifier for the analysis
+    -   developmentDatabase - the database used to develop the model
+    -   attrition - the attrition
+    -   trainingTime - how long it took to train the model
+    -   trainingDate - date of model training
+    -   hyperParamSearch - the hyper-parameter search used to train the model
+    -   any other objects specific to training
+-   covariateImportance - a data.frame containing the columns 'covariateId', 'covariateValue' (the variable importance) and 'columnId' (the column number that the variable need to be mapped to when implementing the model)
+
 In additon the plpModel requires two attributes:
 
-  * predictionFunction - the name of the function used to make predictions
-  * modelType  - whether the model is 'binary' or 'survival' 
+-   predictionFunction - the name of the function used to make predictions
+-   modelType - whether the model is 'binary' or 'survival'
 
-For example `attr(result, 'predictionFunction') <- 'madeupPrediction'` means when the model is applied to new data, the 'madeupPrediction' function is called to make predictions.  If this doesnt exist, then the model will fail.  The other attribute is the modelType `attr(result, 'modelType') <- 'binary'` this is needed when evaluating the model to ensure the correct evaluation is applied.  Currently the evaluation supports 'binary' and 'survival' modelType.  
+For example `attr(result, 'predictionFunction') <- 'madeupPrediction'` means when the model is applied to new data, the 'madeupPrediction' function is called to make predictions. If this doesnt exist, then the model will fail. The other attribute is the modelType `attr(result, 'modelType') <- 'binary'` this is needed when evaluating the model to ensure the correct evaluation is applied. Currently the evaluation supports 'binary' and 'survival' modelType.
 
-Note: If a new modelType is desired, then the evalaution code within PatientLevelPrediction must be updated to specify how the new type is evaluated.  This requires making edits to PatientLevelPrediction and then making a pull request to the PatientLevelPrediction github.  The evaluation cannot have one off customization because the evaluation must be standardized to enable comparison across similar models.
+Note: If a new modelType is desired, then the evalaution code within PatientLevelPrediction must be updated to specify how the new type is evaluated. This requires making edits to PatientLevelPrediction and then making a pull request to the PatientLevelPrediction github. The evaluation cannot have one off customization because the evaluation must be standardized to enable comparison across similar models.
 
 A full example of a custom 'binary' classifier fit function is:
 
 ```{r tidy=FALSE,eval=FALSE}
-fitMadeUp <- function(trainData, param, search, analysisId){
+fitMadeUp <- function(trainData, modelSettings, search, analysisId){
+  
+  param <- modelSettings$param
   
   # **************** code to train the model here
   # trainedModel <- this code should apply each hyper-parameter combination   
@@ -152,13 +164,20 @@ fitMadeUp <- function(trainData, param, search, analysisId){
   # construct the standard output for a model:
   result <- list(model = trainedModel,
                  prediction = prediction, # the train and maybe the cross validation predictions for the trainData
-    settings = list(
+                 preprocessing = list(
+                   featureEngineering = attr(trainData$covariateData, "metaData")$featureEngineering,
+      tidyCovariates = attr(trainData$covariateData, "metaData")$tidyCovariateDataSettings, 
+      requireDenseMatrix = attr(param, 'settings')$requiresDenseMatrix,
+      
+                 ),
+    modelDesign = list(
+      outcomeId = attr(trainData, "metaData")$outcomeId,
+      targetId = attr(trainData, "metaData")$targetId,
       plpDataSettings = attr(trainData, "metaData")$plpDataSettings,
       covariateSettings = attr(trainData, "metaData")$covariateSettings,
       populationSettings = attr(trainData, "metaData")$populationSettings,
-      featureEngineering = attr(trainData$covariateData, "metaData")$featureEngineering,
-      tidyCovariates = attr(trainData$covariateData, "metaData")$tidyCovariateDataSettings, 
-      requireDenseMatrix = attr(param, 'settings')$requiresDenseMatrix,
+      featureEngineeringSettings = attr(trainData$covariateData, "metaData")$featureEngineeringSettings,
+      prerocessSettings = attr(trainData$covariateData, "metaData")$prerocessSettings, 
       modelSettings = list(
         model = attr(param, 'settings')$modelName, # the model name
         param = param,
@@ -171,9 +190,7 @@ fitMadeUp <- function(trainData, param, search, analysisId){
     
     trainDetails = list(
       analysisId = analysisId,
-      cdmDatabaseSchema = attr(trainData, "metaData")$cdmDatabaseSchema,
-      outcomeId = attr(trainData, "metaData")$outcomeId,
-      cohortId = attr(trainData, "metaData")$cohortId,
+      developmentDatabase = attr(trainData, "metaData")$cdmDatabaseSchema,
       attrition = attr(trainData, "metaData")$attrition, 
       trainingTime = timeToTrain, # how long it took to train the model
       trainingDate = Sys.Date(),
@@ -189,12 +206,14 @@ fitMadeUp <- function(trainData, param, search, analysisId){
 }
 
 ```
-You could make the fitMadeUp function cleaner by adding helper function in the MadeUp.R file that are called by the fit function (for example a function to run cross validation).  It is important to ensure there is a valid prediction function (the one specified by `attr(result, 'predictionFunction') <- 'madeupPrediction'` is `madeupPrediction()`) as specified below.
+
+You could make the fitMadeUp function cleaner by adding helper function in the MadeUp.R file that are called by the fit function (for example a function to run cross validation). It is important to ensure there is a valid prediction function (the one specified by `attr(result, 'predictionFunction') <- 'madeupPrediction'` is `madeupPrediction()`) as specified below.
 
 ## Predict
-The prediction function takes as input the plpModel returned by fit, new data and a corresponding cohort.  It returns a data.frame with the same columns as cohort but with an additional column: 
 
- * value \- the predicted risk from the plpModel for each patient in the cohort
+The prediction function takes as input the plpModel returned by fit, new data and a corresponding cohort. It returns a data.frame with the same columns as cohort but with an additional column:
+
+-   value - the predicted risk from the plpModel for each patient in the cohort
 
 For example:
 
@@ -215,7 +234,7 @@ madeupPrediction <- function(plpModel, data, cohort){
 
 # Algorithm Example
 
-Below a fully functional algorithm example is given, however we highly recommend you to have a look at the available algorithms in the package (see GradientBoostingMachine.R for the set function, RClassifier.R for the fit and prediction function for R classifiers). 
+Below a fully functional algorithm example is given, however we highly recommend you to have a look at the available algorithms in the package (see GradientBoostingMachine.R for the set function, RClassifier.R for the fit and prediction function for R classifiers).
 
 ## Set
 
@@ -253,12 +272,13 @@ setMadeUp <- function(a=c(1,4,6), b=2, seed=NULL){
 
 ```
 
+## Fit
 
-## Fit 
 ```{r tidy=FALSE,eval=FALSE}
-fitMadeUp <- function(trainData, param, search, analysisId){
+fitMadeUp <- function(trainData, modelSettings, search, analysisId){
 
   # set the seed for reproducibility
+  param <- modelSettings$param
   set.seed(attr(param, 'settings')$seed)
   
   # add folds to labels:
@@ -324,13 +344,20 @@ fitMadeUp <- function(trainData, param, search, analysisId){
   # construct the standard output for a model:
   result <- list(model = trainedModel,
                  prediction = prediction, 
-    settings = list(
+    preprocessing = list(
+                   featureEngineering = attr(trainData$covariateData, "metaData")$featureEngineering,
+      tidyCovariates = attr(trainData$covariateData, "metaData")$tidyCovariateDataSettings, 
+      requireDenseMatrix = attr(param, 'settings')$requiresDenseMatrix,
+      
+                 ),
+    modelDesign = list(
+      outcomeId = attr(trainData, "metaData")$outcomeId,
+      targetId = attr(trainData, "metaData")$targetId,
       plpDataSettings = attr(trainData, "metaData")$plpDataSettings,
       covariateSettings = attr(trainData, "metaData")$covariateSettings,
       populationSettings = attr(trainData, "metaData")$populationSettings,
-      featureEngineering = attr(trainData$covariateData, "metaData")$featureEngineering,
-      tidyCovariates = attr(trainData$covariateData, "metaData")$tidyCovariateDataSettings, 
-      requireDenseMatrix = attr(param, 'settings')$requiresDenseMatrix,
+      featureEngineeringSettings = attr(trainData$covariateData, "metaData")$featureEngineeringSettings,
+      prerocessSettings = attr(trainData$covariateData, "metaData")$prerocessSettings, 
       modelSettings = list(
         model = attr(param, 'settings')$modelName, # the model name
         param = param,
@@ -343,14 +370,14 @@ fitMadeUp <- function(trainData, param, search, analysisId){
     
     trainDetails = list(
       analysisId = analysisId,
-      cdmDatabaseSchema = attr(trainData, "metaData")$cdmDatabaseSchema,
-      outcomeId = attr(trainData, "metaData")$outcomeId,
-      cohortId = attr(trainData, "metaData")$cohortId,
+      developmentDatabase = attr(trainData, "metaData")$cdmDatabaseSchema,
       attrition = attr(trainData, "metaData")$attrition, 
       trainingTime = timeToTrain, # how long it took to train the model
       trainingDate = Sys.Date(),
       hyperParamSearch = hyperSummary # the hyper-parameters and performance data.frame
     ),
+    covariateImportance = merge(trainData$covariateData$covariateRef, varImp, by='covariateId') # add variable importance to covariateRef if possible
+  ),
     covariateImportance = varImp
   )
   class(result) <- 'plpModel'
@@ -363,6 +390,7 @@ fitMadeUp <- function(trainData, param, search, analysisId){
 ```
 
 ## Helpers
+
 In the fit model a helper function `made_up_model` is called, this is the function that trains a model given the data, labels and hyper-parameters.
 
 ```{r tidy=FALSE,eval=FALSE}
@@ -419,7 +447,8 @@ made_up_model <- function(param, data, final=F, labels){
 ```
 
 ## Predict
-The final step is to create a predict function for the model.  In the example above the predeiction function `attr(result, 'predictionFunction') <- 'madeupPrediction'` was madeupPrediction, so a `madeupPrediction` function is required when applying the model.  The predict function needs to take as input the plpModel returned by the fit function, new data to apply the model on and the cohort specifying the patients of interest to make the prediction for.
+
+The final step is to create a predict function for the model. In the example above the predeiction function `attr(result, 'predictionFunction') <- 'madeupPrediction'` was madeupPrediction, so a `madeupPrediction` function is required when applying the model. The predict function needs to take as input the plpModel returned by the fit function, new data to apply the model on and the cohort specifying the patients of interest to make the prediction for.
 
 ```{r tidy=FALSE,eval=FALSE}
 madeupPrediction <- function(plpModel, data , cohort){ 
@@ -430,7 +459,7 @@ madeupPrediction <- function(plpModel, data , cohort){
       plpData = data, 
       cohort = cohort,
       map = plpModel$covariateImportance %>% 
-        dplyr::select(.data$columnId, .data$covariateId)
+        dplyr::select("columnId", "covariateId")
     )
     
     newData <- matrixObjects$dataMatrix
@@ -451,14 +480,15 @@ madeupPrediction <- function(plpModel, data , cohort){
   # fix the rowIds to be the old ones
   # now use the originalRowId and remove the matrix rowId
   cohort <- cohort %>% 
-    dplyr::select(-.data$rowId) %>%
-    dplyr::rename(rowId = .data$originalRowId)
+    dplyr::select(-"rowId") %>%
+    dplyr::rename(rowId = "originalRowId")
   
   attr(cohort, "metaData") <- list(modelType = attr(plpModel, 'modelType')) 
   return(cohort)
   
 }
 ```
+
 As the madeup model uses the standard R prediction, it has the same prediction function as xgboost, so we could have not added a new prediction function and instead made the predictionFunction of the result returned by fitMadeUpModel to `attr(result, 'predictionFunction') <- 'predictXgboost'`.
 
 # Acknowledgments
@@ -473,8 +503,4 @@ citation("PatientLevelPrediction")
 
 [Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.](http://dx.doi.org/10.1093/jamia/ocy032)
 
-
 This work is supported in part through the National Science Foundation grant IIS 1251151.
-
-
-
diff --git a/vignettes/AddingCustomSamples.Rmd b/vignettes/AddingCustomSamples.Rmd
index 4f08a8791..eb2da04f1 100644
--- a/vignettes/AddingCustomSamples.Rmd
+++ b/vignettes/AddingCustomSamples.Rmd
@@ -7,10 +7,10 @@ header-includes:
     - \pagestyle{fancy}
     - \fancyhead{}
     - \fancyhead[CO,CE]{Custom Patient-Level Prediction Algorithms}
-    - \fancyfoot[CO,CE]{PatientLevelPrediction Package Version `r    utils::packageVersion("PatientLevelPrediction")`}
     - \fancyfoot[LE,RO]{\thepage}
     - \renewcommand{\headrulewidth}{0.4pt}
     - \renewcommand{\footrulewidth}{0.4pt}
+    - \fancyfoot[CO,CE]{PatientLevelPrediction Package Version `r    utils::packageVersion("PatientLevelPrediction")`}
 output:
   pdf_document:
     number_sections: yes
@@ -18,46 +18,73 @@ output:
   html_document:
     number_sections: yes
     toc: yes
+editor_options: 
+  markdown: 
+    wrap: 72
 ---
+
+```{=html}
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Adding Custom Sampling}
 -->
-
+```
 ```{r, echo = FALSE, message = FALSE, warning = FALSE}
 library(PatientLevelPrediction)
 ```
-# Introduction
-
-This vignette describes how you can add your own custom function for sampling the target population in the Observational Health Data Sciencs and Informatics (OHDSI) [`PatientLevelPrediction`](http://github.com/OHDSI/PatientLevelPrediction) package. This vignette assumes you have read and are comfortable with building single patient level prediction models as described in the  [`BuildingPredictiveModels` vignette](https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf).
-
-**We invite you to share your new sample functions with the OHDSI community through our [GitHub repository](http://github.com/OHDSI/PatientLevelPrediction).**
-
-# Sample Function Code Structure 
 
-To make a sampling function that can be used within PatientLevelPrediction you need to write two different functions.  The 'create' function and the 'implement' function.  
-
-The 'create' function, e.g., create\<SampleFunctionName\>, takes the parameters of the sample 'implement' function as input, checks these are valid and outputs these as a list of class 'sampleSettings' with the 'fun' attribute specifying the 'implement' function to call.
+# Introduction
 
-The 'implement' function, e.g., implement\<SampleFunctionName\>,  must take as input:
- * trainData - a list containing:
-   - covariateData: the plpData$covariateData restricted to the training patients
-   - labels: a data frame that contain rowId (patient identifier) and outcomeCount (the class labels)
-   - folds: a data.frame that contains rowId (patient identifier) and index (the cross validation fold)
- * sampleSettings - the output of your create\<SampleFunctionName\>
- 
-The 'implement' function can then do any manipulation of the trainData (such as undersampling or oversampling) but must output a trainData object containing the covariateData, labels and folds for the new training data sample.
+This vignette describes how you can add your own custom function for
+sampling the target population in the Observational Health Data Sciencs
+and Informatics (OHDSI)
+[`PatientLevelPrediction`](http://github.com/OHDSI/PatientLevelPrediction)
+package. This vignette assumes you have read and are comfortable with
+building single patient level prediction models as described in the
+[`BuildingPredictiveModels`
+vignette](https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf).
+
+**We invite you to share your new sample functions with the OHDSI
+community through our [GitHub
+repository](http://github.com/OHDSI/PatientLevelPrediction).**
+
+# Sample Function Code Structure
+
+To make a sampling function that can be used within
+PatientLevelPrediction you need to write two different functions. The
+'create' function and the 'implement' function.
+
+The 'create' function, e.g., create\<SampleFunctionName\>, takes the
+parameters of the sample 'implement' function as input, checks these are
+valid and outputs these as a list of class 'sampleSettings' with the
+'fun' attribute specifying the 'implement' function to call.
+
+The 'implement' function, e.g., implement\<SampleFunctionName\>, must
+take as input: \* trainData - a list containing: - covariateData: the
+plpData\$covariateData restricted to the training patients - labels: a
+data frame that contain rowId (patient identifier) and outcomeCount (the
+class labels) - folds: a data.frame that contains rowId (patient
+identifier) and index (the cross validation fold) \* sampleSettings -
+the output of your create\<SampleFunctionName\>
+
+The 'implement' function can then do any manipulation of the trainData
+(such as undersampling or oversampling) but must output a trainData
+object containing the covariateData, labels and folds for the new
+training data sample.
 
 # Example
 
-Let's consider the situation where we wish to take a random sample of the training data population.  To make this custom sampling function we need to write the 'create' and 'implement' R functions.
+Let's consider the situation where we wish to take a random sample of
+the training data population. To make this custom sampling function we
+need to write the 'create' and 'implement' R functions.
 
 ## Create function
 
-Our random sampling function will randomly sample `n` patients from the trainData.  Therefore, the inputs for this are: 
- * `n` an integer/double specifying the number of patients to sample
- * `sampleSeed` an integer/double specifying the seed for reproducibility
- 
+Our random sampling function will randomly sample `n` patients from the
+trainData. Therefore, the inputs for this are: \* `n` an integer/double
+specifying the number of patients to sample \* `sampleSeed` an
+integer/double specifying the seed for reproducibility
+
 ```{r, echo = TRUE, eval=FALSE}
 createRandomSampleSettings <- function(
                      n = 10000,
@@ -85,13 +112,18 @@ createRandomSampleSettings <- function(
 }
 ```
 
-We now need to create the 'implement' function `implementRandomSampleSettings()`
+We now need to create the 'implement' function
+`implementRandomSampleSettings()`
 
 ## Implement function
 
-All 'implement' functions must take as input the trainData and the sampleSettings (this is the output of the 'create' function).  They must return a trainData object containing the covariateData, labels and folds.
+All 'implement' functions must take as input the trainData and the
+sampleSettings (this is the output of the 'create' function). They must
+return a trainData object containing the covariateData, labels and
+folds.
 
-In our example, the `createRandomSampleSettings()` will return a list with 'n' and 'sampleSeed'.  The sampleSettings therefore contains these.
+In our example, the `createRandomSampleSettings()` will return a list
+with 'n' and 'sampleSeed'. The sampleSettings therefore contains these.
 
 ```{r tidy=FALSE,eval=FALSE}
 implementRandomSampleSettings <- function(trainData, sampleSettings){
@@ -138,10 +170,10 @@ implementRandomSampleSettings <- function(trainData, sampleSettings){
 
 ```
 
-
 # Acknowledgments
 
-Considerable work has been dedicated to provide the `PatientLevelPrediction` package.
+Considerable work has been dedicated to provide the
+`PatientLevelPrediction` package.
 
 ```{r tidy=TRUE,eval=TRUE}
 citation("PatientLevelPrediction")
@@ -149,10 +181,11 @@ citation("PatientLevelPrediction")
 
 **Please reference this paper if you use the PLP Package in your work:**
 
-[Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.](http://dx.doi.org/10.1093/jamia/ocy032)
-
-
-This work is supported in part through the National Science Foundation grant IIS 1251151.
-
-
+[Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and
+implementation of a standardized framework to generate and evaluate
+patient-level prediction models using observational healthcare data. J
+Am Med Inform Assoc.
+2018;25(8):969-975.](http://dx.doi.org/10.1093/jamia/ocy032)
 
+This work is supported in part through the National Science Foundation
+grant IIS 1251151.
diff --git a/vignettes/AddingCustomSplitting.Rmd b/vignettes/AddingCustomSplitting.Rmd
index b8b22d60e..96cef75d3 100644
--- a/vignettes/AddingCustomSplitting.Rmd
+++ b/vignettes/AddingCustomSplitting.Rmd
@@ -19,31 +19,32 @@ output:
     number_sections: yes
     toc: yes
 ---
+
+```{=html}
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Adding Custom Data Splitting}
 -->
-
+```
 ```{r, echo = FALSE, message = FALSE, warning = FALSE}
 library(PatientLevelPrediction)
 ```
+
 # Introduction
 
-This vignette describes how you can add your own custom function for splitting the labelled data into training data and validation data in the Observational Health Data Sciencs and Informatics (OHDSI) [`PatientLevelPrediction`](http://github.com/OHDSI/PatientLevelPrediction) package. This vignette assumes you have read and are comfortable with building single patient level prediction models as described in the  [`BuildingPredictiveModels` vignette](https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf).
+This vignette describes how you can add your own custom function for splitting the labelled data into training data and validation data in the Observational Health Data Sciencs and Informatics (OHDSI) [`PatientLevelPrediction`](http://github.com/OHDSI/PatientLevelPrediction) package. This vignette assumes you have read and are comfortable with building single patient level prediction models as described in the [`BuildingPredictiveModels` vignette](https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/BuildingPredictiveModels.pdf).
 
 **We invite you to share your new data splitting functions with the OHDSI community through our [GitHub repository](http://github.com/OHDSI/PatientLevelPrediction).**
 
-# Data Splitting Function Code Structure 
+# Data Splitting Function Code Structure
 
-To make a custom data splitting function that can be used within PatientLevelPrediction you need to write two different functions.  The 'create' function and the 'implement' function.  
+To make a custom data splitting function that can be used within PatientLevelPrediction you need to write two different functions. The 'create' function and the 'implement' function.
 
 The 'create' function, e.g., create\<DataSplittingFunction\>, takes the parameters of the data splitting 'implement' function as input, checks these are valid and outputs these as a list of class 'splitSettings' with the 'fun' attribute specifying the 'implement' function to call.
 
-The 'implement' function, e.g., implement\<DataSplittingFunction\>,  must take as input:
- * population: a data frame that contain rowId (patient identifier), ageYear, gender and outcomeCount (the class labels)
- * splitSettings - the output of your create\<DataSplittingFunction\>
- 
-The 'implement' function then needs to implement code to assign each rowId in the population to a splitId (<0 means in the train data, 0 means not used and >0 means in the training data with the value defining the cross validation fold).
+The 'implement' function, e.g., implement\<DataSplittingFunction\>, must take as input: \* population: a data frame that contain rowId (patient identifier), ageYear, gender and outcomeCount (the class labels) \* splitSettings - the output of your create\<DataSplittingFunction\>
+
+The 'implement' function then needs to implement code to assign each rowId in the population to a splitId (\<0 means in the train data, 0 means not used and \>0 means in the training data with the value defining the cross validation fold).
 
 # Example
 
@@ -52,7 +53,7 @@ Let's consider the situation where we wish to create a split where females are u
 ## Create function
 
 Our gender split function requires a single parameter, the number of folds used in cross validation. Therefore create a function with a single nfold input that returns a list of class 'splitSettings' with the 'fun' attribute specifying the 'implement' function we will use.
- 
+
 ```{r, echo = TRUE, eval=FALSE}
 createGenderSplit <- function(nfold)
   {
@@ -74,10 +75,9 @@ We now need to create the 'implement' function `implementGenderSplit()`
 
 ## Implement function
 
-All 'implement' functions for data splitting must take as input the population and the splitSettings (this is the output of the 'create' function).  They must return a data.frame containing columns: rowId and index.
-
-The index is used to determine whether the patient (identifed by the rowId) is in the test set (index = -1) or train set (index > 0).  In in the train set, the value corresponds to the cross validation fold.  For example, if rowId 2 is assigned index 5, then it means the patient with the rowId 2 is used to train the model and is in fold 5.
+All 'implement' functions for data splitting must take as input the population and the splitSettings (this is the output of the 'create' function). They must return a data.frame containing columns: rowId and index.
 
+The index is used to determine whether the patient (identifed by the rowId) is in the test set (index = -1) or train set (index \> 0). In in the train set, the value corresponds to the cross validation fold. For example, if rowId 2 is assigned index 5, then it means the patient with the rowId 2 is used to train the model and is in fold 5.
 
 ```{r tidy=FALSE,eval=FALSE}
 implementGenderSplit <- function(population, splitSettings){
@@ -100,7 +100,6 @@ implementGenderSplit <- function(population, splitSettings){
 
 ```
 
-
 # Acknowledgments
 
 Considerable work has been dedicated to provide the `PatientLevelPrediction` package.
@@ -113,8 +112,4 @@ citation("PatientLevelPrediction")
 
 [Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.](http://dx.doi.org/10.1093/jamia/ocy032)
 
-
 This work is supported in part through the National Science Foundation grant IIS 1251151.
-
-
-
diff --git a/vignettes/BenchmarkTasks.Rmd b/vignettes/BenchmarkTasks.Rmd
new file mode 100644
index 000000000..d046b5890
--- /dev/null
+++ b/vignettes/BenchmarkTasks.Rmd
@@ -0,0 +1,75 @@
+---
+title: "Benchmark Tasks"
+author: "Jenna Reps, Ross Williams, Peter R. Rijnbeek"
+date: '`r Sys.Date()`'
+header-includes:
+    - \usepackage{fancyhdr}
+    - \pagestyle{fancy}
+    - \fancyhead{}
+    - \fancyhead[CO,CE]{Installation Guide}
+    - \fancyfoot[CO,CE]{PatientLevelPrediction Package Version `r    utils::packageVersion("PatientLevelPrediction")`}
+    - \fancyfoot[LE,RO]{\thepage}
+    - \renewcommand{\headrulewidth}{0.4pt}
+    - \renewcommand{\footrulewidth}{0.4pt}
+output:
+  pdf_document:
+    includes:
+      in_header: preamble.tex
+    number_sections: yes
+    toc: yes
+  word_document:
+    toc: yes
+  html_document:
+    number_sections: yes
+    toc: yes
+---
+
+```{=html}
+<!--
+%\VignetteEngine{knitr::knitr}
+%\VignetteIndexEntry{Benchmark Tasks}
+-->
+```
+## Benchmark Tasks For Large-Scale Empirical Analyses
+
+Here we provide a set of diverse prediction tasks that can be used when evaluating the impact of the model design choice when developing models using observational data.
+
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
+| Target Cohort (index)                                                                                                                                                                                                                                                                                                                                                                                                                                                                         | Outcome                                                                                                                                                 | Time-at-risk                | Link |
++===============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================+=========================================================================================================================================================+=============================+======+
+| Patients with an outpatient visit in 2017 with no prior cancer (first visit in 2017)                                                                                                                                                                                                                                                                                                                                                                                                          | Lung cancer                                                                                                                                             | 1 day - 3 years after index |      |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
+| Patients newly diagnosed with major depressive disorder (date of first record)                                                                                                                                                                                                                                                                                                                                                                                                                | Bipolar                                                                                                                                                 | 1 day - 365 day after index |      |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
+| Patients with an outpatient visit in 2019                                                                                                                                                                                                                                                                                                                                                                                                                                                     | Dementia                                                                                                                                                | 1 day - 3 years after index |      |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
+| Patients with an outpatient visit and a positive COVID test                                                                                                                                                                                                                                                                                                                                                                                                                                   | Hospitalization with pneumonia                                                                                                                          | 1 day - 30 days after index |      |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
+| Patients with an outpatient visit and a positive COVID test                                                                                                                                                                                                                                                                                                                                                                                                                                   | Hospitalization with pneumonia that required intensive services (ventilation, intubation, tracheotomy, or extracorporeal membrane oxygenation) or death | 1 day - 30 days after index |      |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
+| Patients with an outpatient visit and a positive COVID test                                                                                                                                                                                                                                                                                                                                                                                                                                   | Death                                                                                                                                                   | 1 day - 30 days after index |      |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
+| Patients with T2DM who were treated with metformin and who became new adult users of one of sulfonylureas, thiazolidinediones, dipeptidyl peptidase-4 inhibitors, glucagon-like peptide-1 receptor agonists, or sodium-glucose co-transporter-2 inhibitors (date of secondary drug). Patients with HF or patients treated with insulin on or prior to the index date were excluded from the analysis. Patients were required to have been enrolled for at least 365 days before cohort entry. | Heart Failure                                                                                                                                           | 1 to 365 days               |      |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
+| Patients newly diagnosed with atrial fibrilation (date of initial afib record)                                                                                                                                                                                                                                                                                                                                                                                                                | Ischemic stroke                                                                                                                                         | 1 to 365 days               |      |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
+| Patients undergoing elective major non-cardiac surgery (date of surgery). Patients were required to have been enrolled for at least 365 days before cohort entry.                                                                                                                                                                                                                                                                                                                             | Earliest of AMI cardiac arrest or death (MACE)                                                                                                          | O to 30 days                |      |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
+| Patients starting intravitreal Anti-VEGF (date of administration)                                                                                                                                                                                                                                                                                                                                                                                                                             | Kidney Failure                                                                                                                                          | 1 to 365 days               |      |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
+| Pregnancy women (start of pregnancy)                                                                                                                                                                                                                                                                                                                                                                                                                                                          | Preeclampsia                                                                                                                                            | During pregnancy            |      |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
+| Pregnancy women (start of pregnancy)                                                                                                                                                                                                                                                                                                                                                                                                                                                          | Still birth                                                                                                                                             | During pregnancy            |      |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
+| Patients with COPD (first record)                                                                                                                                                                                                                                                                                                                                                                                                                                                             | Cardiovascular event and death                                                                                                                          | 1-30 days and 1-90 days     |      |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
+| Patients starting menopause (first record)                                                                                                                                                                                                                                                                                                                                                                                                                                                    | Depression                                                                                                                                              | 1 day - 3-years             |      |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
+| Patients with anemia (date of first anemia record)                                                                                                                                                                                                                                                                                                                                                                                                                                            | Colorectal cancer                                                                                                                                       | 1 day - 1-year              |      |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
+| Patients with quadriplegia (date of first quadriplegia record)                                                                                                                                                                                                                                                                                                                                                                                                                                | Death                                                                                                                                                   | 1 day - 1-year              |      |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
+| Patient undergoing                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |                                                                                                                                                         |                             |      |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
+|                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |                                                                                                                                                         |                             |      |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------+
diff --git a/vignettes/BestPractices.rmd b/vignettes/BestPractices.rmd
index 9c27f3f7c..bfc1bc792 100644
--- a/vignettes/BestPractices.rmd
+++ b/vignettes/BestPractices.rmd
@@ -101,7 +101,7 @@ Data Creation
 Impact of over/under-sampling
 </td>
 <td>
-Study being developed
+<a href='https://link.springer.com/article/10.1186/s40537-023-00857-7'> Journal of big data</a>
 </td>
 </tr>
 
@@ -125,7 +125,7 @@ Model development
 How much data do we need for prediction - Learning curves at scale
 </td>
 <td>
-<a href='https://arxiv.org/abs/2008.07361'>Preprint link</a>
+<a href='https://www.sciencedirect.com/science/article/pii/S1386505622000764'>International Journal of Medical Informatics </a>
 </td>
 </tr>
 
@@ -165,6 +165,18 @@ Study needs to be done
 </td>
 </tr>
 
+<tr>
+<td>
+Model development	
+</td>
+<td>
+Can we use ensembles to combine different algorithm models within a database to improve models transportability? 
+</td>
+<td>
+<a href='https://ebooks.iospress.nl/doi/10.3233/SHTI230080'> Caring is Sharing–Exploiting the Value in Data for Health and Innovation  </a>
+</td>
+</tr>
+
 <tr>
 <td>
 Model development	
@@ -173,10 +185,59 @@ Model development
 Can we use ensembles to combine models developed using different databases to improve models transportability?
 </td>
 <td>
-<a href=''> Paper under review at BMC </a>
+<a href='https://link.springer.com/article/10.1186/s12911-022-01879-6'> BMC Medical Informatics and Decision Making </a>
+</td>
+</tr>
+
+<tr>
+<td>
+Model development	
+</td>
+<td>
+Impact of regularization method 
+</td>
+<td>
+<a href='https://academic.oup.com/jamia/article/31/7/1514/7676584'> JAMIA </a>
+</td>
+</tr>
+
+<tr>
+<td>
+Evaluation 	
+</td>
+<td>
+Why prediction is not suitable for risk factor identification  
+</td>
+<td>
+<a href='https://proceedings.mlr.press/v182/markus22a.html'> Machine Learning for Healthcare Conference </a>
+</td>
+</tr>
+
+<tr>
+<td>
+Evaluation 	
+</td>
+<td>
+Iterative pairwise external validation to put validation into context 
+</td>
+<td>
+<a href='https://link.springer.com/article/10.1007/s40264-022-01161-8'> Drug Safety </a>
 </td>
 </tr>
 
+
+<tr>
+<td>
+Evaluation 	
+</td>
+<td>
+A novel method to estimate external validation using aggregate statistics  
+</td>
+<td>
+<a href=''> Study under review </a>
+</td>
+</tr>    
+
 <tr>
 <td>
 Evaluation
@@ -225,4 +286,4 @@ Is there a way to automatically simplify models?
 </td>
 </tr>
 </table>
-    
\ No newline at end of file
+
diff --git a/vignettes/BuildingEnsembleModels.Rmd b/vignettes/BuildingEnsembleModels.Rmd
deleted file mode 100644
index 3c052b18d..000000000
--- a/vignettes/BuildingEnsembleModels.Rmd
+++ /dev/null
@@ -1,175 +0,0 @@
----
-title: "Building Ensemble Models"
-author: "Xiaoyong Pan, Jenna Reps, Peter R. Rijnbeek"
-date: '`r Sys.Date()`'
-header-includes:
-    - \usepackage{fancyhdr}
-    - \pagestyle{fancy}
-    - \fancyhead{}
-    - \fancyhead[CO,CE]{Installation Guide}
-    - \fancyfoot[CO,CE]{PatientLevelPrediction Package Version `r    utils::packageVersion("PatientLevelPrediction")`}
-    - \fancyfoot[LE,RO]{\thepage}
-    - \renewcommand{\headrulewidth}{0.4pt}
-    - \renewcommand{\footrulewidth}{0.4pt}
-output:
-  pdf_document:
-    includes:
-      in_header: preamble.tex
-    number_sections: yes
-    toc: yes
-  word_document:
-    toc: yes
-  html_document:
-    number_sections: yes
-    toc: yes
----
-<!--
-%\VignetteEngine{knitr}
-%\VignetteIndexEntry{Building Ensemble Models}
--->
-
-```{r setup, include=FALSE}
-knitr::opts_chunk$set(echo = TRUE)
-```
-
-# Introduction
-Ensemble models combine several models to improve the overall performance. Traditionally, weak learners were combined to boost performance but recent results show that combining several strong approaches can also result in a better performance. There are many examples in literature where ensemble models outperform individual models using stacking, i.e. a final logistic regresssion layer accross the individual model outputs, but other approaches like weigthing has also shown promising results. 
-
-This vignette describes how you can use the Observational Health Data Sciencs and Informatics (OHDSI) [`PatientLevelPrediction`](http://github.com/OHDSI/PatientLevelPrediction) package to build ensemble models. This vignette assumes you have read and are comfortable with building single patient level prediction models as described in the  [`BuildingPredictiveModels` vignette](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingPredictiveModels.pdf).
-
-This will enable studying ensemble methods at scale in the OHDSI data network.
-
-![Ensemble model](ensemble.png)
-
-In PatientLevelPrediction package, four ensemble strategies have been implemented:
-
-1. average ensemble: Calculate the average probability from individual models
-2. product ensemble:  Calculate the product of probabilites from individual models.
-3. weighted ensemble: Calculate the weighted average probability from individual models using train AUC as weights.
-4. stacked ensemble:  Train a logistics regression on outputs from individual models
-
-# Usage
-
-Use the [`PatientLevelPrediction`](http://github.com/OHDSI/PatientLevelPrediction) package to generate a `population` and `plpData` object. Alternatively, you can make use of the data simulator. The following code snippet creates a population of 12000 patients.
-
-```{r eval=FALSE}
-data(plpDataSimulationProfile)
-set.seed(1234)
-sampleSize <- 2000
-plpData <- simulatePlpData(
-  plpDataSimulationProfile,
-  n = sampleSize
-)
-
-population <- createStudyPopulation(
-  plpData,
-  outcomeId = 2,
-  binary = TRUE,
-  firstExposureOnly = FALSE,
-  washoutPeriod = 0,
-  removeSubjectsWithPriorOutcome = FALSE,
-  priorOutcomeLookback = 99999,
-  requireTimeAtRisk = FALSE,
-  minTimeAtRisk = 0,
-  riskWindowStart = 0,
-  addExposureDaysToStart = FALSE,
-  riskWindowEnd = 365,
-  addExposureDaysToEnd = FALSE,
-  verbosity = "INFO"
-)
-```
-
-Specify the prediction algorithms to be combined.
-
-```{r eval=FALSE}
-# Use LASSO logistic regression and Random Forest as base predictors
-model1 <- setLassoLogisticRegression()
-model2 <- setRandomForest()
-```
-
-Specify a test fraction and a sequence of training set fractions.
-
-```{r eval = FALSE}
-testFraction <- 0.2
-```
-
-Specify an ensembleStrategy to combine multiple predictors. 
-The strategy used for ensembling the outputs from different models, 
-it can be 'mean', 'product',  'weighted' and 'stacked':
-'mean'     the average probability from differnt models
-'product'  the product rule
-'weighted' the weighted average probability from different models using train AUC as weights.
-'stacked'  the stakced ensemble trains a logistics regression on different models.
-
-```{r eval = FALSE}
-ensembleStrategy <- 'stacked'
-```
-
-Specify the test split to be used.
-
-```{r}
-# Use a split by person, alterantively a time split is possible
-testSplit <- 'person'
-```
-
-Run the ensemble learning to combine model1 and model2. You can also use different plpData for different models.
-
-```{r eval=FALSE}
-ensembleResults <- PatientLevelPrediction::runEnsembleModel(population, 
-                                   dataList = list(plpData, plpData), 
-                                   modelList = list(model1, model2),
-                                   testSplit=testSplit,
-                                   testFraction=testFraction,
-                                   nfold=3, splitSeed=1000, 
-                                   ensembleStrategy = ensembleStrategy) 
-```
-## Saving and loading the ensemble model
-
-You can save and load the model using:
-
-```{r tidy=TRUE,eval=FALSE}
-saveEnsemblePlpModel(ensembleResults$model, dirPath = file.path(getwd(),'model'))
-ensembleModel <- loadEnsemblePlpModel(getwd(),'model')
-```
-
-# Apply Ensemble model
-```{r eval=FALSE}
-plpData <- loadPlpData("<data file>")
-populationSettings <- ensembleModel$populationSettings
-populationSettings$plpData <- plpData
-population <- do.call(createStudyPopulation, populationSettings)
-```
-Load the model.
-```{r eval=FALSE}
-ensembleModel <- loadEnsemblePlpModel("<model folder>")
-```
-
-Get the predictions by applying the model:
-```{r eval=FALSE}
-prediction <- applyEnsembleModel(population,
-                                  dataList = list(plpData, plpData),
-                                  ensembleModel = ensembleModel)$prediction
-```
-
-# Demo
-
-We have added a demo of the ensemble training:
-
-``` {r eval=FALSE}
-# Show all demos in our package: 
- demo(package = "PatientLevelPrediction")
-
-# Run the learning curve
- demo("EnsembleModelDemo", package = "PatientLevelPrediction")
-```
-# Acknowledgments
-
-Considerable work has been dedicated to provide the `PatientLevelPrediction` package.
-
-```{r tidy=TRUE,eval=TRUE}
-citation("PatientLevelPrediction")
-```
-
-**Please reference this paper if you use the PLP Package in your work:**
-
-[Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.](http://dx.doi.org/10.1093/jamia/ocy032)
diff --git a/vignettes/BuildingMultiplePredictiveModels.Rmd b/vignettes/BuildingMultiplePredictiveModels.Rmd
index 8d7e218ad..d23c7326e 100644
--- a/vignettes/BuildingMultiplePredictiveModels.Rmd
+++ b/vignettes/BuildingMultiplePredictiveModels.Rmd
@@ -23,21 +23,24 @@ output:
     number_sections: yes
     toc: yes
 ---
+
+```{=html}
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Building multiple patient-level predictive models}
 -->
-
+```
 # Introduction
+
 In our [`paper`](https://academic.oup.com/jamia/article/25/8/969/4989437), we propose a standardised framework for patient-level prediction that utilizes the OMOP CDM and standardized vocabularies, and describe the open-source software that we developed implementing the framework’s pipeline. The framework is the first to enforce existing best practice guidelines and will enable open dissemination of models that can be extensively validated across the network of OHDSI collaborators.
 
-One our best practices is that we see the selection of models and all study setting as an emperical question, i.e. we should use a data-driven approach in which we try many settings. This vignette describes how you can use the Observational Health Data Sciencs and Informatics (OHDSI) [`PatientLevelPrediction`](http://github.com/OHDSI/PatientLevelPrediction) package to automatically build multiple patient-level predictive models, e.g. different population settings, covariate settings, and modelsetting. This vignette assumes you have read and are comfortable with building single patient level prediction models as described in the  [`BuildingPredictiveModels` vignette](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingPredictiveModels.pdf). 
+One our best practices is that we see the selection of models and all study setting as an emperical question, i.e. we should use a data-driven approach in which we try many settings. This vignette describes how you can use the Observational Health Data Sciencs and Informatics (OHDSI) [`PatientLevelPrediction`](http://github.com/OHDSI/PatientLevelPrediction) package to automatically build multiple patient-level predictive models, e.g. different population settings, covariate settings, and modelsetting. This vignette assumes you have read and are comfortable with building single patient level prediction models as described in the [`BuildingPredictiveModels` vignette](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingPredictiveModels.pdf).
 
 Note that it is also possible to generate a Study Package directly in Atlas that allows for multiple patient-level prediction analyses this is out-of-scope for this vignette.
 
 # Creating a model design
 
-The first step is to specify each model you wish to develop by using the `createModelDesign` function.  This function requires the following:
+The first step is to specify each model you wish to develop by using the `createModelDesign` function. This function requires the following:
 
 ```{r echo = FALSE, results ='asis'}
 library(knitr)
@@ -73,7 +76,8 @@ kable(
 ```
 
 ## Model design example 1
-For example, if we wanted to predict the outcome (id 2) occuring for the first time within 180 days of the the target population index date (id 1).  We are only interested in index dates betwrrn 2018-2020.  Finally, we only want to use age, gender in 5 year buckets and conditions as features.  The model can be specified by:
+
+For example, if we wanted to predict the outcome (id 2) occuring for the first time within 180 days of the the target population index date (id 1). We are only interested in index dates betwrrn 2018-2020. Finally, we only want to use age, gender in 5 year buckets and conditions as features. The model can be specified by:
 
 ```{r tidy=FALSE,eval=FALSE}
 
@@ -111,6 +115,7 @@ modelDesign1 <- createModelDesign(
   covariateSettings = covariateSettings, 
   featureEngineeringSettings = createFeatureEngineeringSettings(),
   sampleSettings = createSampleSettings(), 
+  splitSettings = createDefaultSplitSetting(), 
   preprocessSettings = createPreprocessSettings(), 
   modelSettings = setLassoLogisticRegression()
   )
@@ -118,7 +123,8 @@ modelDesign1 <- createModelDesign(
 ```
 
 ## Model design example 2
-For the second example, we want to predict the outcome (id 2) occuring for the first time within 730 days of the the target population index date (id 1). We want to train a random forest classifier. Finally, we want to use age, gender in 5 year buckets, drug ingredients (and groups) and conditions as features.  The model can be specified by:
+
+For the second example, we want to predict the outcome (id 2) occuring for the first time within 730 days of the the target population index date (id 1). We want to train a random forest classifier. Finally, we want to use age, gender in 5 year buckets, drug ingredients (and groups) and conditions as features. The model can be specified by:
 
 ```{r tidy=FALSE,eval=FALSE}
 
@@ -155,6 +161,7 @@ modelDesign2 <- createModelDesign(
   covariateSettings = covariateSettings, 
   featureEngineeringSettings = createRandomForestFeatureSelection(ntrees = 500, maxDepth = 7),
   sampleSettings = createSampleSettings(), 
+  splitSettings = createDefaultSplitSetting(), 
   preprocessSettings = createPreprocessSettings(), 
   modelSettings = setRandomForest()
   )
@@ -162,7 +169,8 @@ modelDesign2 <- createModelDesign(
 ```
 
 ## Model design example 3
-For the third example, we want to predict the outcome (id 5) occuring during the cohort exposure of the the target population (id 1).  We want to train a gradient boosting machine.  Finally, we want to use age, gender in 5 year buckets and indications of measurements taken as features.  The model can be specified by:
+
+For the third example, we want to predict the outcome (id 5) occuring during the cohort exposure of the the target population (id 1). We want to train a gradient boosting machine. Finally, we want to use age, gender in 5 year buckets and indications of measurements taken as features. The model can be specified by:
 
 ```{r tidy=FALSE,eval=FALSE}
 
@@ -200,16 +208,16 @@ modelDesign3 <- createModelDesign(
   covariateSettings = covariateSettings, 
   featureEngineeringSettings = createFeatureEngineeringSettings(),
   sampleSettings = createSampleSettings(), 
+  splitSettings = createDefaultSplitSetting(), 
   preprocessSettings = createPreprocessSettings(), 
   modelSettings = setGradientBoostingMachine()
   )
 
 ```
 
-
 # Running multiple models
-As we will be downloading loads of data in the multiple plp analysis it is useful to set the Andromeda temp folder to a directory with write access and plenty of space.
-`options(andromedaTempFolder = "c:/andromedaTemp")`
+
+As we will be downloading loads of data in the multiple plp analysis it is useful to set the Andromeda temp folder to a directory with write access and plenty of space. `options(andromedaTempFolder = "c:/andromedaTemp")`
 
 To run the study requires setting up a connectionDetails object
 
@@ -261,17 +269,18 @@ results <- runMultiplePlp(
     modelDesign3
     ), 
   onlyFetchData = F, 
-  splitSettings = createDefaultSplitSetting(), 
   logSettings = createLogSettings(), 
   saveDirectory =  "./PlpMultiOutput"
   )
 
 ```
-This will then save all the plpData objects from the study into "./PlpMultiOutput/plpData_T1_L<i>" and the results into "./PlpMultiOutput/Analysis_<i>".  The csv file named settings.csv found in "./PlpMultiOutput" has a row for each prediction model developed and points to the plpData and settings used for the model development, it also has descriptions of the cohorts if these are input by the user.
+
+This will then save all the plpData objects from the study into "./PlpMultiOutput/plpData_T1_L<i>" and the results into "./PlpMultiOutput/Analysis\_<i>". The csv file named settings.csv found in "./PlpMultiOutput" has a row for each prediction model developed and points to the plpData and settings used for the model development, it also has descriptions of the cohorts if these are input by the user.
 
 Note that if for some reason the run is interrupted, e.g. because of an error, a new call to `runMultiplePlp` will continue and not restart until you remove the output folder.
 
 # Validating multiple models
+
 If you have access to multiple databases on the same server in different schemas you could evaluate accross these using this call:
 
 ```{r tidy=FALSE,eval=FALSE}
@@ -292,18 +301,22 @@ val <- validateMultiplePlp(
   valdiationDatabaseDetails = validationDatabaseDetails,
   validationRestrictPlpDataSettings = createRestrictPlpDataSettings(),
   recalibrate = NULL,
-  saveDirectory = "./PlpMultiOutput/validation"
+  saveDirectory = "./PlpMultiOutput/Validation"
   )
 
 ```
-This then saves the external validation results in the validation folder of the main study (the outputLocation you used in runPlpAnalyses).
+
+This then saves the external validation results in the `Validation` folder of the main study (the outputLocation you used in runPlpAnalyses).
 
 # Viewing the results
+
 To view the results for the multiple prediction analysis:
+
 ```{r tidy=FALSE,eval=FALSE}
 viewMultiplePlp(analysesLocation="./PlpMultiOutput")
 ```
-If the validation directory in "./PlpMultiOutput" has results, the external validation will also be displayed.
+
+If the validation directory in "./PlpMultiOutput" has a sqlite results database, the external validation will also be displayed.
 
 # Acknowledgments
 
@@ -316,4 +329,3 @@ citation("PatientLevelPrediction")
 **Please reference this paper if you use the PLP Package in your work:**
 
 [Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.](http://dx.doi.org/10.1093/jamia/ocy032)
-
diff --git a/vignettes/BuildingPredictiveModels.Rmd b/vignettes/BuildingPredictiveModels.Rmd
index 7249f6421..d805c3e1c 100644
--- a/vignettes/BuildingPredictiveModels.Rmd
+++ b/vignettes/BuildingPredictiveModels.Rmd
@@ -23,18 +23,19 @@ output:
     number_sections: yes
     toc: yes
 ---
+
+```{=html}
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Building patient-level predictive models}
 -->
-
-  
+```
 ```{r echo=FALSE,message=FALSE,warning=FALSE,eval=TRUE}
 library(PatientLevelPrediction)
 vignetteDataFolder <- "s:/temp/plpVignette"
 # Load all needed data if it exists on this computer:
 if (file.exists(vignetteDataFolder)){
-  plpModel <- loadPlpModel(vignetteDataFolder,'model')
+  plpModel <- loadPlpModel(file.path(vignetteDataFolder,'model'))
   lrResults <- loadPlpModel(file.path(vignetteDataFolder,'results'))
 } 
 ```
@@ -53,16 +54,13 @@ In our [`paper`](https://academic.oup.com/jamia/article/25/8/969/4989437), we pr
 
 Figure 1, illustrates the prediction problem we address. Among a population at risk, we aim to predict which patients at a defined moment in time (t = 0) will experience some outcome during a time-at-risk. Prediction is done using only information about the patients in an observation window prior to that moment in time.
 
-![The prediction problem](Figure1.png)
-
-
-
-As shown in Figure 2, to define a prediction problem we have to define t=0 by a Target Cohort (T), the outcome we like to predict by an outcome cohort (O), and the time-at-risk (TAR). Furthermore, we  have to make design choices for the model we like to develop, and determine the observational datasets to perform internal and external validation. This conceptual framework works for all type of prediction problems, for example those presented in Figure 3.
+![The prediction problem](Figure1.webp)
 
-![Design choices](studydesign.png)
+As shown in Figure 2, to define a prediction problem we have to define t=0 by a Target Cohort (T), the outcome we like to predict by an outcome cohort (O), and the time-at-risk (TAR). Furthermore, we have to make design choices for the model we like to develop, and determine the observational datasets to perform internal and external validation. This conceptual framework works for all type of prediction problems, for example those presented in Figure 3.
 
+![Design choices](studydesign.webp)
 
-![Examples of prediction problems](problems.png)
+![Examples of prediction problems](problems.webp)
 
 This vignette describes how you can use the `PatientLevelPrediction` package to build patient-level predictive models. The package enables data extraction, model building, and model evaluation using data from databases that are translated into the OMOP CDM. In this vignette we assume you have installed the package correctly using the [`InstallationGuide`](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/InstallationGuide.pdf).
 
@@ -72,43 +70,41 @@ We have to clearly specify our study upfront to be able to implement it. This me
 
 ## Problem definition 1: Stroke in afibrilation patients
 
-Atrial fibrillation is a disease characterized by an irregular heart rate that can cause poor blood flow. Patients with atrial fibrillation are at increased risk of ischemic stroke.  Anticoagulation is a recommended prophylaxis treatment strategy for patients at high risk of stroke, though the underuse of anticoagulants and persistent severity of ischemic stroke represents a substantial unmet medical need.  Various strategies have been developed to predict risk of ischemic stroke in patients with atrial fibrillation.  CHADS2 (Gage JAMA 2001) was developed as a risk score based on history of congestive heart failure, hypertension, age>=75, diabetes and stroke.  CHADS2 was initially derived using Medicare claims data, where it achieved good discrimination (AUC=0.82).  However, subsequent external validation studies revealed the CHADS2 had substantially lower predictive accuracy (Keogh Thromb Haemost 2011).  Subsequent stroke risk calculators have been developed and evaluated, including the extension of CHADS2Vasc.  The management of atrial fibrillation has evolved substantially over the last decade, for various reasons that include the introduction of novel oral anticoagulants.  With these innovations has come a renewed interest in greater precision medicine for stroke prevention. 
+Atrial fibrillation is a disease characterized by an irregular heart rate that can cause poor blood flow. Patients with atrial fibrillation are at increased risk of ischemic stroke. Anticoagulation is a recommended prophylaxis treatment strategy for patients at high risk of stroke, though the underuse of anticoagulants and persistent severity of ischemic stroke represents a substantial unmet medical need. Various strategies have been developed to predict risk of ischemic stroke in patients with atrial fibrillation. CHADS2 (Gage JAMA 2001) was developed as a risk score based on history of congestive heart failure, hypertension, age\>=75, diabetes and stroke. CHADS2 was initially derived using Medicare claims data, where it achieved good discrimination (AUC=0.82). However, subsequent external validation studies revealed the CHADS2 had substantially lower predictive accuracy (Keogh Thromb Haemost 2011). Subsequent stroke risk calculators have been developed and evaluated, including the extension of CHADS2Vasc. The management of atrial fibrillation has evolved substantially over the last decade, for various reasons that include the introduction of novel oral anticoagulants. With these innovations has come a renewed interest in greater precision medicine for stroke prevention.
 
 We will apply the PatientLevelPrediction package to observational healthcare data to address the following patient-level prediction question:
-  
-Amongst patients who are newly diagnosed with Atrial Fibrillation, which patients will go on to have Ischemic Stroke within 1 year? 
-  
-We will define 'patients who are newly diagnosed with Atrial Fibrillation' as the first condition record of cardiac arrhythmia, which is followed by another cardiac arrhythmia condition record, at least two drug records for a drug used to treat arrhythmias, or a procedure to treat arrhythmias.  We will define 'Ischemic stroke events' as ischemic stroke condition records during an inpatient or ER visit;  successive records with > 180 day gap are considered independent episodes.
+
+Amongst patients who are newly diagnosed with Atrial Fibrillation, which patients will go on to have Ischemic Stroke within 1 year?
+
+We will define 'patients who are newly diagnosed with Atrial Fibrillation' as the first condition record of cardiac arrhythmia, which is followed by another cardiac arrhythmia condition record, at least two drug records for a drug used to treat arrhythmias, or a procedure to treat arrhythmias. We will define 'Ischemic stroke events' as ischemic stroke condition records during an inpatient or ER visit; successive records with \> 180 day gap are considered independent episodes.
 
 ## Problem definition 2: Angioedema in ACE inhibitor users
 
-Angiotensin converting enzyme inhibitors (ACE inhibitors) are medications used by patients with hypertension that widen the blood vessles and therefore increse the amount of blood pumped by the heart and decreases blood pressure.  Ace inhibitors reduce a patients risk of cardiovasular disease but can lead to drug-induced angioedema.
+Angiotensin converting enzyme inhibitors (ACE inhibitors) are medications used by patients with hypertension that widen the blood vessles and therefore increse the amount of blood pumped by the heart and decreases blood pressure. Ace inhibitors reduce a patients risk of cardiovasular disease but can lead to drug-induced angioedema.
 
 We will apply the PatientLevelPrediction package to observational healthcare data to address the following patient-level prediction question:
-  
-Amongst patients who are newly dispensed an ACE inhibitor, which patients will go on to have angioedema within 1 year? 
-  
-We will define 'patients who are newly dispensed an ACE inhibitor' as the first drug record of sny ACE inhibitor, [...]which is followed by another cardiac arrhythmia condition record, at least two drug records for a drug used to treat arrhythmias, or a procedure to treat arrhythmias.  We will define 'angioedema' as an angioedema condition record.
 
+Amongst patients who are newly dispensed an ACE inhibitor, which patients will go on to have angioedema within 1 year?
+
+We will define 'patients who are newly dispensed an ACE inhibitor' as the first drug record of sny ACE inhibitor, [...]which is followed by another cardiac arrhythmia condition record, at least two drug records for a drug used to treat arrhythmias, or a procedure to treat arrhythmias. We will define 'angioedema' as an angioedema condition record.
 
 ## Study population definition
 
-The final study population in which we will develop our model is often a subset of the Target population, because we will e.g. apply criteria that are dependent on T and O or we want to do sensitivity analyses with subpopulations of T.  For this we have to answer the following questions:
-  
-- *What is the minimum amount of observation time we require before the start of the target cohort?* This choice could depend on the available patient time in your training data, but also on the time you expect  to be available in the data sources you want to apply the model on in the future. The longer the minimum observation time, the more baseline history time is available for each person to use for feature extraction, but the fewer patients will qualify for analysis.  Moreover, there could  be clinical reasons to choose a short or longer lookback period. For our example, we will use a prior history as lookback period (washout period).
+The final study population in which we will develop our model is often a subset of the Target population, because we will e.g. apply criteria that are dependent on T and O or we want to do sensitivity analyses with subpopulations of T. For this we have to answer the following questions:
 
-- *Can patients enter the target cohort multiple times?* In the target cohort definition, a person may qualify for the cohort multiple times during different spans of time, for example if they had different episodes of a disease or separate periods of exposure to a medical product.  The cohort definition does not necessarily apply a restriction to only let the patients enter once, but in the context of a particular patient-level prediction problem, a user may want to restrict the cohort to the first qualifying episode. In our example, a person could only enter the target cohort once since our criteria was based on first occurrence of atrial fibrillation. 
+-   *What is the minimum amount of observation time we require before the start of the target cohort?* This choice could depend on the available patient time in your training data, but also on the time you expect to be available in the data sources you want to apply the model on in the future. The longer the minimum observation time, the more baseline history time is available for each person to use for feature extraction, but the fewer patients will qualify for analysis. Moreover, there could be clinical reasons to choose a short or longer lookback period. For our example, we will use a prior history as lookback period (washout period).
 
-- *Do we allow persons to enter the cohort if they experienced the outcome before?* Do we allow persons to enter the target cohort if they experienced the outcome before qualifying for the target cohort?  Depending on the particular patient-level prediction problem, there may be a desire to predict ‘incident’ first occurrence of an outcome, in which case patients who have previously experienced the outcome are not ‘at-risk’ for having a first occurrence and therefore should be excluded from the target cohort.  In other circumstances, there may be a desire to predict ‘prevalent’ episodes, whereby patients with prior outcomes can be included in the analysis and the prior outcome itself can be a predictor of future outcomes.  For our prediction example, the answer to this question is ‘Yes, allow persons with prior outcomes’ because we know from the CHADS2 score that prior strokes are very predictive of future strokes. If this answer would have been ‘No’ we also have to decide how long we would look back for previous occurrences of the outcome.
+-   *Can patients enter the target cohort multiple times?* In the target cohort definition, a person may qualify for the cohort multiple times during different spans of time, for example if they had different episodes of a disease or separate periods of exposure to a medical product. The cohort definition does not necessarily apply a restriction to only let the patients enter once, but in the context of a particular patient-level prediction problem, a user may want to restrict the cohort to the first qualifying episode. In our example, a person could only enter the target cohort once since our criteria was based on first occurrence of atrial fibrillation.
 
-- *How do we define the period in which we will predict our outcome relative to the target cohort start?* We actually have to make two decisions to answer that question. First, does the time-at-risk window start at the date of the start of the target cohort or later? Arguments to make it start later could be that you want to avoid outcomes that were entered late in the record that actually occurred before the start of the target cohort or you want to leave a gap where interventions to prevent the outcome could theoretically be implemented. Second, you need to define the time-at-risk by setting the risk window end, as some specification of days offset relative to the target cohort start or end dates. For our problem we will predict in a ‘time-at-risk’ window starting 1 day after the start of the target cohort up to 365 days later (to look for 1-year risk following atrial fibrillation diagnosis).
+-   *Do we allow persons to enter the cohort if they experienced the outcome before?* Do we allow persons to enter the target cohort if they experienced the outcome before qualifying for the target cohort? Depending on the particular patient-level prediction problem, there may be a desire to predict ‘incident’ first occurrence of an outcome, in which case patients who have previously experienced the outcome are not ‘at-risk’ for having a first occurrence and therefore should be excluded from the target cohort. In other circumstances, there may be a desire to predict ‘prevalent’ episodes, whereby patients with prior outcomes can be included in the analysis and the prior outcome itself can be a predictor of future outcomes. For our prediction example, the answer to this question is ‘Yes, allow persons with prior outcomes’ because we know from the CHADS2 score that prior strokes are very predictive of future strokes. If this answer would have been ‘No’ we also have to decide how long we would look back for previous occurrences of the outcome.
 
+-   *How do we define the period in which we will predict our outcome relative to the target cohort start?* We actually have to make two decisions to answer that question. First, does the time-at-risk window start at the date of the start of the target cohort or later? Arguments to make it start later could be that you want to avoid outcomes that were entered late in the record that actually occurred before the start of the target cohort or you want to leave a gap where interventions to prevent the outcome could theoretically be implemented. Second, you need to define the time-at-risk by setting the risk window end, as some specification of days offset relative to the target cohort start or end dates. For our problem we will predict in a ‘time-at-risk’ window starting 1 day after the start of the target cohort up to 365 days later (to look for 1-year risk following atrial fibrillation diagnosis).
 
-- *Do we require a minimum amount of time-at-risk?* We have to decide if we want to include patients that did not experience the outcome but did leave the database earlier than the end of our time-at-risk period. These patients may experience the outcome when we do not observe them. For our  prediction problem we decide to answer this question with ‘Yes, require a mimimum time-at-risk’ for that reason. Furthermore, we have to decide if this constraint also applies to persons who experienced the outcome or we will include all persons with the outcome irrespective of their total time at risk. For example, if the outcome is death, then persons with the outcome are likely censored before the full time-at-risk period is complete.
+-   *Do we require a minimum amount of time-at-risk?* We have to decide if we want to include patients that did not experience the outcome but did leave the database earlier than the end of our time-at-risk period. These patients may experience the outcome when we do not observe them. For our prediction problem we decide to answer this question with ‘Yes, require a mimimum time-at-risk’ for that reason. Furthermore, we have to decide if this constraint also applies to persons who experienced the outcome or we will include all persons with the outcome irrespective of their total time at risk. For example, if the outcome is death, then persons with the outcome are likely censored before the full time-at-risk period is complete.
 
 ## Model development settings
 
-To develop the model we have to decide which algorithm(s) we like to train. We see the selection of the best algorithm for a certain prediction problem as an empirical question, i.e. you need to let the data speak for itself and try different approaches to find the best one. There is no algorithm that will work best for all problems (no free lunch). In our package we therefore aim to implement many algorithms. Furthermore, we made the system modular so you can add your own custom algorithms as described in more detail in the [`AddingCustomModels`](https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomModels.pdf) vignette. 
+To develop the model we have to decide which algorithm(s) we like to train. We see the selection of the best algorithm for a certain prediction problem as an empirical question, i.e. you need to let the data speak for itself and try different approaches to find the best one. There is no algorithm that will work best for all problems (no free lunch). In our package we therefore aim to implement many algorithms. Furthermore, we made the system modular so you can add your own custom algorithms as described in more detail in the [`AddingCustomModels`](https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomModels.pdf) vignette.
 
 Our package currently contains the following algorithms to choose from:
 
@@ -132,81 +128,81 @@ cat(tabl) # output the table in a format good for HTML/PDF/docx conversion
 Furthermore, we have to decide on the **covariates** that we will use to train our model. This choice can be driven by domain knowledge of available computational resources. In our example, we like to add the Gender, Age, Conditions, Drugs Groups, and Visit Count. We also have to specify in which time windows we will look and we decide to look in year before and any time prior.
 
 Finally, we have to define how we will train and test our model on our data, i.e. how we perform **internal validation**. For this we have to decide how we divide our dataset in a training and testing dataset and how we randomly assign patients to these two sets. Dependent on the size of the training set we can decide how much data we like to use for training, typically this is a 75%, 25% split. If you have very large datasets you can use more data for training. To randomly assign patients to the training and testing set, there are two commonly used approaches:
-  
-1. split by person. In this case a random seed is used to assign the patient to either sets.
-2. split by time. In this case a time point is used to split the persons, e.g. 75% of the data is before and 25% is after this date. The advantage of this is that you take into consideration that the health care system has changed over time.
+
+1.  split by person. In this case a random seed is used to assign the patient to either sets.
+2.  split by time. In this case a time point is used to split the persons, e.g. 75% of the data is before and 25% is after this date. The advantage of this is that you take into consideration that the health care system has changed over time.
 
 We now completely defined our studies and implement them:
 
-- [See example 1: Stroke in afibrilation patients](#example1)
-- [See example 2: Agioedema in ACE inhibitor new users](#example2)
-      
+-   [See example 1: Stroke in afibrilation patients](#example1)
+-   [See example 2: Agioedema in ACE inhibitor new users](#example2)
+
 # Example 1: Stroke in afibrilation patients {#example1}
 
 ## Study Specification
 
 For our first prediction model we decide to start with a Regularized Logistic Regression and will use the default parameters. We will do a 75%-25% split by person.
-      
-Definition | Value
--------------------------------| -------------------------------------- |
-**Problem Definition** | |
-Target Cohort (T) |  'Patients who are newly diagnosed with Atrial Fibrillation' defined as the first condition record of cardiac arrhythmia, which is followed by another cardiac arrhythmia condition record, at least two drug records for a drug used to treat arrhythmias, or a procedure to treat arrhythmias. |
-Outcome Cohort (O) | 'Ischemic stroke events' defined as ischemic stroke condition records during an inpatient or ER visit;  successive records with > 180 day gap are considered independent episodes. |
-Time-at-risk (TAR) | 1 day till 365 days from cohort start|
-| |
-**Population Definition** | |
-Washout Period | 1095 |
-Enter the target cohort multiple times? | No|
-Allow prior outcomes? | Yes |
-Start of time-at-risk | 1 day |
-End of time-at-risk | 365 days |
-Require a minimum amount of time-at-risk? | Yes (364 days)
-| |
-**Model Development** | |
-Algorithm |  Regularized Logistic Regression |
-Hyper-parameters | variance = 0.01 (Default)
-Covariates | Gender, Age, Conditions (ever before, <365), Drugs Groups (ever before, <365), and Visit Count
-Data split | 75% train, 25% test. Randomly assigned by person
-      
+
+| Definition                                | Value                                                                                                                                                                                                                                                                                           |
+|-----------------|-------------------------------------------------------|
+| **Problem Definition**                    |                                                                                                                                                                                                                                                                                                 |
+| Target Cohort (T)                         | 'Patients who are newly diagnosed with Atrial Fibrillation' defined as the first condition record of cardiac arrhythmia, which is followed by another cardiac arrhythmia condition record, at least two drug records for a drug used to treat arrhythmias, or a procedure to treat arrhythmias. |
+| Outcome Cohort (O)                        | 'Ischemic stroke events' defined as ischemic stroke condition records during an inpatient or ER visit; successive records with \> 180 day gap are considered independent episodes.                                                                                                              |
+| Time-at-risk (TAR)                        | 1 day till 365 days from cohort start                                                                                                                                                                                                                                                           |
+|                                           |                                                                                                                                                                                                                                                                                                 |
+| **Population Definition**                 |                                                                                                                                                                                                                                                                                                 |
+| Washout Period                            | 1095                                                                                                                                                                                                                                                                                            |
+| Enter the target cohort multiple times?   | No                                                                                                                                                                                                                                                                                              |
+| Allow prior outcomes?                     | Yes                                                                                                                                                                                                                                                                                             |
+| Start of time-at-risk                     | 1 day                                                                                                                                                                                                                                                                                           |
+| End of time-at-risk                       | 365 days                                                                                                                                                                                                                                                                                        |
+| Require a minimum amount of time-at-risk? | Yes (364 days)                                                                                                                                                                                                                                                                                  |
+|                                           |                                                                                                                                                                                                                                                                                                 |
+| **Model Development**                     |                                                                                                                                                                                                                                                                                                 |
+| Algorithm                                 | Regularized Logistic Regression                                                                                                                                                                                                                                                                 |
+| Hyper-parameters                          | variance = 0.01 (Default)                                                                                                                                                                                                                                                                       |
+| Covariates                                | Gender, Age, Conditions (ever before, \<365), Drugs Groups (ever before, \<365), and Visit Count                                                                                                                                                                                                |
+| Data split                                | 75% train, 25% test. Randomly assigned by person                                                                                                                                                                                                                                                |
+
 According to the best practices we need to make a protocol that completely specifies how we plan to execute our study. This protocol will be assessed by the governance boards of the participating data sources in your network study. For this a template could be used but we prefer to automate this process as much as possible by adding functionality to automatically generate study protocol from a study specification. We will discuss this in more detail later.
-      
+
 ## Study implementation
-      
+
 Now we have completely design our study we have to implement the study. We have to generate the target and outcome cohorts and we need to develop the R code to run against our CDM that will execute the full study.
-      
+
 ### Cohort instantiation
-      
+
 For our study we need to know when a person enters the target and outcome cohorts. This is stored in a table on the server that contains the cohort start date and cohort end date for all subjects for a specific cohort definition. This cohort table has a very simple structure as shown below:
-        
-- `cohort_definition_id`, a unique identifier for distinguishing between different types of cohorts, e.g. cohorts of interest and outcome cohorts.
-- `subject_id`, a unique identifier corresponding to the `person_id` in the CDM.
-- `cohort_start_date`, the date the subject enters the cohort.
-- `cohort_end_date`, the date the subject leaves the cohort.
-      
-How do we fill this table according to our cohort definitions? There are two options for this: 
-        
-1) use the interactive cohort builder tool in [ATLAS](www.github.com/OHDSI/ATLAS) which can be used to create cohorts based on inclusion criteria and will automatically populate this cohort table.
-    
-2) write your own custom SQL statements to fill the cohort table. 
+
+-   `cohort_definition_id`, a unique identifier for distinguishing between different types of cohorts, e.g. cohorts of interest and outcome cohorts.
+-   `subject_id`, a unique identifier corresponding to the `person_id` in the CDM.
+-   `cohort_start_date`, the date the subject enters the cohort.
+-   `cohort_end_date`, the date the subject leaves the cohort.
+
+How do we fill this table according to our cohort definitions? There are two options for this:
+
+1)  use the interactive cohort builder tool in [ATLAS](www.github.com/OHDSI/ATLAS) which can be used to create cohorts based on inclusion criteria and will automatically populate this cohort table.
+
+2)  write your own custom SQL statements to fill the cohort table.
 
 Both methods are described below for our example prediction problem.
 
 ### ATLAS cohort builder
 
-![Target Cohort Atrial Fibrillation](example1/ATLAS_T.png)
+![Target Cohort Atrial Fibrillation](example1/ATLAS_T.webp)
 
-ATLAS allows you to define cohorts interactively by specifying cohort entry and cohort exit criteria. Cohort entry criteria involve selecting one or more initial events, which determine the start date for cohort entry, and optionally specifying additional inclusion criteria which filter to the qualifying events. Cohort exit criteria are applied to each cohort entry record to determine the end date when the person's episode no longer qualifies for the cohort. For the outcome cohort the end date is less relevant. As an example, Figure 4 shows how we created the Atrial Fibrillation cohort and Figure 5 shows how we created the stroke cohort in ATLAS. 
+ATLAS allows you to define cohorts interactively by specifying cohort entry and cohort exit criteria. Cohort entry criteria involve selecting one or more initial events, which determine the start date for cohort entry, and optionally specifying additional inclusion criteria which filter to the qualifying events. Cohort exit criteria are applied to each cohort entry record to determine the end date when the person's episode no longer qualifies for the cohort. For the outcome cohort the end date is less relevant. As an example, Figure 4 shows how we created the Atrial Fibrillation cohort and Figure 5 shows how we created the stroke cohort in ATLAS.
 
-![Outcome Cohort Stroke](example1/ATLAS_O.png)
+![Outcome Cohort Stroke](example1/ATLAS_O.webp)
 
-The T and O cohorts can be found here: 
+The T and O cohorts can be found here:
 
-- Atrial Fibrillaton (T): http://www.ohdsi.org/web/atlas/#/cohortdefinition/1769447
-- Stroke (O) : http://www.ohdsi.org/web/atlas/#/cohortdefinition/1769448
+-   Atrial Fibrillaton (T): <http://www.ohdsi.org/web/atlas/#/cohortdefinition/1769447>
+-   Stroke (O) : <http://www.ohdsi.org/web/atlas/#/cohortdefinition/1769448>
 
-In depth explanation of cohort creation in ATLAS is out of scope of this vignette but can be found on the OHDSI wiki pages [(link)](http://www.ohdsi.org/web/wiki/doku.php?id=documentation:software:atlas). 
+In depth explanation of cohort creation in ATLAS is out of scope of this vignette but can be found on the OHDSI wiki pages [(link)](http://www.ohdsi.org/web/wiki/doku.php?id=documentation:software:atlas).
 
-Note that when a cohort is created in ATLAS the cohortid is needed to extract the data in R. The cohortid can be found at the top of the ATLAS screen, e.g. 1769447 in Figure 4. 
+Note that when a cohort is created in ATLAS the cohortid is needed to extract the data in R. The cohortid can be found at the top of the ATLAS screen, e.g. 1769447 in Figure 4.
 
 ### Custom cohorts
 
@@ -313,10 +309,11 @@ FROM
   ;
   
 ```
+
 This is parameterized SQL which can be used by the [`SqlRender`](http://github.com/OHDSI/SqlRender) package. We use parameterized SQL so we do not have to pre-specify the names of the CDM and result schemas. That way, if we want to run the SQL on a different schema, we only need to change the parameter values; we do not have to change the SQL code. By also making use of translation functionality in `SqlRender`, we can make sure the SQL code can be run in many different environments.
-  
+
 To execute this sql against our CDM we first need to tell R how to connect to the server. `PatientLevelPrediction` uses the [`DatabaseConnector`](http://github.com/ohdsi/DatabaseConnector) package, which provides a function called `createConnectionDetails`. Type `?createConnectionDetails` for the specific settings required for the various database management systems (DBMS). For example, one might connect to a PostgreSQL database using this code:
-  
+
 ```{r tidy=FALSE,eval=FALSE}
   connectionDetails <- createConnectionDetails(dbms = "postgresql", 
   server = "localhost/ohdsi", 
@@ -327,10 +324,9 @@ To execute this sql against our CDM we first need to tell R how to connect to th
   cohortsDatabaseSchema <- "my_results"
   cdmVersion <- "5"
 ```
-  
+
 The last three lines define the `cdmDatabaseSchema` and `cohortsDatabaseSchema` variables, as well as the CDM version. We will use these later to tell R where the data in CDM format live, where we want to create the cohorts of interest, and what version CDM is used. Note that for Microsoft SQL Server, databaseschemas need to specify both the database and the schema, so for example `cdmDatabaseSchema <- "my_cdm_data.dbo"`.
-  
-  
+
 ```{r tidy=FALSE,eval=FALSE}
   library(SqlRender)
   sql <- readSql("AfStrokeCohorts.sql")
@@ -344,11 +340,11 @@ The last three lines define the `cdmDatabaseSchema` and `cohortsDatabaseSchema`
   connection <- connect(connectionDetails)
   executeSql(connection, sql)
 ```
-  
+
 In this code, we first read the SQL from the file into memory. In the next line, we replace four parameter names with the actual values. We then translate the SQL into the dialect appropriate for the DBMS we already specified in the `connectionDetails`. Next, we connect to the server, and submit the rendered and translated SQL.
-  
+
 If all went well, we now have a table with the events of interest. We can see how many events per type:
-  
+
 ```{r tidy=FALSE,eval=FALSE}
   sql <- paste("SELECT cohort_definition_id, COUNT(*) AS count",
   "FROM @cohortsDatabaseSchema.AFibStrokeCohort",
@@ -357,20 +353,21 @@ If all went well, we now have a table with the events of interest. We can see ho
   sql <- translateSql(sql, targetDialect = connectionDetails$dbms)$sql
   
   querySql(connection, sql)
-  ```
-  ```{r echo=FALSE,message=FALSE}
-  data.frame(cohort_definition_id = c(1, 2),count = c(527616, 221555))
 ```
-  
+
+```{r echo=FALSE,message=FALSE}
+data.frame(cohort_definition_id = c(1, 2),count = c(527616, 221555))
+```
+
 ### Study script creation
-  
-In this section we assume that our cohorts have been created either by using ATLAS or a custom SQL script. We will first explain how to create an R script yourself that will execute our study as we have defined earlier. 
-  
+
+In this section we assume that our cohorts have been created either by using ATLAS or a custom SQL script. We will first explain how to create an R script yourself that will execute our study as we have defined earlier.
+
 ### Data extraction
-  
-Now we can tell `PatientLevelPrediction` to extract all necessary data for our analysis. This is done using the [`FeatureExtractionPackage`](https://github.com/OHDSI/FeatureExtration). In short the FeatureExtractionPackage allows you to specify which features (covariates) need to be extracted, e.g. all conditions and drug exposures. It also supports the creation of custom covariates. For more detailed information on the FeatureExtraction package see its [vignettes](https://github.com/OHDSI/FeatureExtration). For our example study we decided to use these settings:
-  
-```{r tidy=FALSE,eval=FALSE}		   
+
+Now we can tell `PatientLevelPrediction` to extract all necessary data for our analysis. This is done using the [`FeatureExtractionPackage`](https://github.com/OHDSI/FeatureExtraction). In short the FeatureExtractionPackage allows you to specify which features (covariates) need to be extracted, e.g. all conditions and drug exposures. It also supports the creation of custom covariates. For more detailed information on the FeatureExtraction package see its [vignettes](https://github.com/OHDSI/FeatureExtraction). For our example study we decided to use these settings:
+
+```{r tidy=FALSE,eval=FALSE}
   covariateSettings <- createCovariateSettings(useDemographicsGender = TRUE,
   useDemographicsAge = TRUE,
   useConditionGroupEraLongTerm = TRUE,
@@ -381,9 +378,9 @@ Now we can tell `PatientLevelPrediction` to extract all necessary data for our a
   longTermStartDays = -365,
   endDays = -1)
 ```
-  
-The final step for extracting the data is to run the `getPlpData` function and input the connection details, the database schema where the cohorts are stored, the cohort definition ids for the cohort and outcome, and the washoutPeriod which is the minimum number of days prior to cohort index date that the person must have been observed to be included into the data, and finally input the previously constructed covariate settings. 
-  
+
+The final step for extracting the data is to run the `getPlpData` function and input the connection details, the database schema where the cohorts are stored, the cohort definition ids for the cohort and outcome, and the washoutPeriod which is the minimum number of days prior to cohort index date that the person must have been observed to be included into the data, and finally input the previously constructed covariate settings.
+
 ```{r tidy=FALSE,eval=FALSE}
 
 databaseDetails <- createDatabaseDetails(
@@ -410,26 +407,25 @@ restrictPlpDataSettings <- createRestrictPlpDataSettings(sampleSize = 10000)
     restrictPlpDataSettings = restrictPlpDataSettings
   )
 ```
-  
-Note that if the cohorts are created in ATLAS its corresponding cohort database schema needs to be selected. There are many additional parameters for the `createRestrictPlpDataSettings` function which are all documented in the `PatientLevelPrediction` manual. The resulting `plpData` object uses the package `Andromeda` (which uses [SQLite](https://www.sqlite.org/index.html)) to store information in a way that ensures R does not run out of memory, even when the data are large. 
-  
+
+Note that if the cohorts are created in ATLAS its corresponding cohort database schema needs to be selected. There are many additional parameters for the `createRestrictPlpDataSettings` function which are all documented in the `PatientLevelPrediction` manual. The resulting `plpData` object uses the package `Andromeda` (which uses [SQLite](https://www.sqlite.org/index.html)) to store information in a way that ensures R does not run out of memory, even when the data are large.
+
 Creating the `plpData` object can take considerable computing time, and it is probably a good idea to save it for future sessions. Because `plpData` uses `Andromeda`, we cannot use R's regular save function. Instead, we'll have to use the `savePlpData()` function:
-  
+
 ```{r tidy=TRUE,eval=FALSE}
   savePlpData(plpData, "stroke_in_af_data")
 ```
-  
+
 We can use the `loadPlpData()` function to load the data in a future session.
-  
-  
+
 ### Additional inclusion criteria
-  
+
 To completely define the prediction problem the final study population is obtained by applying additional constraints on the two earlier defined cohorts, e.g., a minumim time at risk can be enforced (`requireTimeAtRisk, minTimeAtRisk`) and we can specify if this also applies to patients with the outcome (`includeAllOutcomes`). Here we also specify the start and end of the risk window relative to target cohort start. For example, if we like the risk window to start 30 days after the at-risk cohort start and end a year later we can set `riskWindowStart = 30` and `riskWindowEnd = 365`. In some cases the risk window needs to start at the cohort end date. This can be achieved by setting `addExposureToStart = TRUE` which adds the cohort (exposure) time to the start date.
-  
+
 In Appendix 1, we demonstrate the effect of these settings on the subset of the persons in the target cohort that end up in the final study population.
-  
+
 In the example below all the settings we defined for our study are imposed:
-  
+
 ```{r tidy=FALSE,eval=FALSE}
   populationSettings <- createStudyPopulationSettings(
   washoutPeriod = 1095,
@@ -446,14 +442,13 @@ In the example below all the settings we defined for our study are imposed:
   )
 ```
 
-
 ### Spliting the data into training/validation/testing datasets
 
-When developing a prediction model using supervised learning (when you have features paired with labels for a set of patients), the first step is to design  the development/internal validation process.  This requires specifying how to select the model hyper-parameters, how to learn the model parameters and how to fairly evaluate the model.  In general, the validation set is used to pick hyper-parameters, the training set is used to learn the model parameters and the test set is used to perform fair internal validation.  However, cross-validation can be implemented to pick the hyper-parameters on the training data (so a validation data set is not required).  Cross validation can also be used to estimate internal validation (so a testing data set is not required).
+When developing a prediction model using supervised learning (when you have features paired with labels for a set of patients), the first step is to design the development/internal validation process. This requires specifying how to select the model hyper-parameters, how to learn the model parameters and how to fairly evaluate the model. In general, the validation set is used to pick hyper-parameters, the training set is used to learn the model parameters and the test set is used to perform fair internal validation. However, cross-validation can be implemented to pick the hyper-parameters on the training data (so a validation data set is not required). Cross validation can also be used to estimate internal validation (so a testing data set is not required).
 
-In small data the best approach for internal validation has been shown to be boostrapping.  However, in big data (many patients and many features) bootstrapping is generally not feasible. In big data our research has shown that it is just important to have some form of fair evaluation (use a test set or cross validation).  For full details see [our BMJ open paper](add link). 
+In small data the best approach for internal validation has been shown to be boostrapping. However, in big data (many patients and many features) bootstrapping is generally not feasible. In big data our research has shown that it is just important to have some form of fair evaluation (use a test set or cross validation). For full details see [our BMJ open paper](add%20link).
 
-In the PatientLevelPrediction package, the splitSettings define how the plpData are partitioned into training/validation/testing data.  Cross validation is always done, but using a test set is optional (when the data are small, it may be optimal to not use a test set).  For the splitSettings we can use the type (stratified/time/subject) and testFraction parameters to split the data in a 75%-25% split and run the patient-level prediction pipeline:
+In the PatientLevelPrediction package, the splitSettings define how the plpData are partitioned into training/validation/testing data. Cross validation is always done, but using a test set is optional (when the data are small, it may be optimal to not use a test set). For the splitSettings we can use the type (stratified/time/subject) and testFraction parameters to split the data in a 75%-25% split and run the patient-level prediction pipeline:
 
 ```{r tidy=FALSE,eval=FALSE}
   splitSettings <- createDefaultSplitSetting(
@@ -469,20 +464,17 @@ Note: it is possible to add a custom method to specify how the plpData are parti
 
 ### Preprocessing the training data
 
-There a numerous data processing settings that a user must specify when developing a prediction model.  These are: 
- * Whether to under-sample or over-sample the training data (this may be useful when there is class imballance (e.g., the outcome is very rare or very common))
- * Whether to perform feature engineering or feature selection (e.g., create latent variables that are not observed in the data or reduce the dimensionality of the data)
- * Whether to remove redundant features and normalize the data (this is required for some models)
- 
-The default sample settings does nothing, it simply returns the trainData as input, see below: 
+There a numerous data processing settings that a user must specify when developing a prediction model. These are: \* Whether to under-sample or over-sample the training data (this may be useful when there is class imballance (e.g., the outcome is very rare or very common)) \* Whether to perform feature engineering or feature selection (e.g., create latent variables that are not observed in the data or reduce the dimensionality of the data) \* Whether to remove redundant features and normalize the data (this is required for some models)
+
+The default sample settings does nothing, it simply returns the trainData as input, see below:
 
 ```{r tidy=FALSE,eval=FALSE}
   sampleSettings <- createSampleSettings()
 ```
 
-However, the current package contains methods of under-sampling the non-outcome patients.  To perform undersampling, the `type` input should be 'underSample' and `numberOutcomestoNonOutcomes` must be specified (an integer specifying the number of non-outcomes per outcome).  It is possible to add any custom function for over/under sampling, see [vignette for custom sampling](https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSamples.pdf).
+However, the current package contains methods of under-sampling the non-outcome patients. To perform undersampling, the `type` input should be 'underSample' and `numberOutcomestoNonOutcomes` must be specified (an integer specifying the number of non-outcomes per outcome). It is possible to add any custom function for over/under sampling, see [vignette for custom sampling](https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSamples.pdf).
 
-It is possible to specify a combination of feature engineering functions that take as input the trainData and output a new trainData with different features.  The default feature engineering setting does nothing:
+It is possible to specify a combination of feature engineering functions that take as input the trainData and output a new trainData with different features. The default feature engineering setting does nothing:
 
 ```{r tidy=FALSE,eval=FALSE}
   featureEngineeringSettings <- createFeatureEngineeringSettings()
@@ -490,7 +482,7 @@ It is possible to specify a combination of feature engineering functions that ta
 
 However, it is possible to add custom feature engineering functions into the pipeline, see [vignette for custom feature engineering](https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomFeatureEngineering.pdf).
 
-Finally, the preprocessing setting is required.  For this setting the user can define `minFraction`, this removes any features that is observed in the training data for less than 0.01 fraction of the patients.  So, if  `minFraction = 0.01` then any feature that is seen in less than 1 percent of the target population is removed.  The input `normalize` specifies whether the features are scaled between 0 and 1, this is required for certain models (e.g., LASSO logistic regression).  The input `removeRedundancy` specifies whether features that are observed in all of the target population are removed.
+Finally, the preprocessing setting is required. For this setting the user can define `minFraction`, this removes any features that is observed in the training data for less than 0.01 fraction of the patients. So, if `minFraction = 0.01` then any feature that is seen in less than 1 percent of the target population is removed. The input `normalize` specifies whether the features are scaled between 0 and 1, this is required for certain models (e.g., LASSO logistic regression). The input `removeRedundancy` specifies whether features that are observed in all of the target population are removed.
 
 ```{r tidy=FALSE,eval=FALSE}
   preprocessSettingsSettings <- createPreprocessSettings(
@@ -501,17 +493,17 @@ Finally, the preprocessing setting is required.  For this setting the user can d
 ```
 
 ### Model Development
-  
+
 In the set function of an algorithm the user can specify a list of eligible values for each hyper-parameter. All possible combinations of the hyper-parameters are included in a so-called grid search using cross-validation on the training set. If a user does not specify any value then the default value is used instead.
-  
-For example, if we use the following settings for the gradientBoostingMachine: ntrees=c(100,200), maxDepth=4 the grid search will apply the gradient boosting machine algorithm with ntrees=100 and maxDepth=4 plus  the  default  settings  for  other  hyper-parameters  and  ntrees=200  and  maxDepth=4  plus  the  default settings  for  other  hyper-parameters.  The  hyper-parameters  that lead to the  bestcross-validation  performance will then be chosen for the final  model. For our problem we choose to build a logistic regression model with the default hyper-parameters
-  
+
+For example, if we use the following settings for the gradientBoostingMachine: ntrees=c(100,200), maxDepth=4 the grid search will apply the gradient boosting machine algorithm with ntrees=100 and maxDepth=4 plus the default settings for other hyper-parameters and ntrees=200 and maxDepth=4 plus the default settings for other hyper-parameters. The hyper-parameters that lead to the bestcross-validation performance will then be chosen for the final model. For our problem we choose to build a logistic regression model with the default hyper-parameters
+
 ```{r tidy=TRUE,eval=FALSE}
 lrModel <- setLassoLogisticRegression()
 ```
-  
-The `runPlP` function requires the `plpData`, the `outcomeId` specifying the outcome being predicted and the settings: `populationSettings`, `splitSettings`, `sampleSettings`, `featureEngineeringSettings`, `preprocessSettings` and `modelSettings` to train and evaluate the model. 
-  
+
+The `runPlP` function requires the `plpData`, the `outcomeId` specifying the outcome being predicted and the settings: `populationSettings`, `splitSettings`, `sampleSettings`, `featureEngineeringSettings`, `preprocessSettings` and `modelSettings` to train and evaluate the model.
+
 ```{r tidy=FALSE,eval=FALSE}
   lrResults <- runPlp(
     plpData = plpData,
@@ -536,104 +528,106 @@ The `runPlP` function requires the `plpData`, the `outcomeId` specifying the out
     saveDirectory = file.path(getwd(), 'singlePlp')
     )
 ```
-Under the hood the package will now use the [`Cyclops`](www.github.com/OHDSI/Cyclops) package to fit a large-scale regularized regression using 75% of the data and will evaluate the model on the remaining 25%. A results data structure is returned containing information about the model, its performance etc. 
-  
+
+Under the hood the package will now use the [`Cyclops`](www.github.com/OHDSI/Cyclops) package to fit a large-scale regularized regression using 75% of the data and will evaluate the model on the remaining 25%. A results data structure is returned containing information about the model, its performance etc.
+
 You can save the model using:
-  
+
 ```{r tidy=TRUE,eval=FALSE}
 savePlpModel(lrResults$model, dirPath = file.path(getwd(), "model"))
 ```
-  
+
 You can load the model using:
-  
+
 ```{r tidy=TRUE,eval=FALSE}
   plpModel <- loadPlpModel(file.path(getwd(),'model'))
 ```
-  
+
 You can also save the full results structure using:
-  
+
 ```{r tidy=TRUE,eval=FALSE}
   savePlpResult(lrResults, location = file.path(getwd(),'lr'))
 ```
-  
+
 To load the full results structure use:
-  
+
 ```{r tidy=TRUE,eval=FALSE}
   lrResults <- loadPlpResult(file.path(getwd(),'lr'))
 ```
 
 \newpage
+
 # Example 2: Angioedema in ACE inhibitor users {#example2}
 
 ## Study Specification
-  
-  Definition | Value
-  -------------------------------| -------------------------------------- |
-  **Problem Definition** | |
-  Target Cohort (T) |  'Patients who are newly dispensed an ACE inhibitor' defined as the first drug record of any ACE inhibitor |
-  Outcome Cohort (O) | 'Angioedema' defined as an angioedema condition record during an inpatient or ER visit|
-  Time-at-risk (TAR) | 1 day till 365 days from cohort start|
-  | |
-  **Population Definition** | |
-  Washout Period | 365 |
-  Enter the target cohort multiple times? | No|
-  Allow prior outcomes? | No |
-  Start of time-at-risk | 1 day |
-  End of time-at-risk | 365 days |
-  Require a minimum amount of time-at-risk? | Yes (364 days)
-  | |
-  **Model Development** | |
-  Algorithm |  Gradient Boosting Machine |
-  Hyper-parameters | ntree:5000, max depth:4 or 7 or 10 and learning rate: 0.001 or 0.01 or 0.1 or 0.9
-  Covariates | Gender, Age, Conditions (ever before, <365), Drugs Groups (ever before, <365), and Visit Count
-  Data split | 75% train, 25% test. Randomly assigned by person
-  
-  According to the best practices we need to make a protocol that completely specifies how we plan to execute our study. This protocol will be assessed by the governance boards of the participating data sources in your network study. For this a template could be used but we prefer to automate this process as much as possible by adding functionality to automatically generate study protocol from a study specification. We will discuss this in more detail later.
-  
+
+| Definition                                | Value                                                                                                     |
+|----------------------|--------------------------------------------------|
+| **Problem Definition**                    |                                                                                                           |
+| Target Cohort (T)                         | 'Patients who are newly dispensed an ACE inhibitor' defined as the first drug record of any ACE inhibitor |
+| Outcome Cohort (O)                        | 'Angioedema' defined as an angioedema condition record during an inpatient or ER visit                    |
+| Time-at-risk (TAR)                        | 1 day till 365 days from cohort start                                                                     |
+|                                           |                                                                                                           |
+| **Population Definition**                 |                                                                                                           |
+| Washout Period                            | 365                                                                                                       |
+| Enter the target cohort multiple times?   | No                                                                                                        |
+| Allow prior outcomes?                     | No                                                                                                        |
+| Start of time-at-risk                     | 1 day                                                                                                     |
+| End of time-at-risk                       | 365 days                                                                                                  |
+| Require a minimum amount of time-at-risk? | Yes (364 days)                                                                                            |
+|                                           |                                                                                                           |
+| **Model Development**                     |                                                                                                           |
+| Algorithm                                 | Gradient Boosting Machine                                                                                 |
+| Hyper-parameters                          | ntree:5000, max depth:4 or 7 or 10 and learning rate: 0.001 or 0.01 or 0.1 or 0.9                         |
+| Covariates                                | Gender, Age, Conditions (ever before, \<365), Drugs Groups (ever before, \<365), and Visit Count          |
+| Data split                                | 75% train, 25% test. Randomly assigned by person                                                          |
+
+According to the best practices we need to make a protocol that completely specifies how we plan to execute our study. This protocol will be assessed by the governance boards of the participating data sources in your network study. For this a template could be used but we prefer to automate this process as much as possible by adding functionality to automatically generate study protocol from a study specification. We will discuss this in more detail later.
+
 ## Study implementation
-  
+
 Now we have completely design our study we have to implement the study. We have to generate the target and outcome cohorts and we need to develop the R code to run against our CDM that will execute the full study.
-  
+
 ### Cohort instantiation
-  
+
 For our study we need to know when a person enters the target and outcome cohorts. This is stored in a table on the server that contains the cohort start date and cohort end date for all subjects for a specific cohort definition. This cohort table has a very simple structure as shown below:
-  
-- `cohort_definition_id`, a unique identifier for distinguishing between different types of cohorts, e.g. cohorts of interest and outcome cohorts.
-- `subject_id`, a unique identifier corresponding to the `person_id` in the CDM.
-- `cohort_start_date`, the date the subject enters the cohort.
-- `cohort_end_date`, the date the subject leaves the cohort.
-  
-How do we fill this table according to our cohort definitions? There are two options for this: 
-  
-1) use the interactive cohort builder tool in [ATLAS](www.github.com/OHDSI/ATLAS) which can be used to create cohorts based on inclusion criteria and will automatically populate this cohort table.
-  
-2) write your own custom SQL statements to fill the cohort table. 
-  
+
+-   `cohort_definition_id`, a unique identifier for distinguishing between different types of cohorts, e.g. cohorts of interest and outcome cohorts.
+-   `subject_id`, a unique identifier corresponding to the `person_id` in the CDM.
+-   `cohort_start_date`, the date the subject enters the cohort.
+-   `cohort_end_date`, the date the subject leaves the cohort.
+
+How do we fill this table according to our cohort definitions? There are two options for this:
+
+1)  use the interactive cohort builder tool in [ATLAS](www.github.com/OHDSI/ATLAS) which can be used to create cohorts based on inclusion criteria and will automatically populate this cohort table.
+
+2)  write your own custom SQL statements to fill the cohort table.
+
 Both methods are described below for our example prediction problem.
-  
+
 ### ATLAS cohort builder
-  
-![Target Cohort ACE inhibitors](example2/aceinhibitors.png)
-  
-ATLAS allows you to define cohorts interactively by specifying cohort entry and cohort exit criteria. Cohort entry criteria involve selecting one or more initial events, which determine the start date for cohort entry, and optionally specifying additional inclusion criteria which filter to the qualifying events. Cohort exit criteria are applied to each cohort entry record to determine the end date when the person's episode no longer qualifies for the cohort. For the outcome cohort the end date is less relevant. As an example, Figure 6 shows how we created the ACE inhibitors cohort and Figure 7 shows how we created the angioedema cohort in ATLAS. 
-  
-![Outcome Cohort Angioedema](example2/angioedema.png)
-  
-The T and O cohorts can be found here: 
-    
-- Ace inhibitors (T): http://www.ohdsi.org/web/atlas/#/cohortdefinition/1770617
-- Angioedema (O) : http://www.ohdsi.org/web/atlas/#/cohortdefinition/1770616
-    
-In depth explanation of cohort creation in ATLAS is out of scope of this vignette but can be found on the OHDSI wiki pages [(link)](http://www.ohdsi.org/web/wiki/doku.php?id=documentation:software:atlas). 
-  
-Note that when a cohort is created in ATLAS the cohortid is needed to extract the data in R. The cohortid can be found at the top of the ATLAS screen, e.g. 1770617 in Figure 6. 
-  
+
+![Target Cohort ACE inhibitors](example2/aceinhibitors.webp)
+
+ATLAS allows you to define cohorts interactively by specifying cohort entry and cohort exit criteria. Cohort entry criteria involve selecting one or more initial events, which determine the start date for cohort entry, and optionally specifying additional inclusion criteria which filter to the qualifying events. Cohort exit criteria are applied to each cohort entry record to determine the end date when the person's episode no longer qualifies for the cohort. For the outcome cohort the end date is less relevant. As an example, Figure 6 shows how we created the ACE inhibitors cohort and Figure 7 shows how we created the angioedema cohort in ATLAS.
+
+![Outcome Cohort Angioedema](example2/angioedema.webp)
+
+The T and O cohorts can be found here:
+
+-   Ace inhibitors (T): <http://www.ohdsi.org/web/atlas/#/cohortdefinition/1770617>
+-   Angioedema (O) : <http://www.ohdsi.org/web/atlas/#/cohortdefinition/1770616>
+
+In depth explanation of cohort creation in ATLAS is out of scope of this vignette but can be found on the OHDSI wiki pages [(link)](http://www.ohdsi.org/web/wiki/doku.php?id=documentation:software:atlas).
+
+Note that when a cohort is created in ATLAS the cohortid is needed to extract the data in R. The cohortid can be found at the top of the ATLAS screen, e.g. 1770617 in Figure 6.
+
 ### Custom cohorts
-  
+
 It is also possible to create cohorts without the use of ATLAS. Using custom cohort code (SQL) you can make more advanced cohorts if needed.
-  
+
 For our example study, we need to create at table to hold the cohort data and we need to create SQL code to instantiate this table for both the AF and Stroke cohorts. Therefore, we create a file called *AceAngioCohorts.sql* with the following contents:
-    
+
 ```{sql, eval=FALSE}
   /***********************************
     File AceAngioCohorts.sql 
@@ -711,9 +705,9 @@ For our example study, we need to create at table to hold the cohort data and we
 ```
 
 This is parameterized SQL which can be used by the [`SqlRender`](http://github.com/OHDSI/SqlRender) package. We use parameterized SQL so we do not have to pre-specify the names of the CDM and result schemas. That way, if we want to run the SQL on a different schema, we only need to change the parameter values; we do not have to change the SQL code. By also making use of translation functionality in `SqlRender`, we can make sure the SQL code can be run in many different environments.
-    
+
 To execute this sql against our CDM we first need to tell R how to connect to the server. `PatientLevelPrediction` uses the [`DatabaseConnector`](http://github.com/ohdsi/DatabaseConnector) package, which provides a function called `createConnectionDetails`. Type `?createConnectionDetails` for the specific settings required for the various database management systems (DBMS). For example, one might connect to a PostgreSQL database using this code:
-      
+
 ```{r tidy=FALSE,eval=FALSE}
     connectionDetails <- createConnectionDetails(dbms = "postgresql", 
                                                  server = "localhost/ohdsi", 
@@ -724,10 +718,9 @@ To execute this sql against our CDM we first need to tell R how to connect to th
     cohortsDatabaseSchema <- "my_results"
     cdmVersion <- "5"
 ```
-    
+
 The last three lines define the `cdmDatabaseSchema` and `cohortsDatabaseSchema` variables, as well as the CDM version. We will use these later to tell R where the data in CDM format live, where we want to create the cohorts of interest, and what version CDM is used. Note that for Microsoft SQL Server, databaseschemas need to specify both the database and the schema, so for example `cdmDatabaseSchema <- "my_cdm_data.dbo"`.
-    
-    
+
 ```{r tidy=FALSE,eval=FALSE}
     library(SqlRender)
     sql <- readSql("AceAngioCohorts.sql")
@@ -739,11 +732,11 @@ The last three lines define the `cdmDatabaseSchema` and `cohortsDatabaseSchema`
     connection <- connect(connectionDetails)
     executeSql(connection, sql)
 ```
-    
+
 In this code, we first read the SQL from the file into memory. In the next line, we replace four parameter names with the actual values. We then translate the SQL into the dialect appropriate for the DBMS we already specified in the `connectionDetails`. Next, we connect to the server, and submit the rendered and translated SQL.
-    
+
 If all went well, we now have a table with the events of interest. We can see how many events per type:
-      
+
 ```{r tidy=FALSE,eval=FALSE}
     sql <- paste("SELECT cohort_definition_id, COUNT(*) AS count",
                  "FROM @cohortsDatabaseSchema.AceAngioCohort",
@@ -753,19 +746,20 @@ If all went well, we now have a table with the events of interest. We can see ho
     
     querySql(connection, sql)
 ```
+
 ```{r echo=FALSE,message=FALSE}
     data.frame(cohort_definition_id = c(1, 2),count = c(0, 0))
 ```
-    
+
 ### Study script creation
-    
-In this section we assume that our cohorts have been created either by using ATLAS or a custom SQL script. We will first explain how to create an R script yourself that will execute our study as we have defined earlier. 
-    
+
+In this section we assume that our cohorts have been created either by using ATLAS or a custom SQL script. We will first explain how to create an R script yourself that will execute our study as we have defined earlier.
+
 ### Data extraction
-    
-Now we can tell `PatientLevelPrediction` to extract all necessary data for our analysis. This is done using the [`FeatureExtractionPackage`](https://github.com/OHDSI/FeatureExtration). In short the FeatureExtractionPackage allows you to specify which features (covariates) need to be extracted, e.g. all conditions and drug exposures. It also supports the creation of custom covariates. For more detailed information on the FeatureExtraction package see its [vignettes](https://github.com/OHDSI/FeatureExtration). For our example study we decided to use these settings:
-      
-```{r tidy=FALSE,eval=FALSE}		   
+
+Now we can tell `PatientLevelPrediction` to extract all necessary data for our analysis. This is done using the [`FeatureExtractionPackage`](https://github.com/OHDSI/FeatureExtraction). In short the FeatureExtractionPackage allows you to specify which features (covariates) need to be extracted, e.g. all conditions and drug exposures. It also supports the creation of custom covariates. For more detailed information on the FeatureExtraction package see its [vignettes](https://github.com/OHDSI/FeatureExtraction). For our example study we decided to use these settings:
+
+```{r tidy=FALSE,eval=FALSE}
     covariateSettings <- createCovariateSettings(useDemographicsGender = TRUE,
                                                  useDemographicsAge = TRUE,
                                                  useConditionGroupEraLongTerm = TRUE,
@@ -776,9 +770,9 @@ Now we can tell `PatientLevelPrediction` to extract all necessary data for our a
                                                  longTermStartDays = -365,
                                                  endDays = -1)
 ```
-    
-The final step for extracting the data is to run the `getPlpData` function and input the connection details, the database schema where the cohorts are stored, the cohort definition ids for the cohort and outcome, and the washoutPeriod which is the minimum number of days prior to cohort index date that the person must have been observed to be included into the data, and finally input the previously constructed covariate settings. 
-    
+
+The final step for extracting the data is to run the `getPlpData` function and input the connection details, the database schema where the cohorts are stored, the cohort definition ids for the cohort and outcome, and the washoutPeriod which is the minimum number of days prior to cohort index date that the person must have been observed to be included into the data, and finally input the previously constructed covariate settings.
+
 ```{r tidy=FALSE,eval=FALSE}
 
 databaseDetails <- createDatabaseDetails(
@@ -803,25 +797,25 @@ plpData <- getPlpData(
   )
 
 ```
-    
-Note that if the cohorts are created in ATLAS its corresponding cohort database schema needs to be selected. There are many additional parameters for the `getPlpData` function which are all documented in the `PatientLevelPrediction` manual. The resulting `plpData` object uses the package `ff` to store information in a way that ensures R does not run out of memory, even when the data are large. 
-    
+
+Note that if the cohorts are created in ATLAS its corresponding cohort database schema needs to be selected. There are many additional parameters for the `getPlpData` function which are all documented in the `PatientLevelPrediction` manual. The resulting `plpData` object uses the package `ff` to store information in a way that ensures R does not run out of memory, even when the data are large.
+
 Creating the `plpData` object can take considerable computing time, and it is probably a good idea to save it for future sessions. Because `plpData` uses `ff`, we cannot use R's regular save function. Instead, we'll have to use the `savePlpData()` function:
-      
+
 ```{r tidy=TRUE,eval=FALSE}
     savePlpData(plpData, "angio_in_ace_data")
 ```
-    
+
 We can use the `loadPlpData()` function to load the data in a future session.
-    
+
 ### Additional inclusion criteria
-    
+
 To completely define the prediction problem the final study population is obtained by applying additional constraints on the two earlier defined cohorts, e.g., a minumim time at risk can be enforced (`requireTimeAtRisk, minTimeAtRisk`) and we can specify if this also applies to patients with the outcome (`includeAllOutcomes`). Here we also specify the start and end of the risk window relative to target cohort start. For example, if we like the risk window to start 30 days after the at-risk cohort start and end a year later we can set `riskWindowStart = 30` and `riskWindowEnd = 365`. In some cases the risk window needs to start at the cohort end date. This can be achieved by setting `addExposureToStart = TRUE` which adds the cohort (exposure) time to the start date.
-    
+
 In Appendix 1, we demonstrate the effect of these settings on the subset of the persons in the target cohort that end up in the final study population.
-    
+
 In the example below all the settings we defined for our study are imposed:
-      
+
 ```{r tidy=FALSE,eval=FALSE}
     populationSettings <- createStudyPopulationSettings(
       washoutPeriod = 364,
@@ -840,11 +834,11 @@ In the example below all the settings we defined for our study are imposed:
 
 ### Spliting the data into training/validation/testing datasets
 
-When developing a prediction model using supervised learning (when you have features paired with labels for a set of patients), the first step is to design  the development/internal validation process.  This requires specifying how to select the model hyper-parameters, how to learn the model parameters and how to fairly evaluate the model.  In general, the validation set is used to pick hyper-parameters, the training set is used to learn the model parameters and the test set is used to perform fair internal validation.  However, cross-validation can be implemented to pick the hyper-parameters on the training data (so a validation data set is not required).  Cross validation can also be used to estimate internal validation (so a testing data set is not required).
+When developing a prediction model using supervised learning (when you have features paired with labels for a set of patients), the first step is to design the development/internal validation process. This requires specifying how to select the model hyper-parameters, how to learn the model parameters and how to fairly evaluate the model. In general, the validation set is used to pick hyper-parameters, the training set is used to learn the model parameters and the test set is used to perform fair internal validation. However, cross-validation can be implemented to pick the hyper-parameters on the training data (so a validation data set is not required). Cross validation can also be used to estimate internal validation (so a testing data set is not required).
 
-In small data the best approach for internal validation has been shown to be boostrapping.  However, in big data (many patients and many features) bootstrapping is generally not feasible. In big data our research has shown that it is just important to have some form of fair evaluation (use a test set or cross validation).  For full details see [our BMJ open paper](add link). 
+In small data the best approach for internal validation has been shown to be boostrapping. However, in big data (many patients and many features) bootstrapping is generally not feasible. In big data our research has shown that it is just important to have some form of fair evaluation (use a test set or cross validation). For full details see [our BMJ open paper](add%20link).
 
-In the PatientLevelPrediction package, the splitSettings define how the plpData are partitioned into training/validation/testing data.  Cross validation is always done, but using a test set is optional (when the data are small, it may be optimal to not use a test set).  For the splitSettings we can use the type (stratified/time/subject) and testFraction parameters to split the data in a 75%-25% split and run the patient-level prediction pipeline:
+In the PatientLevelPrediction package, the splitSettings define how the plpData are partitioned into training/validation/testing data. Cross validation is always done, but using a test set is optional (when the data are small, it may be optimal to not use a test set). For the splitSettings we can use the type (stratified/time/subject) and testFraction parameters to split the data in a 75%-25% split and run the patient-level prediction pipeline:
 
 ```{r tidy=FALSE,eval=FALSE}
   splitSettings <- createDefaultSplitSetting(
@@ -856,24 +850,21 @@ In the PatientLevelPrediction package, the splitSettings define how the plpData
     )
 ```
 
-Note: it is possible to add a custom method to specify how the plpData are partitioned into training/validation/testing data, see [vignette for custom splitting](https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSplitting.pdf). 
+Note: it is possible to add a custom method to specify how the plpData are partitioned into training/validation/testing data, see [vignette for custom splitting](https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSplitting.pdf).
 
 ### Preprocessing the training data
 
-There a numerous data processing settings that a user must specify when developing a prediction model.  These are: 
- * Whether to under-sample or over-sample the training data (this may be useful when there is class imballance (e.g., the outcome is very rare or very common))
- * Whether to perform feature engineering or feature selection (e.g., create latent variables that are not observed in the data or reduce the dimensionality of the data)
- * Whether to remove redundant features and normalize the data (this is required for some models)
- 
-The default sample settings does nothing, it simply returns the trainData as input, see below: 
+There a numerous data processing settings that a user must specify when developing a prediction model. These are: \* Whether to under-sample or over-sample the training data (this may be useful when there is class imballance (e.g., the outcome is very rare or very common)) \* Whether to perform feature engineering or feature selection (e.g., create latent variables that are not observed in the data or reduce the dimensionality of the data) \* Whether to remove redundant features and normalize the data (this is required for some models)
+
+The default sample settings does nothing, it simply returns the trainData as input, see below:
 
 ```{r tidy=FALSE,eval=FALSE}
   sampleSettings <- createSampleSettings()
 ```
 
-However, the current package contains methods of under-sampling the non-outcome patients.  To perform undersampling, the `type` input should be 'underSample' and `numberOutcomestoNonOutcomes` must be specified (an integer specifying the number of non-outcomes per outcome).  It is possible to add any custom function for over/under sampling, see [vignette for custom sampling](https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSamples.pdf).
+However, the current package contains methods of under-sampling the non-outcome patients. To perform undersampling, the `type` input should be 'underSample' and `numberOutcomestoNonOutcomes` must be specified (an integer specifying the number of non-outcomes per outcome). It is possible to add any custom function for over/under sampling, see [vignette for custom sampling](https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomSamples.pdf).
 
-It is possible to specify a combination of feature engineering functions that take as input the trainData and output a new trainData with different features.  The default feature engineering setting does nothing:
+It is possible to specify a combination of feature engineering functions that take as input the trainData and output a new trainData with different features. The default feature engineering setting does nothing:
 
 ```{r tidy=FALSE,eval=FALSE}
   featureEngineeringSettings <- createFeatureEngineeringSettings()
@@ -881,7 +872,7 @@ It is possible to specify a combination of feature engineering functions that ta
 
 However, it is possible to add custom feature engineering functions into the pipeline, see [vignette for custom feature engineering](https://github.com/OHDSI/PatientLevelPrediction/blob/master/inst/doc/AddingCustomfeatureEngineering.pdf).
 
-Finally, the preprocessing setting is required.  For this setting the user can define `minFraction`, this removes any features that is observed in the training data for less than 0.01 fraction of the patients.  So, if  `minFraction = 0.01` then any feature that is seen in less than 1 percent of the target population is removed.  The input `normalize` specifies whether the features are scaled between 0 and 1, this is required for certain models (e.g., LASSO logistic regression).  The input `removeRedundancy` specifies whether features that are observed in all of the target population are removed.
+Finally, the preprocessing setting is required. For this setting the user can define `minFraction`, this removes any features that is observed in the training data for less than 0.01 fraction of the patients. So, if `minFraction = 0.01` then any feature that is seen in less than 1 percent of the target population is removed. The input `normalize` specifies whether the features are scaled between 0 and 1, this is required for certain models (e.g., LASSO logistic regression). The input `removeRedundancy` specifies whether features that are observed in all of the target population are removed.
 
 ```{r tidy=FALSE,eval=FALSE}
   preprocessSettingsSettings <- createPreprocessSettings(
@@ -892,11 +883,11 @@ Finally, the preprocessing setting is required.  For this setting the user can d
 ```
 
 ### Model Development
-    
+
 In the set function of an algorithm the user can specify a list of eligible values for each hyper-parameter. All possible combinations of the hyper-parameters are included in a so-called grid search using cross-validation on the training set. If a user does not specify any value then the default value is used instead.
-    
-For example, if we use the following settings for the gradientBoostingMachine: ntrees=c(100,200), maxDepth=4 the grid search will apply the gradient boosting machine algorithm with ntrees=100 and maxDepth=4 plus  the  default  settings  for  other  hyper-parameters  and  ntrees=200  and  maxDepth=4  plus  the  default settings  for  other  hyper-parameters.  The  hyper-parameters  that lead to the  bestcross-validation  performance will then be chosen for the final  model. For our problem we choose to build a logistic regression model with the default hyper-parameters
-    
+
+For example, if we use the following settings for the gradientBoostingMachine: ntrees=c(100,200), maxDepth=4 the grid search will apply the gradient boosting machine algorithm with ntrees=100 and maxDepth=4 plus the default settings for other hyper-parameters and ntrees=200 and maxDepth=4 plus the default settings for other hyper-parameters. The hyper-parameters that lead to the bestcross-validation performance will then be chosen for the final model. For our problem we choose to build a logistic regression model with the default hyper-parameters
+
 ```{r tidy=TRUE,eval=FALSE}
     gbmModel <- setGradientBoostingMachine(
       ntrees = 5000, 
@@ -904,9 +895,9 @@ For example, if we use the following settings for the gradientBoostingMachine: n
       learnRate = c(0.001,0.01,0.1,0.9)
       )
 ```
-    
-The `runPlP` function requires the `plpData`, the `outcomeId` specifying the outcome being predicted and the settings: `populationSettings`, `splitSettings`, `sampleSettings`, `featureEngineeringSettings`, `preprocessSettings` and `modelSettings` to train and evaluate the model. 
-  
+
+The `runPlP` function requires the `plpData`, the `outcomeId` specifying the outcome being predicted and the settings: `populationSettings`, `splitSettings`, `sampleSettings`, `featureEngineeringSettings`, `preprocessSettings` and `modelSettings` to train and evaluate the model.
+
 ```{r tidy=FALSE,eval=FALSE}
   gbmResults <- runPlp(
     plpData = plpData,
@@ -932,79 +923,103 @@ The `runPlP` function requires the `plpData`, the `outcomeId` specifying the out
     )
 ```
 
-Under the hood the package will now use the R xgboost package to fit a a gradient boosting machine model using 75% of the data and will evaluate the model on the remaining 25%. A results data structure is returned containing information about the model, its performance etc. 
-    
+Under the hood the package will now use the R xgboost package to fit a a gradient boosting machine model using 75% of the data and will evaluate the model on the remaining 25%. A results data structure is returned containing information about the model, its performance etc.
+
 You can save the model using:
-      
+
 ```{r tidy=TRUE,eval=FALSE}
     savePlpModel(gbmResults$model, dirPath = file.path(getwd(), "model"))
 ```
-    
+
 You can load the model using:
-      
+
 ```{r tidy=TRUE,eval=FALSE}
     plpModel <- loadPlpModel(file.path(getwd(),'model'))
 ```
-    
+
 You can also save the full results structure using:
-      
+
 ```{r tidy=TRUE,eval=FALSE}
     savePlpResult(gbmResults, location = file.path(getwd(),'gbm'))
 ```
-    
+
 To load the full results structure use:
-      
+
 ```{r tidy=TRUE,eval=FALSE}
     gbmResults <- loadPlpResult(file.path(getwd(),'gbm'))
 ```
-    
-    
-    
-    
+
 \newpage
-# Study package creation 
-    
+
+# Study package creation
+
 The script we created manually above can also be automatically created using a powerful feature in ATLAS. By creating a new prediction study (left menu) you can select the Target and Outcome as created in ATLAS, set all the study parameters, and then you can download a R package that you can use to execute your study. What is really powerful is that you can add multiple Ts, Os, covariate settings etc. The package will then run all the combinations of automatically as separate analyses. The screenshots below explain this process.
-    
+
 1)  <dl>
+
     <dt>Create a new prediction study and select your target and outcome cohorts.</dt>
-    <dd>![](atlasplp1.png)</dd>
+
+    <dd>![](atlasplp1.webp)</dd>
+
     </dl>
-    
+
 2)  <dl>
-  <dt>Specify one or more analysis settings.</dt>
-  <dd>![](atlasplp2.png)</dd>
-  </dl>
-  \newpage
+
+    <dt>Specify one or more analysis settings.</dt>
+
+    <dd>![](atlasplp2.web)</dd>
+
+    </dl>
+
+    \newpage
+
 3)  <dl>
-  <dt>Specify the trainings settigns</dt>
-  <dd>![](atlasplp3.png)</dd>
-  </dl>
-  
+
+    <dt>Specify the trainings settigns</dt>
+
+    <dd>![](atlasplp3.webp)</dd>
+
+    </dl>
+
 4)  <dl>
-  <dt>Specify the execution settings</dt>
-  <dd>![](atlasplp4.png)</dd>
-  </dl>  
-  
+
+    <dt>Specify the execution settings</dt>
+
+    <dd>![](atlasplp4.web)</dd>
+
+    </dl>
+]
 \newpage
-ATLAS can build a R package for you that will execute the full study against you CDM.
-Below the steps are explained how to do this in ATLAS.
 
-1) <dl>
-  <dt>Under utilities you can find download. Click on the button to review the full study specification</dt>
-  <dd>![R package download functionality in ATLAS](atlasdownload1.png)</dd>
-  </dl>
-  
-2) <dl>
-  <dt>You now have to review that you indeed want to run all these analyses (cartesian product of all the settings for each T and O combination.</dt>
-   <dd>![R package download functionality in ATLAS](atlasdownload2.png)</dd>
-   </dl>
-                                                                               
-3) If you agree, you give the package a name, and download the package as a zipfile. 
+ATLAS can build a R package for you that will execute the full study against you CDM. Below the steps are explained how to do this in ATLAS.
+
+1)  <dl>
+
+    <dt>Under utilities you can find download. Click on the button to review the full study specification</dt>
 
-4) By opening the R package in R studio and building the package you can run the study using the `execute` function. Theres is also an example CodeToRun.R script available in the extras folder of the package with extra instructions.
+    <dd>
+
+    ![R package download functionality in ATLAS](atlasdownload1.webp)
+
+    </dd>
+
+    </dl>
+
+2)  <dl>
+
+    <dt>You now have to review that you indeed want to run all these analyses (cartesian product of all the settings for each T and O combination.</dt>
+
+    <dd>
+
+    ![R package download functionality in ATLAS](atlasdownload2.webp)
+
+    </dd>
+
+    </dl>
 
+3)  If you agree, you give the package a name, and download the package as a zipfile.
 
+4)  By opening the R package in R studio and building the package you can run the study using the `execute` function. Theres is also an example CodeToRun.R script available in the extras folder of the package with extra instructions.
 
 # Internal validation
 
@@ -1012,39 +1027,60 @@ Once we execute the study, the runPlp() function returns the trained model and t
 
 You can interactively view the results by running: `viewPlp(runPlp=lrResults)`. This will generate a Shiny App in your browser in which you can view all performance measures created by the framework as shown in the figure below.
 
-<dd>![Summary of all the performance measures of the analyses](shinysummary.png)</dd>
-  
-  Furthermore, many interactive plots are available in the Shiny App, for example the ROC curve in which you can move over the plot to see the threshold and the corresponding sensitivity and specificity values.
-<dd>![Example of the interactive ROC curve](shinyroc.png)</dd>
-  
-  To generate and save all the evaluation plots to a folder run the following code:
-  
+<dd>
+
+![Summary of all the performance measures of the analyses](shinysummary.webp)
+
+</dd>
+
+Furthermore, many interactive plots are available in the Shiny App, for example the ROC curve in which you can move over the plot to see the threshold and the corresponding sensitivity and specificity values.
+
+<dd>
+
+![Example of the interactive ROC curve](shinyroc.webp)
+
+</dd>
+
+To generate and save all the evaluation plots to a folder run the following code:
+
 ```{r tidy=TRUE,eval=FALSE}
 plotPlp(lrResults, dirPath=getwd())
 ```
+
 The plots are described in more detail in the next sections.
 
 \newpage
+
 ## Discrimination
 
-The Receiver Operating Characteristics (ROC) plot shows the sensitivity against 1-specificity on the test set. The plot illustrates how well the model is able to discriminate between the people with the outcome and those without. The dashed diagonal line is the performance of a model that randomly assigns predictions. The higher the area under the ROC plot the better the discrimination of the model. The plot is created by changing the probability threshold to assign the positive class. 
+The Receiver Operating Characteristics (ROC) plot shows the sensitivity against 1-specificity on the test set. The plot illustrates how well the model is able to discriminate between the people with the outcome and those without. The dashed diagonal line is the performance of a model that randomly assigns predictions. The higher the area under the ROC plot the better the discrimination of the model. The plot is created by changing the probability threshold to assign the positive class.
 
-<dd>![Receiver Operating Characteristic Plot](sparseRoc.png)</dd>
-  
-  \newpage
-## Calibration
+<dd>
+
+![Receiver Operating Characteristic Plot](sparseRoc.webp)
+
+</dd>
+
+\newpage
+
+\## Calibration
 
 The calibration plot shows how close the predicted risk is to the observed risk. The diagonal dashed line thus indicates a perfectly calibrated model. The ten (or fewer) dots represent the mean predicted values for each quantile plotted against the observed fraction of people in that quantile who had the outcome (observed fraction). The straight black line is the linear regression using these 10 plotted quantile mean predicted vs observed fraction points. The straight vertical lines represented the 95% lower and upper confidence intervals of the slope of the fitted line.
 
-<dd>![Calibration Plot](sparseCalibration.png)</dd>
-  
+<dd>
+
+![Calibration Plot](sparseCalibration.webp)
+
+</dd>
+
 \newpage
+
 ## Smooth Calibration
 
 Similar to the traditional calibration shown above the Smooth Calibration plot shows the relationship between predicted and observed risk. the major difference is that the smooth fit allows for a more fine grained examination of this. Whereas the traditional plot will be heavily influenced by the areas with the highest density of data the smooth plot will provide the same information for this region as well as a more accurate interpretation of areas with lower density. the plot also contains information on the distribution of the outcomes relative to predicted risk.
 
 However, the increased information gain comes at a computational cost. It is recommended to use the traditional plot for examination and then to produce the smooth plot for final versions. To create the smooth calibarion plot you have to run the follow command:
-  
+
 ```{r tidy=TRUE,eval=FALSE}
 plotSmoothCalibration(lrResults)
 ```
@@ -1053,76 +1089,115 @@ See the help function for more information, on how to set the smoothing method e
 
 The example below is from another study that better demonstrates the impact of using a smooth calibration plot. The default line fit would not highlight the miss-calibration at the lower predicted probability levels that well.
 
-<dd>![Smooth Calibration plot](smoothCalibration.jpeg)</dd>
-  
-  \newpage
-## Preference distribution
+<dd>
+
+![Smooth Calibration plot](smoothCalibration.jpeg)
+
+</dd>
+
+\newpage
+
+\## Preference distribution
 
 The preference distribution plots are the preference score distributions corresponding to i) people in the test set with the outcome (red) and ii) people in the test set without the outcome (blue).
 
-<dd>![Preference Plot](preferencePDF.png)</dd>
-  
-  \newpage
-## Predicted probability distribution
+<dd>
+
+![Preference Plot](preferencePDF.webp)
+
+</dd>
+
+\newpage
+
+\## Predicted probability distribution
 
-The prediction distribution box plots are for the predicted risks of the people in the test set with the outcome (class 1: blue) and without the outcome (class 0: red). 
+The prediction distribution box plots are for the predicted risks of the people in the test set with the outcome (class 1: blue) and without the outcome (class 0: red).
 
 The box plots in the Figure show that the predicted probability of the outcome is indeed higher for those with the outcome but there is also overlap between the two distribution which lead to an imperfect discrimination.
 
-<dd>![Prediction Distribution Box Plot](predictionDistribution.png)</dd>
-  
-  \newpage
-## Test-Train similarity 
+<dd>
+
+![Prediction Distribution Box Plot](predictionDistribution.wwebp)
+
+</dd>
+
+\newpage
+
+\## Test-Train similarity
 
 The test-train similarity is assessed by plotting the mean covariate values in the train set against those in the test set for people with and without the outcome.
 
 The results for our example of look very promising since the mean values of the covariates are on the diagonal.
 
-<dd>![Similarity plots of train and test set](generalizability.png)</dd>
-  
-  \newpage
-## Variable scatter plot
+<dd>
+
+![Similarity plots of train and test set](generalizability.webp)
+
+</dd>
+
+\newpage
+
+\## Variable scatter plot
 
 The variable scatter plot shows the mean covariate value for the people with the outcome against the mean covariate value for the people without the outcome. The color of the dots corresponds to the inclusion (green) or exclusion in the model (blue), respectively. It is highly recommended to use the Shiny App since this allows you to hoover over a covariate to show more details (name, value etc).
 
-The plot shows that the mean of most of the covariates is higher for subjects with the outcome compared to those without. 
+The plot shows that the mean of most of the covariates is higher for subjects with the outcome compared to those without.
 
-<dd>![Variabel scatter Plot](variableScatterplot.png)</dd>
-  
-  \newpage
-## Precision recall
+<dd>
+
+![Variabel scatter Plot](variableScatterplot.webp)
+
+</dd>
+
+\newpage
+
+\## Precision recall
 
 Precision (P) is defined as the number of true positives (Tp) over the number of true positives plus the number of false positives (Fp).
 
 ```{r tidy=TRUE,eval=FALSE}
 P <- Tp/(Tp+Fp)
 ```
+
 Recall (R) is defined as the number of true positives (Tp) over the number of true positives plus the number of false negatives (Fn).
+
 ```{r tidy=TRUE,eval=FALSE}
 R <- Tp/(Tp + Fn)
 ```
+
 These quantities are also related to the (F1) score, which is defined as the harmonic mean of precision and recall.
+
 ```{r tidy=TRUE,eval=FALSE}
 F1 <- 2*P*R/(P+R)
 ```
-Note that the precision can either decrease or increase if the threshold is lowered.  Lowering the threshold of a classifier may increase the denominator, by increasing the number of results returned. If the threshold was previously set too high, the new results may all be true positives, which will increase precision. If the previous threshold was about right or too low, further lowering the threshold will introduce false positives, decreasing precision.
+
+Note that the precision can either decrease or increase if the threshold is lowered. Lowering the threshold of a classifier may increase the denominator, by increasing the number of results returned. If the threshold was previously set too high, the new results may all be true positives, which will increase precision. If the previous threshold was about right or too low, further lowering the threshold will introduce false positives, decreasing precision.
 
 For Recall the denominator does not depend on the classifier threshold (Tp+Fn is a constant). This means that lowering the classifier threshold may increase recall, by increasing the number of true positive results. It is also possible that lowering the threshold may leave recall unchanged, while the precision fluctuates.
 
-<dd>![Precision Recall Plot](precisionRecall.png)</dd>
-  
-  \newpage
-## Demographic summary 
+<dd>
+
+![Precision Recall Plot](precisionRecall.webp)
+
+</dd>
+
+\newpage
+
+\## Demographic summary
 
 This plot shows for females and males the expected and observed risk in different age groups together with a confidence area.
 
 The results show that our model is well calibrated across gender and age groups.
 
-<dd>![Demographic Summary Plot](demographicSummary.png)</dd>
-  
-  
-  \newpage
-# External validation
+<dd>
+
+![Demographic Summary Plot](demographicSummary.webp)
+
+</dd>
+
+\newpage
+
+\# External validation
 
 We recommend to always perform external validation, i.e. apply the final model on as much new datasets as feasible and evaluate its performance.
 
@@ -1146,30 +1221,29 @@ externalValidateDbPlp(
 
 ```
 
-This will extract the new plpData from the specified schemas and cohort tables.  It will then apply the same population settings and the trained plp model.  Finally, it will evaluate the performance and return the standard output as `validation$performanceEvaluation` and it will also return the prediction on the population as `validation$prediction`.  They can be inserted into the shiny app for viewing the model and validation by running: `viewPlp(runPlp=plpResult, validatePlp=validation )`.
-
+This will extract the new plpData from the specified schemas and cohort tables. It will then apply the same population settings and the trained plp model. Finally, it will evaluate the performance and return the standard output as `validation$performanceEvaluation` and it will also return the prediction on the population as `validation$prediction`. They can be inserted into the shiny app for viewing the model and validation by running: `viewPlp(runPlp=plpResult, validatePlp=validation )`.
 
 \newpage
+
 # Other functionality
 
 The package has much more functionality than described in this vignette and contributions have been made my many persons in the OHDSI community. The table below provides an overview:
-  
-  Functionality | Description | Vignette
---------------- | --------------------------------------------- | ---------------
-Builing Multiple Models | This vignette describes how you can run multiple models automatically | [`Vignette`](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingMultiplePredictiveModels.pdf)
-Custom Models | This vignette describes how you can add your own custom algorithms in the framework | [`Vignette`](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/AddingCustomModels.pdf)
-Custom Splitting Functions | This vignette describes how you can add your own custom training/validation/testing splitting functions in the framework | [`Vignette`](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/AddingCustomSplitting.pdf)
-Custom Sampling Functions | This vignette describes how you can add your own custom sampling functions in the framework | [`Vignette`](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/AddingCustomSamples.pdf)
-Custom Feature Engineering/Selection | This vignette describes how you can add your own custom feature engineering and selection functions in the framework | [`Vignette`](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/AddingCustomFeatureEngineering.pdf)
-Ensemble models | This vignette describes how you can use the framework to build ensemble models, i.e combine multiple models in a super learner | [`Vignette`](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingEnsembleModels.pdf)
-Learning curves | Learning curves assess the effect of training set size on model performance by training a sequence of prediction models on successively larger subsets of the training set. A learning curve plot can also help in diagnosing a bias or variance problem as explained below. | [`Vignette`](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/GeneratingLearningCurves.pdf)
 
+| Functionality                        | Description                                                                                                                                                                                                                                                                  | Vignette                                                                                                              |
+|-----------------|--------------------------------------|-----------------|
+| Builing Multiple Models              | This vignette describes how you can run multiple models automatically                                                                                                                                                                                                        | [`Vignette`](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingMultiplePredictiveModels.pdf) |
+| Custom Models                        | This vignette describes how you can add your own custom algorithms in the framework                                                                                                                                                                                          | [`Vignette`](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/AddingCustomModels.pdf)               |
+| Custom Splitting Functions           | This vignette describes how you can add your own custom training/validation/testing splitting functions in the framework                                                                                                                                                     | [`Vignette`](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/AddingCustomSplitting.pdf)            |
+| Custom Sampling Functions            | This vignette describes how you can add your own custom sampling functions in the framework                                                                                                                                                                                  | [`Vignette`](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/AddingCustomSamples.pdf)              |
+| Custom Feature Engineering/Selection | This vignette describes how you can add your own custom feature engineering and selection functions in the framework                                                                                                                                                         | [`Vignette`](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/AddingCustomFeatureEngineering.pdf)   |
+| Ensemble models                      | This vignette describes how you can use the framework to build ensemble models, i.e combine multiple models in a super learner                                                                                                                                               | [`Vignette`](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingEnsembleModels.pdf)           |
+| Learning curves                      | Learning curves assess the effect of training set size on model performance by training a sequence of prediction models on successively larger subsets of the training set. A learning curve plot can also help in diagnosing a bias or variance problem as explained below. | [`Vignette`](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/GeneratingLearningCurves.pdf)         |
 
 # Demos
 
 We have added several demos in the package that run on simulated data:
-  
-``` {r eval=FALSE}
+
+```{r eval=FALSE}
 # Show all demos in our package: 
 demo(package = "PatientLevelPrediction")
 
@@ -1194,47 +1268,73 @@ citation("Cyclops")
 ```
 
 **Please reference this paper if you use the PLP Package in your work:**
-  
-  [Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.](http://dx.doi.org/10.1093/jamia/ocy032)
+
+[Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.](http://dx.doi.org/10.1093/jamia/ocy032)
 
 This work is supported in part through the National Science Foundation grant IIS 1251151.
 
 \newpage
 
-# Appendix 1: Study population settings details {-}
+# Appendix 1: Study population settings details {.unnumbered}
 
 In the figures below the effect is shown of the removeSubjectsWithPriorOutcome, requireTimAtRisk, and includeAllOutcomes booleans on the final study population. We start with a Target Cohort with firstExposureOnly = false and we require a washout period = 1095. We then subset the target cohort based on additional constraints. The final study population in the Venn diagrams below are colored green.
 
 1)  <dl>
-  <dt>Require minimum time-at-risk for all person in the target cohort</dt>
-  <dd>![](popdef1.png)</dd>
-  </dl>
-  
-2) <dl>
-  <dt>Require minumum time-at-risk for target cohort, except for persons with outcomes during time-at-risk.</dt>
-  <dd>![](popdef2.png)</dd>
-  </dl>
-  
-  \newpage
-3) <dl>
-  <dt>Include all persons in the target cohort exclude persons with prior outcomes</dt>
-  <dd>![](popdef3.png)</dd>
-  </dl>
-  
-4) <dl>
-  <dt>Require minimum time-at-risk for target cohort, except for persons with outcomes during time-at-risk, exclude persons with prior outcomes</dt>
-  <dd>![](popdef4.png)</dd>
-  </dl>
-  
-  \newpage
-5) <dl>
-  <dt>Include all persons in target cohort exclude persons with prior outcomes</dt>
-  <dd>![](popdef5.png)</dd>
-  </dl>
-  
-6) <dl>
-  <dt>Include all persons in target cohort</dt>
-  <dd>![](popdef6.png)</dd>
-  </dl>
-  
-  
+
+    <dt>Require minimum time-at-risk for all person in the target cohort</dt>
+
+    <dd>![](popdef1.webp)</dd>
+
+    </dl>
+
+2)  <dl>
+
+    <dt>Require minumum time-at-risk for target cohort, except for persons with outcomes during time-at-risk.</dt>
+
+    <dd>![](popdef2.webp)</dd>
+
+    </dl>
+
+```{=tex}
+\newpage
+3
+```
+)
+
+<dl>
+
+<dt>Include all persons in the target cohort exclude persons with prior outcomes</dt>
+
+<dd>![](popdef3.webp)</dd>
+
+</dl>
+
+4)  <dl>
+
+    <dt>Require minimum time-at-risk for target cohort, except for persons with outcomes during time-at-risk, exclude persons with prior outcomes</dt>
+
+    <dd>![](popdef4.webp)</dd>
+
+    </dl>
+
+```{=tex}
+\newpage
+5
+```
+)
+
+<dl>
+
+<dt>Include all persons in target cohort exclude persons with prior outcomes</dt>
+
+<dd>![](popdef5.webp)</dd>
+
+</dl>
+
+6)  <dl>
+
+    <dt>Include all persons in target cohort</dt>
+
+    <dd>![](popdef6.webp)</dd>
+
+    </dl>
diff --git a/vignettes/ClinicalModels.rmd b/vignettes/ClinicalModels.rmd
new file mode 100644
index 000000000..3b6a5e5ae
--- /dev/null
+++ b/vignettes/ClinicalModels.rmd
@@ -0,0 +1,46 @@
+---
+title: "Clinical Models"
+author: "Jenna Reps, Peter R. Rijnbeek"
+date: '`r Sys.Date()`'
+header-includes:
+    - \usepackage{fancyhdr}
+    - \pagestyle{fancy}
+    - \fancyhead{}
+    - \fancyhead[CO,CE]{Installation Guide}
+    - \fancyfoot[CO,CE]{PatientLevelPrediction Package Version `r    utils::packageVersion("PatientLevelPrediction")`}
+    - \fancyfoot[LE,RO]{\thepage}
+    - \renewcommand{\headrulewidth}{0.4pt}
+    - \renewcommand{\footrulewidth}{0.4pt}
+output:
+  pdf_document:
+    includes:
+      in_header: preamble.tex
+    number_sections: yes
+    toc: yes
+  word_document:
+    toc: yes
+  html_document:
+    number_sections: yes
+    toc: yes
+---
+
+```{=html}
+<!--
+%\VignetteEngine{knitr::knitr}
+%\VignetteIndexEntry{Clinical Models}
+-->
+```
+
+## Clinical models developed using the OHDSI PatientLevelPrediction framework
+
+| Title                | Link  |
+|----------------------|-------|
+| Using Machine Learning Applied to Real-World Healthcare Data for Predictive Analytics: An Applied Example in Bariatric Surgery |  [Value in Health](https://www.sciencedirect.com/science/article/pii/S1098301519300737) |   
+| Development and validation of a prognostic model predicting symptomatic hemorrhagic transformation in acute ischemic stroke at scale in the OHDSI network |  [PLoS One](https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0226718) |  
+| Wisdom of the CROUD: development and validation of a patient-level prediction model for opioid use disorder using population-level claims data |  [PLoS One](https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0228632) | 
+| Developing predictive models to determine Patients in End-of-life Care in Administrative datasets |  [Drug Safety](https://link.springer.com/article/10.1007/s40264-020-00906-7) |   
+| Predictors of diagnostic transition from major depressive disorder to bipolar disorder: a retrospective observational network study |  [Translational psychiatry](https://www.nature.com/articles/s41398-021-01760-6) |   
+| Seek COVER: using a disease proxy to rapidly develop and validate a personalized risk calculator for COVID-19 outcomes in an international network |  [BMC Medical Research Methodology](https://link.springer.com/article/10.1186/s12874-022-01505-z) |   
+| 90-Day all-cause mortality can be predicted following a total knee replacement: an international, network study to develop and validate a prediction model |  [Knee Surgery, Sports Traumatology, Arthroscopy](https://link.springer.com/article/10.1007/s00167-021-06799-y) |   
+| Machine learning and real-world data to predict lung cancer risk in routine care |  [Cancer Epidemiology, Biomarkers & Prevention](https://aacrjournals.org/cebp/article-abstract/32/3/337/718495) |   
+| Development and validation of a patient-level model to predict dementia across a network of observational databases |  [BMC medicine](https://link.springer.com/article/10.1186/s12916-024-03530-9) |   
\ No newline at end of file
diff --git a/vignettes/ConstrainedPredictors.Rmd b/vignettes/ConstrainedPredictors.Rmd
new file mode 100644
index 000000000..226bbca06
--- /dev/null
+++ b/vignettes/ConstrainedPredictors.Rmd
@@ -0,0 +1,125 @@
+---
+title: "Constrained predictors"
+author: "Jenna Reps"
+date: '`r Sys.Date()`'
+header-includes:
+    - \usepackage{fancyhdr}
+    - \pagestyle{fancy}
+    - \fancyhead{}
+    - \fancyhead[CO,CE]{Installation Guide}
+    - \fancyfoot[CO,CE]{PatientLevelPrediction Package Version `r    utils::packageVersion("PatientLevelPrediction")`}
+    - \fancyfoot[LE,RO]{\thepage}
+    - \renewcommand{\headrulewidth}{0.4pt}
+    - \renewcommand{\footrulewidth}{0.4pt}
+output:
+  pdf_document:
+    includes:
+      in_header: preamble.tex
+    number_sections: yes
+    toc: yes
+  word_document:
+    toc: yes
+  html_document:
+    number_sections: yes
+    toc: yes
+---
+
+```{=html}
+<!--
+%\VignetteEngine{knitr::knitr}
+%\VignetteIndexEntry{Constrainted Predictors}
+-->
+```
+## Constrained Predictors
+
+### How to use the PhenotypeLibrary R package
+
+Here we provide a set of phenotypes that can be used as predictors in prediction models or best practice research.
+
+These phenotypes can be extracted from the PhenotypeLibrary R package. To install the R package run:
+
+```{r, echo = TRUE, message = FALSE, warning = FALSE, tidy = FALSE, eval=FALSE}
+remotes::install_github('ohdsi/PhenotypeLibrary')
+```
+
+To extract the cohort definition for Alcoholism with an id of 1165, just run:
+
+```{r echo = TRUE, message = FALSE, warning = FALSE, tidy = FALSE, eval = FALSE}
+PhenotypeLibrary::getPlCohortDefinitionSet(1165)
+```
+
+in general you can extract all the cohorts by running:
+
+```{r echo = TRUE, message = FALSE, warning = FALSE, tidy = FALSE, eval = FALSE}
+phenotypeDefinitions <- PhenotypeLibrary::getPlCohortDefinitionSet(1152:1215)
+```
+
+### The full set of predictor phenotypes
+
+| Phenotype Name                                  | Disorder classification     | OHDSI Phenotype library ID |
+|--------------------------------|--------------------|-------------------|
+| Alcoholism                                      | Behavioral                  | 1165                       |
+| Smoking                                         | Behavioral                  | 1166                       |
+| Anemia                                          | Blood                       | 1188                       |
+| Osteoarthritis                                  | Bone                        | 1184                       |
+| Osteoporosis                                    | Bone                        | 1185                       |
+| Cancer                                          | Cancer                      | 1215                       |
+| Atrial fibrillation                             | Cardiovascular              | 1160                       |
+| Congestive heart failure                        | Cardiovascular              | 1154                       |
+| Coronary artery disease                         | Cardiovascular              | 1162                       |
+| Heart valve disorder                            | Cardiovascular              | 1172                       |
+| Hyperlipidemia                                  | Cardiovascular              | 1170                       |
+| Hypertension                                    | Cardiovascular              | 1198                       |
+| Angina                                          | Cardiovascular              | 1159                       |
+| Skin Ulcer                                      | Debility                    | 1168                       |
+| Diabetes type 1                                 | Endocrine                   | 1193                       |
+| Diabetes type 2                                 | Endocrine                   | 1194                       |
+| Hypothyroidism                                  | Endocrine                   | 1171                       |
+| Obesity                                         | Endocrine                   | 1179                       |
+| Gastroesophageal reflux disease (GERD)          | GI                          | 1178                       |
+| Gastrointestinal (GI) bleed                     | GI                          | 1197                       |
+| Inflammatory bowel disorder (IBD)               | GI/Rheumatology             | 1180                       |
+| Hormonal contraceptives                         | Gynecologic                 | 1190                       |
+| Antibiotics Aminoglycosides                     | Infection                   | 1201                       |
+| Antibiotics Carbapenems                         | Infection                   | 1202                       |
+| Antibiotics Cephalosporins                      | Infection                   | 1203                       |
+| Antibiotics Fluoroquinolones                    | Infection                   | 1204                       |
+| Antibiotics Glycopeptides and lipoglycopeptides | Infection                   | 1205                       |
+| Antibiotics Macrolides                          | Infection                   | 1206                       |
+| Antibiotics Monobactams                         | Infection                   | 1207                       |
+| Antibiotics Oxazolidinones                      | Infection                   | 1208                       |
+| Antibiotics Penicillins                         | Infection                   | 1209                       |
+| Antibiotics Polypeptides                        | Infection                   | 1210                       |
+| Antibiotics Rifamycins                          | Infection                   | 1211                       |
+| Antibiotics Sulfonamides                        | Infection                   | 1212                       |
+| Antibiotics Streptogramins                      | Infection                   | 1213                       |
+| Antibiotics Tetracyclines                       | Infection                   | 1214                       |
+| Pneumonia                                       | Infection/Respiratory       | 1199                       |
+| Sepsis                                          | Infection                   | 1176                       |
+| Urinary tract infection (UTI)                   | Infection                   | 1186                       |
+| Hepatitis                                       | Liver                       | 1169                       |
+| Anxiety                                         | Mood                        | 1189                       |
+| Depression (MDD)                                | Mood                        | 1161                       |
+| Psychotic disorder                              | Mood                        | 1175                       |
+| Antiepileptics (pain)                           | Neurology/Pain              | 1183                       |
+| Seizure                                         | Neurology                   | 1153                       |
+| Hemorrhagic stroke                              | Neurology/Vascular          | 1156                       |
+| Non-hemorrhagic stroke                          | Neurology/Vascular          | 1155                       |
+| Acetaminophen prescription                      | Pain/Infection              | 1187                       |
+| Low back pain                                   | Pain                        | 1173                       |
+| Neuropathy                                      | Pain/Neurology              | 1174                       |
+| Opioids                                         | Pain                        | 1182                       |
+| Acute kidney injury                             | Kidney                      | 1163                       |
+| Chronic kidney disease                          | Kidney                      | 1191                       |
+| Asthma                                          | Respiratory                 | 1164                       |
+| Chronic obstructive pulmonary disorder (COPD)   | Respiratory                 | 1192                       |
+| Dyspnea                                         | Respiratory                 | 1195                       |
+| Respiratory failure                             | Respiratory                 | 1177                       |
+| Sleep apnea                                     | Respiratory                 | 1167                       |
+| Rheumatoid arthritis                            | Rheumatology                | 1200                       |
+| Steroids                                        | Rheumatology/Pain/Pulmonary | 1181                       |
+| Peripheral vascular disease                     | Vascular                    | 1157                       |
+| Aspirin                                         | Vascular                    | 1158                       |
+| Deep vein thrombosis (DVT)                      | Vascular                    | 1152                       |
+| Edema                                           | Vascular                    | 1196                       |
+| Inpatient visit                                 | NA                          | NA                         |
diff --git a/vignettes/CovCNN.png b/vignettes/CovCNN.png
deleted file mode 100644
index 82dd2832f..000000000
Binary files a/vignettes/CovCNN.png and /dev/null differ
diff --git a/vignettes/CreatingLearningCurves.Rmd b/vignettes/CreatingLearningCurves.Rmd
index 74a9e36d5..0db8d4a79 100644
--- a/vignettes/CreatingLearningCurves.Rmd
+++ b/vignettes/CreatingLearningCurves.Rmd
@@ -25,12 +25,13 @@ output:
   word_document:
     toc: yes
 ---
+
+```{=html}
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Creating Learning Curves}
 -->
-
-
+```
 ```{r setup, include=FALSE}
 knitr::opts_chunk$set(echo = TRUE)
 ```
@@ -47,7 +48,7 @@ if (file.exists(vignetteDataFolder)){
 
 # Introduction
 
-This vignette describes how you can use the Observational Health Data Sciences and Informatics (OHDSI) [`PatientLevelPrediction`](http://github.com/OHDSI/PatientLevelPrediction) package to create learning curves. This vignette assumes you have read and are comfortable with building patient level prediction models as described in the  [`BuildingPredictiveModels` vignette](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingPredictiveModels.pdf).
+This vignette describes how you can use the Observational Health Data Sciences and Informatics (OHDSI) [`PatientLevelPrediction`](http://github.com/OHDSI/PatientLevelPrediction) package to create learning curves. This vignette assumes you have read and are comfortable with building patient level prediction models as described in the [`BuildingPredictiveModels` vignette](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingPredictiveModels.pdf).
 
 Prediction models will show overly-optimistic performance when predicting on the same data as used for training. Therefore, best-practice is to partition our data into a training set and testing set. We then train our prediction model on the training set portion and asses its ability to generalize to unseen data by measuring its performance on the testing set.
 
@@ -57,9 +58,7 @@ Learning curves assess the effect of training set size on model performance by t
 
 Figure 1, shows an example of learning curve plot in which the vertical axis represents the model performance and the horizontal axis the training set size. If training set size is small, the performance on the training set is high, because a model can often be fitted well to a limited number of training examples. At the same time, the performance on the testing set will be poor, because the model trained on such a limited number of training examples will not generalize well to unseen data in the testing set. As the training set size increases, the performance of the model on the training set will decrease. It becomes more difficult for the model to find a good fit through all the training examples. Also, the model will be trained on a more representative portion of training examples, making it generalize better to unseen data. This can be observed by the increasin testing set performance.
 
-The learning curve can help us in diagnosing bias and variance problems with our classifier which will provide guidance on how to further improve our model. We can observe high variance (overfitting) in a prediction model if it performs well on the training set, but poorly on the testing set (Figure 2). Adding additional data is a common approach to counteract high variance. From the learning curve it becomes apparent, that adding additional data may improve performance on the testing set a little further, as the learning curve has not yet plateaued and, thus, the model is not saturated yet. Therefore, adding more data will decrease the gap between training set and testing set, which is the main indicator for a high variance problem. 
-
-
+The learning curve can help us in diagnosing bias and variance problems with our classifier which will provide guidance on how to further improve our model. We can observe high variance (overfitting) in a prediction model if it performs well on the training set, but poorly on the testing set (Figure 2). Adding additional data is a common approach to counteract high variance. From the learning curve it becomes apparent, that adding additional data may improve performance on the testing set a little further, as the learning curve has not yet plateaued and, thus, the model is not saturated yet. Therefore, adding more data will decrease the gap between training set and testing set, which is the main indicator for a high variance problem.
 
 ![Prediction model suffering from high variance.](learningCurveVariance.png)
 
@@ -107,7 +106,7 @@ modelSettings <- setLassoLogisticRegression()
 
 ```
 
-Specify the split settings and a sequence of training set fractions (these over ride the splitSetting trainFraction). Alternatively, instead of `trainFractions`,  you can provide a sequence of training events (`trainEvents`) instead of the training set fractions. This is recommended, because our research has shown that number of events is the important determinant of model performance. Make sure that your training set contains the number of events specified.
+Specify the split settings and a sequence of training set fractions (these over ride the splitSetting trainFraction). Alternatively, instead of `trainFractions`, you can provide a sequence of training events (`trainEvents`) instead of the training set fractions. This is recommended, because our research has shown that number of events is the important determinant of model performance. Make sure that your training set contains the number of events specified.
 
 ```{r eval = FALSE}
 
@@ -123,7 +122,6 @@ trainFractions <- seq(0.1, 0.8, 0.1) # Create eight training set fractions
 # trainEvents <- seq(100, 5000, 100)
 ```
 
-
 Create the learning curve object.
 
 ```{r eval=FALSE}
@@ -170,9 +168,9 @@ plotLearningCurve(
 
 ![Learning curve plot.](learningCurvePlot.png)
 
-
 # Parallel processing
-The learning curve object can be created in parallel, which can reduce computation time significantly.  Whether to run the code in parallel or not is specified using the `parallel` input. Currently this functionality is only available for LASSO logistic regression and gradient boosting machines. Depending on the number of parallel workers it may require a significant amount of memory. We advise to use the parallelized learning curve function for parameter search and exploratory data analysis.
+
+The learning curve object can be created in parallel, which can reduce computation time significantly. Whether to run the code in parallel or not is specified using the `parallel` input. Currently this functionality is only available for LASSO logistic regression and gradient boosting machines. Depending on the number of parallel workers it may require a significant amount of memory. We advise to use the parallelized learning curve function for parameter search and exploratory data analysis.
 
 When running in parrallel, R will find the number of available processing cores automatically and register the required parallel backend. Alternatively, you can provide the number of cores you wish to use via the `cores` input.
 
@@ -180,7 +178,7 @@ When running in parrallel, R will find the number of available processing cores
 
 We have added a demo of the learningcurve:
 
-``` {r eval=FALSE}
+```{r eval=FALSE}
 # Show all demos in our package: 
  demo(package = "PatientLevelPrediction")
 
@@ -192,8 +190,7 @@ Do note that running this demo can take a considerable amount of time (15 min on
 
 # Publication
 
-A publication titled 'How little data do we need for patient-level prediction?' uses the learning curve functionality in this package and can be accessed as preprint in the arXiv archives at
-[https://arxiv.org/abs/2008.07361](https://arxiv.org/abs/2008.07361).
+A publication titled 'How little data do we need for patient-level prediction?' uses the learning curve functionality in this package and can be accessed as preprint in the arXiv archives at <https://arxiv.org/abs/2008.07361>.
 
 # Acknowledgments
 
diff --git a/vignettes/CreatingNetworkStudies.Rmd b/vignettes/CreatingNetworkStudies.Rmd
index 26204f34f..f35961d35 100644
--- a/vignettes/CreatingNetworkStudies.Rmd
+++ b/vignettes/CreatingNetworkStudies.Rmd
@@ -14,17 +14,19 @@ output:
     number_sections: yes
     toc: yes
 ---
+
+```{=html}
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Making  patient-level predictive network study packages}
 -->
-  
+```
 ```{r echo=FALSE,message=FALSE,warning=FALSE,eval=TRUE}
 library(PatientLevelPrediction)
 vignetteDataFolder <- "s:/temp/plpVignette"
 # Load all needed data if it exists on this computer:
 if (file.exists(vignetteDataFolder)){
-  plpModel <- loadPlpModel(vignetteDataFolder,'model')
+  plpModel <- loadPlpModel(file.path(vignetteDataFolder,'model'))
   lrResults <- loadPlpModel(file.path(vignetteDataFolder,'results'))
 } 
 ```
@@ -32,49 +34,50 @@ if (file.exists(vignetteDataFolder)){
 ```{r, echo = FALSE, message = FALSE, warning = FALSE}
 library(PatientLevelPrediction)
 ```
+
 \newpage
+
 # Introduction
-The OHDSI Patient Level Prediction (PLP) package provides the framework to implement prediction models at scale.  This can range from developing a large number of models across sites (methodology and study design insight) to extensive external validation of existing models in the OHDSI PLP framework (model insight). This vignette describes how you can use the `PatientLevelPrediction` package to  create a network study package.
+
+The OHDSI Patient Level Prediction (PLP) package provides the framework to implement prediction models at scale. This can range from developing a large number of models across sites (methodology and study design insight) to extensive external validation of existing models in the OHDSI PLP framework (model insight). This vignette describes how you can use the `PatientLevelPrediction` package to create a network study package.
 
 # Useful publication
-The open access publication [A standardized analytics pipeline for reliable and rapid development and validation of prediction models using observational health data](https://pubmed.ncbi.nlm.nih.gov/34560604/) details the process used to develop and validate prediction models using the OHDSI prediction framework and tools.  This publication describes each of the steps and then demonstrates these by focusing on predicting death in those who have covid-19.
+
+The open access publication [A standardized analytics pipeline for reliable and rapid development and validation of prediction models using observational health data](https://pubmed.ncbi.nlm.nih.gov/34560604/) details the process used to develop and validate prediction models using the OHDSI prediction framework and tools. This publication describes each of the steps and then demonstrates these by focusing on predicting death in those who have covid-19.
 
 # Main steps for running a network study
+
 ## Step 1 – developing the study
 
- *	Design the study: target/outcome cohort logic, concept sets for medical definitions, settings for developing new model or validation of adding existing models to framework.  Suggestion: look in literature for validated definitions.
- *	Write a protocol that motivates the study and provides full details (sufficient for people to replicate the study in the future).
- *	Write an R package for implementing the study across diverse computational environments [see guidance below for structure of package and use the skeleton github package here: https://github.com/OHDSI/SkeletonPredictionStudy ]
+-   Design the study: target/outcome cohort logic, concept sets for medical definitions, settings for developing new model or validation of adding existing models to framework. Suggestion: look in literature for validated definitions.
+-   Write a protocol that motivates the study and provides full details (sufficient for people to replicate the study in the future).
+-   Write an R package for implementing the study across diverse computational environments [see guidance below for structure of package and use the skeleton github package here: <https://github.com/OHDSI/SkeletonPredictionStudy> ]
 
 ## Step 2 – implementing the study part 1
 
- *	Get contributors to install the package and dependencies.  Ensure the package is installed correctly for each contributor by asking them to run the checkInstall functions (as specified in the InstallationGuide).
- *	Get contributors to run the createCohort function to inspect the target/outcome definitions.  If the definitions are not suitable for a site, go back to step 1 and revise the cohort definitions.   
+-   Get contributors to install the package and dependencies. Ensure the package is installed correctly for each contributor by asking them to run the checkInstall functions (as specified in the InstallationGuide).
+-   Get contributors to run the createCohort function to inspect the target/outcome definitions. If the definitions are not suitable for a site, go back to step 1 and revise the cohort definitions.
 
 ## Step 3 – implementing the study part 2 (make sure the package is functioning as planned and the definitions are valid across sites)
 
- *	Get contributors to run the main.R with the settings configured to their environment
- *	Get the contributors to submit the results
+-   Get contributors to run the main.R with the settings configured to their environment
+-   Get the contributors to submit the results
 
 ## Step 4 – Publication
 
-The study creator has the first option to be first author, if he/she does not wish to be first author then he/she can pick the most suitable person from the contributors.  All contributors will be listed as authors on the paper.  The last author will be the person who lead/managed the study, if this was the first author then the first author can pick the most suitable last author.  All authors between the first and last author will be alphabetical by last name.
-
+The study creator has the first option to be first author, if he/she does not wish to be first author then he/she can pick the most suitable person from the contributors. All contributors will be listed as authors on the paper. The last author will be the person who lead/managed the study, if this was the first author then the first author can pick the most suitable last author. All authors between the first and last author will be alphabetical by last name.
 
 # Package Skeleton - File Structure
 
- *	DESCRIPTION: This file describes the R package and the dependencies
- *	NAMESPACE: This file is created automatically by Roxygen
- *	Readme.md: This file should provide the step by step guidance on implementing the package
- *	R 
-   -	helpers.r: all the custom functions used by the package should be in this file (e.g., checkInstall)
-   -	main.r: this file will call the functions in helpers.r to execute the full study
-   - submit.r: this file will be called at the end to submit the compressed folder to the study creator/manager.
- *	Man: this folder will contain the documentation for the functions in helpers.r (this should be automatically generated by roxygen)
- *	Inst
-   -	sql/sql_sever
-     * targetCohort: the target cohort parameterised sql code
-     * outcomeCohort: the outcome cohort parameterised sql code
-   - plp_models: place any PLP models here
- * Extras 
-
+-   DESCRIPTION: This file describes the R package and the dependencies
+-   NAMESPACE: This file is created automatically by Roxygen
+-   Readme.md: This file should provide the step by step guidance on implementing the package
+-   R
+-   helpers.r: all the custom functions used by the package should be in this file (e.g., checkInstall)
+-   main.r: this file will call the functions in helpers.r to execute the full study
+-   submit.r: this file will be called at the end to submit the compressed folder to the study creator/manager.
+-   Man: this folder will contain the documentation for the functions in helpers.r (this should be automatically generated by roxygen)
+-   Inst
+-   sql/sql_sever \* targetCohort: the target cohort parameterised sql code \* outcomeCohort: the outcome cohort parameterised sql code
+-   plp_models: place any PLP models here
+-   Extras
diff --git a/vignettes/CreatingShinyApp.Rmd b/vignettes/CreatingShinyApp.Rmd
deleted file mode 100644
index 59b1e099d..000000000
--- a/vignettes/CreatingShinyApp.Rmd
+++ /dev/null
@@ -1,298 +0,0 @@
----
-title: "Creating Shiny App"
-author: "Jenna Reps"
-date: '`r Sys.Date()`'
-header-includes:
-    - \usepackage{fancyhdr}
-    - \pagestyle{fancy}
-    - \fancyhead{}
-    - \fancyhead[CO,CE]{Installation Guide}
-    - \fancyfoot[CO,CE]{PatientLevelPrediction Package Version `r    utils::packageVersion("PatientLevelPrediction")`}
-    - \fancyfoot[LE,RO]{\thepage}
-    - \renewcommand{\headrulewidth}{0.4pt}
-    - \renewcommand{\footrulewidth}{0.4pt}
-output:
-  pdf_document:
-    includes:
-      in_header: preamble.tex
-    number_sections: yes
-    toc: yes
-  word_document:
-    toc: yes
-  html_document:
-    number_sections: yes
-    toc: yes
----
-<!--
-%\VignetteEngine{knitr}
-%\VignetteIndexEntry{Creating Shiny App}
--->
-
-```{r setup, include=FALSE}
-knitr::opts_chunk$set(echo = TRUE)
-```
-
-# Introduction
-In this vignette we will show with example code how to create a shiny app and add the shiny app online for other researcher around the whole to explore.
-
-There are two ways to create the shiny app:
-1) Using the atlas R generated prediction package 
-2) Manually using the PatientLevelPrediction functions in a script
-
-We assume you have experience with using the OHDSI PatientLevelPrediction package to develop and externally validate prediction models using data in the OMOP CDM.  If you do not have experience with this then please first read our general vignette [`BuildingPredictiveModels` vignette](https://github.com/OHDSI/PatientLevelPrediction/blob/main/inst/doc/BuildingPredictiveModels.pdf).
-
-# Atlas Development Shiny App
-
-## Step 1: Run the model development package to get results
-To create a shiny app project via the Atlas auto-generated prediction R package you named 'myPackage' you need to run the execute function:
-```{r eval = FALSE}
-library(myPackage)
-myPackage::execute(connectionDetails = connectionDetails,
-        cdmDatabaseSchema = 'myDatabaseSchema.dbo',
-        cdmDatabaseName = 'MyDatabase',
-        cohortDatabaseSchema = 'myDatabaseSchema.ohdsi_results',
-        cohortTable = 'cohort',
-        outputFolder = 'C:/myResults',
-        createProtocol = F,
-        createCohorts = F,
-        runAnalyses = T,
-        createResultsDoc = F,
-        packageResults = F,
-        createValidationPackage = F, 
-        minCellCount= 5,
-        createShiny = F,
-        createJournalDocument = F,
-        analysisIdDocument = 1)
-```
-
-This will extract data based on the settings you supplied in the Atlas prediction design from cohort tables already generated in your CDM database schema.  The PatientLevelPrediction framework will then run and develop/evaluate models saving the results to the location specified by outputFolder (e.g., 'C:/myResults').
-
-## Step 2: Create the shiny app
-To create a shiny app project with these results you can then simply run:
-```{r eval = FALSE}
-myPackage::execute(connectionDetails = connectionDetails,
-        cdmDatabaseSchema = 'myDatabaseSchema.dbo',
-        cdmDatabaseName = 'MyDatabase',
-        cohortDatabaseSchema = 'myDatabaseSchema.ohdsi_results',
-        cohortTable = 'cohort',
-        outputFolder = 'C:/myResults',
-        minCellCount= 5,
-        createShiny = T)
-```
-
-making sure the outputFolder is the same location used when you ran the analysis.  This code populates a shiny app project with the results but removes sensitive date such as connection settings, the cdmDatabaseSchema setting, the predicton matrix and any sensitive counts less than 'minCellCount' from the covariate summary and performance evalaution.
-
-The shiny app project populated with the model development results can then be found at '[outputFolder]/ShinyApp' e.g., 'C:/myResults/ShinyApp'.
-
-
-### Testing (Optional but recommended)
-You can test the app by opening the shiny project within the [outputFolder]/ShinyApp' folder, double click on the file named 'PLPViewer.Rproj'.  This will open an R studio session with the shiny app project loaded. Now load the 'ui.R' files within this R studio session and you will see a green arrow with the words 'Run App' at the top right of the script.  Click on this and the shiny app with open.  Note: You may need to install some R pacakge dependancies for the shiny app to work.
-
-## Step 3: Sharing the shiny app
-Once you are happy with your app, you can publish it onto https://data.ohdsi.org by adding the folder 'ShinyApp' to the OHDSI githib ShinyDeploy (https://github.com/OHDSI/ShinyDeploy/).  Continuing the example, we would copy the folder '[outputFolder]/ShinyApp' and paste it to the local github clone of ShinyDeploy.  We recommend renaming the folder from 'ShinyApp' to a name that describes your prediction, e.g., 'StrokeinAF'.  Then commit the changes and make a pull request to ShinyDeploy.  Once accepted your shiny app will be viewable at 'https://data.ohdsi.org'.  If you commited the folder named 'StrokeInAF' then the shiny app will be hosted at 'https://data.ohdsi.org/StrokeInAF'.
-
-
-# Atlas External Validation
-
-To include external validation results you can use the Atlas generated R study package to create the external validation package:
-```{r eval = FALSE}
-myPackage::execute(connectionDetails = connectionDetails,
-        cdmDatabaseSchema = 'myDatabaseSchema.dbo',
-        cdmDatabaseName = 'MyDatabase',
-        cohortDatabaseSchema = 'myDatabaseSchema.ohdsi_results',
-        cohortTable = 'cohort',
-        outputFolder = 'C:/myResults',
-        createValidationPackage = T)
-```
-
-This will create a new R package inside the 'outputFolder' location with the word 'Validation' appended the name of your development package.  For example, if your 'outputFolder' was 'C:/myResults' and your development package was named  'myPackage' then the validation package will be found at: 'C:/myResults/myPackageValidation'.  When running the valdiation package make sure to set the 'outputFolder' to the Validation folder within your model development outputFolder location: 
-
-```{r eval = FALSE}
-myPackageValidation::execute(connectionDetails = connectionDetails,
-                 databaseName = databaseName,
-                 cdmDatabaseSchema = cdmDatabaseSchema,
-                 cohortDatabaseSchema = cohortDatabaseSchema,
-                 oracleTempSchema = oracleTempSchema,
-                 cohortTable = cohortTable,
-                 outputFolder = 'C:/myResults/Validation',
-                 createCohorts = T,
-                 runValidation = T,
-                 packageResults = F,
-                 minCellCount = 5,
-                 sampleSize = NULL)
-
-```
-
-Now you can rerun Steps 2-3 to populate the shiny app project that will also include the validation results (as long as the validation results are in the Validation folder found in the Step 1 outputFolder location e.g., in 'C:/myResults/Validation').
-
-
-# Combining multiple atlas results into one shiny app:
-The code below can be used to combine multiple Atlas packages' results into one shiny app:
-
-```{r eval = FALSE}
-populateMultipleShinyApp <- function(shinyDirectory,
-                             resultDirectory,
-                             minCellCount = 10,
-                             databaseName = 'sharable name of development data'){
-  
-  #check inputs
-  if(missing(shinyDirectory)){
-    shinyDirectory <- system.file("shiny", "PLPViewer", package = "SkeletonPredictionStudy")
-  }
-  if(missing(resultDirectory)){
-    stop('Need to enter the resultDirectory')
-  }
-  
-
-    for(i in 1:length(resultDirectory)){
-      if(!dir.exists(resultDirectory[i])){
-        stop(paste('resultDirectory ',i,' does not exist'))
-      }
-    }
-  
-  outputDirectory <- file.path(shinyDirectory,'data')
-  
-  # create the shiny data folder
-  if(!dir.exists(outputDirectory)){
-    dir.create(outputDirectory, recursive = T)
-  }
-  
-  
-  # need to edit settings ...
-  files <- c()
-  for(i in 1:length(resultDirectory)){
-  # copy the settings csv
-  file <- utils::read.csv(file.path(resultDirectory[i],'settings.csv'))
-  file$analysisId <- 1000*as.double(file$analysisId)+i
-  files <- rbind(files, file)
-  }
-  utils::write.csv(files, file.path(outputDirectory,'settings.csv'), row.names = F)
-  
-  for(i in 1:length(resultDirectory)){
-  # copy each analysis as a rds file and copy the log
-  files <- dir(resultDirectory[i], full.names = F)
-  files <- files[grep('Analysis', files)]
-  for(file in files){
-    
-    if(!dir.exists(file.path(outputDirectory,paste0('Analysis_',1000*as.double(gsub('Analysis_','',file))+i)))){
-      dir.create(file.path(outputDirectory,paste0('Analysis_',1000*as.double(gsub('Analysis_','',file))+i)))
-    }
-    
-    if(dir.exists(file.path(resultDirectory[i],file, 'plpResult'))){
-      res <- PatientLevelPrediction::loadPlpResult(file.path(resultDirectory[i],file, 'plpResult'))
-      res <- PatientLevelPrediction::transportPlp(res, n= minCellCount, 
-                                                  save = F, dataName = databaseName[i])
-      
-      res$covariateSummary <- res$covariateSummary[res$covariateSummary$covariateValue!=0,]
-      covSet <- res$model$metaData$call$covariateSettings
-      res$model$metaData <- NULL
-      res$model$metaData$call$covariateSettings <- covSet
-      res$model$predict <- NULL
-      if(!is.null(res$performanceEvaluation$evaluationStatistics)){
-      res$performanceEvaluation$evaluationStatistics[,1] <- paste0('Analysis_',1000*as.double(gsub('Analysis_','',file))+i)
-      } else{
-        writeLines(paste0(resultDirectory[i],file, '-ev'))
-      }
-      if(!is.null(res$performanceEvaluation$thresholdSummary)){
-      res$performanceEvaluation$thresholdSummary[,1] <- paste0('Analysis_',1000*as.double(gsub('Analysis_','',file))+i)
-      }else{
-        writeLines(paste0(resultDirectory[i],file, '-thres'))
-      }
-      if(!is.null(res$performanceEvaluation$demographicSummary)){
-      res$performanceEvaluation$demographicSummary[,1] <- paste0('Analysis_',1000*as.double(gsub('Analysis_','',file))+i)
-      } else{
-        writeLines(paste0(resultDirectory[i],file, '-dem'))
-      }
-      if(!is.null(res$performanceEvaluation$calibrationSummary)){
-      res$performanceEvaluation$calibrationSummary[,1] <- paste0('Analysis_',1000*as.double(gsub('Analysis_','',file))+i)
-      }else{
-        writeLines(paste0(resultDirectory[i],file, '-cal'))
-      }
-      if(!is.null(res$performanceEvaluation$predictionDistribution)){
-      res$performanceEvaluation$predictionDistribution[,1] <- paste0('Analysis_',1000*as.double(gsub('Analysis_','',file))+i)
-      }else{
-        writeLines(paste0(resultDirectory[i],file, '-dist'))
-      }
-      saveRDS(res, file.path(outputDirectory,paste0('Analysis_',1000*as.double(gsub('Analysis_','',file))+i), 'plpResult.rds'))
-    }
-    if(file.exists(file.path(resultDirectory[i],file, 'plpLog.txt'))){
-      file.copy(from = file.path(resultDirectory[i],file, 'plpLog.txt'), 
-                to = file.path(outputDirectory,paste0('Analysis_',1000*as.double(gsub('Analysis_','',file))+i), 'plpLog.txt'))
-    }
-  }
-  }
-  
-  
-  
-  for(i in 1:length(resultDirectory)){
-  # copy any validation results
-  if(dir.exists(file.path(resultDirectory[i],'Validation'))){
-    valFolders <-  dir(file.path(resultDirectory[i],'Validation'), full.names = F)
-    
-    if(length(valFolders)>0){
-      # move each of the validation rds
-      for(valFolder in valFolders){
-        
-        # get the analysisIds
-        valSubfolders <- dir(file.path(resultDirectory[i],'Validation',valFolder), full.names = F)
-        if(length(valSubfolders)!=0){
-          for(valSubfolder in valSubfolders ){
-            valSubfolderUpdate <- paste0('Analysis_', as.double(gsub('Analysis_','', valSubfolder))*1000+i)
-            valOut <- file.path(valFolder,valSubfolderUpdate)
-            valOutOld <- file.path(valFolder,valSubfolder)
-            if(!dir.exists(file.path(outputDirectory,'Validation',valOut))){
-              dir.create(file.path(outputDirectory,'Validation',valOut), recursive = T)
-            }
-            
-            
-            if(file.exists(file.path(resultDirectory[i],'Validation',valOutOld, 'validationResult.rds'))){
-              res <- readRDS(file.path(resultDirectory[i],'Validation',valOutOld, 'validationResult.rds'))
-              res <- PatientLevelPrediction::transportPlp(res, n= minCellCount, 
-                                                          save = F, dataName = databaseName[i])
-              res$covariateSummary <- res$covariateSummary[res$covariateSummary$covariateValue!=0,]
-              saveRDS(res, file.path(outputDirectory,'Validation',valOut, 'validationResult.rds'))
-            }
-          }
-        }
-        
-      }
-      
-    }
-    
-  }
-  }
-  
-  return(outputDirectory)
-  
-}
-```
-
-## Example code to combine multiple results
-The following code will combine the results found in 'C:/myResults', 'C:/myResults2' and 'C:/myResults3' into the shiny project at 'C:/R/library/myPackage/shiny/PLPViewer':
-```{r eval = FALSE}
-populateMultipleShinyApp(shinyDirectory = 'C:/R/library/myPackage/shiny/PLPViewer',
-                                     resultDirectory = c('C:/myResults',
-                                                         'C:/myResults2',
-                                                         'C:/myResults3'),
-                                     minCellCount = 0,
-                                     databaseName = c('database1','database2','database3'))
-
-```
-
-
-# Manual App Creation
-[instructions coming soon]
-
-
-
-# Acknowledgments
-
-Considerable work has been dedicated to provide the `PatientLevelPrediction` package.
-
-```{r tidy=TRUE,eval=TRUE}
-citation("PatientLevelPrediction")
-```
-
-**Please reference this paper if you use the PLP Package in your work:**
-
-[Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.](http://dx.doi.org/10.1093/jamia/ocy032)
diff --git a/vignettes/Figure1.png b/vignettes/Figure1.png
deleted file mode 100644
index 878a509e2..000000000
Binary files a/vignettes/Figure1.png and /dev/null differ
diff --git a/vignettes/Figure1.webp b/vignettes/Figure1.webp
new file mode 100644
index 000000000..42ad71d7f
Binary files /dev/null and b/vignettes/Figure1.webp differ
diff --git a/vignettes/InstallationGuide.Rmd b/vignettes/InstallationGuide.Rmd
index 2ce3e4972..91788b711 100644
--- a/vignettes/InstallationGuide.Rmd
+++ b/vignettes/InstallationGuide.Rmd
@@ -1,6 +1,6 @@
 ---
 title: "Patient-Level Prediction Installation Guide"
-author: "Jenna Reps, Peter R. Rijnbeek"
+author: "Jenna Reps, Peter R. Rijnbeek, Egill Fridgeirsson"
 date: '`r Sys.Date()`'
 header-includes:
     - \usepackage{fancyhdr}
@@ -23,54 +23,59 @@ output:
     number_sections: yes
     toc: yes
 ---
+
+```{=html}
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Installation guide}
 -->
-
+```
 # Introduction
-This vignette describes how you need to install the Observational Health Data Sciencs and Informatics (OHDSI) [`PatientLevelPrediction`](http://github.com/OHDSI/PatientLevelPrediction) package under Windows, Mac, and Linux.
+
+This vignette describes how you need to install the Observational Health Data Science and Informatics (OHDSI) [`PatientLevelPrediction`](http://github.com/OHDSI/PatientLevelPrediction) package under Windows, Mac, and Linux.
 
 # Software Prerequisites
 
 ## Windows Users
+
 Under Windows the OHDSI Patient Level Prediction (PLP) package requires installing:
 
-* R (https://cran.cnr.berkeley.edu/ ) \- (R >= 3.3.0, but latest is recommended)
-* Rstudio (https://www.rstudio.com/  )
-* Java (http://www.java.com )
-* RTools (https://cran.r-project.org/bin/windows/Rtools/) 
+-   R (<https://cran.cnr.berkeley.edu/> ) - (R \>= 4.0.0, but latest is recommended)
+-   Rstudio (<https://www.rstudio.com/> )
+-   Java (<http://www.java.com> )
+-   RTools (<https://cran.r-project.org/bin/windows/Rtools/>)
 
 ## Mac/Linux Users
-Under Mac and Linux the OHDSI Patient Level Prediction (PLP) package requires installing:
-
-* R (https://cran.cnr.berkeley.edu/ ) \- (R >= 3.3.0, but latest is recommended)
-* Rstudio (https://www.rstudio.com/  )
-* Java (http://www.java.com )
-* Xcode command line tools(run in terminal: xcode-select --install) [MAC USERS ONLY]
 
+Under Mac and Linux the OHDSI Patient Level Prediction (PLP) package requires installing:
 
+-   R (<https://cran.cnr.berkeley.edu/> ) - (R \>= 4.0.0, but latest is recommended)
+-   Rstudio (<https://www.rstudio.com/> )
+-   Java (<http://www.java.com> )
+-   Xcode command line tools(run in terminal: xcode-select --install) [MAC USERS ONLY]
 
 # Installing the Package
+
 The preferred way to install the package is by using `remotes`, which will automatically install the latest release and all the latest dependencies.
 
-If you do not want the official release you could install the bleading edge version of the package (latest develop branch).
+If you do not want the official release you could install the bleeding edge version of the package (latest develop branch).
 
 Note that the latest develop branch could contain bugs, please report them to us if you experience problems.
 
 ## Installing PatientLevelPrediction using remotes
+
 To install using `remotes` run:
+
 ```{r, echo = TRUE, message = FALSE, warning = FALSE,tidy=FALSE,eval=FALSE}
 install.packages("remotes")
-remotes::install_github("OHDSI/FeatureExtraction")
 remotes::install_github("OHDSI/PatientLevelPrediction")
 ```
 
-When installing make sure to close any other Rstudio sessions that are using `PatientLevelPrediction` or any dependency.  Keeping Rstudio sessions open can cause locks that prevent the package installing.
+When installing make sure to close any other Rstudio sessions that are using `PatientLevelPrediction` or any dependency. Keeping Rstudio sessions open can cause locks that prevent the package installing.
 
 # Creating Python Reticulate Environment
 
-Many of the classifiers in the `PatientLevelPrediction` use a Python back end.  To set up a python environment run:
+Many of the classifiers in the `PatientLevelPrediction` use a Python backend. To set up a python environment run:
 
 ```{r, echo = TRUE, message = FALSE, warning = FALSE,tidy=FALSE,eval=FALSE}
 library(PatientLevelPrediction)
@@ -79,60 +84,40 @@ configurePython(envname='r-reticulate', envtype='conda')
 
 ```
 
-Some of the less frequently used classifiers are not installed during this set-up to add them run:
+# Installation issues
 
-For GBM survival:
-```{r, echo = TRUE, message = FALSE, warning = FALSE,tidy=FALSE,eval=FALSE}
-reticulate::conda_install(envname='r-reticulate', packages = c('scikit-survival'), forge = TRUE, pip = FALSE, pip_ignore_installed = TRUE, conda = "auto", channel = 'sebp')
+Installation issues need to be posted in our issue tracker: <http://github.com/OHDSI/PatientLevelPrediction/issues>
 
-```
+The list below provides solutions for some common issues:
 
-# Testing installation
-To test whether the package is installed correctly, using the test script in '/extras', run:
-```{r, echo = TRUE, message = FALSE, warning = FALSE,tidy=FALSE,eval=FALSE}
+1.  If you have an error when trying to install a package in R saying **'Dependancy X not available ...'** then this can sometimes be fixed by running `install.packages('X')` and then once that completes trying to reinstall the package that had the error.
 
-# load the checkPlpInstallation function
-library(devtools)
-source_url('https://raw.github.com/OHDSI/PatientLevelPrediction/issue242/extras/checkPlpInstallation.R')
-
-# set up the database connection details
-library(DatabaseConnector)
-connectionDetails <- createConnectionDetails(
-  dbms = 'sql_server', 
-  user = 'username', 
-  password = 'hidden', 
-  server = 'your server', 
-  port = 'your port'
-  )
-
-# run the test
-checkPlpInstallation(
-  connectionDetails = connectionDetails, 
-  python = T
-  )
-```
+2.  I have found that using the github `remotes` to install packages can be impacted if you have **multiple R sessions** open as one session with a library open can cause the library to be locked and this can prevent an install of a package that depends on that library.
 
-To test the installation (excluding python) run:
-```{r, echo = TRUE, message = FALSE, warning = FALSE,tidy=FALSE,eval=FALSE}
+## Common issues
+
+### python environment Mac/linux users:
 
-checkPlpInstallation(
-  connectionDetails = connectionDetails, 
-  python = F
-  )
+to make sure R uses the r-reticulate python environment you may need to edit your .Rprofile with the location of the python binary for the PLP environment. Edit the .Rprofile by running:
+
+```{r eval=FALSE}
+usethis::edit_r_profile()
 ```
 
-The check can take a while to run since it will build the following models in sequence on simulated data: Logistic Regression, RandomForest, MLP, AdaBoost, Decision Tree, Naive Bayes, KNN, Gradient Boosting.
-Moreover, it will test the database connection. 
+and add
 
-# Installation issues
-Installation issues need to be posted in our issue tracker:
-http://github.com/OHDSI/PatientLevelPrediction/issues
+```{r eval=FALSE}
+Sys.setenv(PATH = paste("your python bin location", Sys.getenv("PATH"), sep=":"))
+```
 
-The list below provides solutions for some common issues:
+to the file then save. Where your python bin location is the location returned by
 
-1. If you have an error when trying to install a package in R saying **'Dependancy X not available ...'** then this can sometimes be fixed by running `install.packages('X')` and then once that completes trying to reinstall the package that had the error.
+```{r eval=FALSE}
+reticulate::conda_list() 
+```
 
-2. I have found that using the github `remotes`` to install packages can be impacted if you have **multiple R sessions** open as one session with a library open can cause the library to be locked and this can prevent an install of a package that depends on that library.
+e.g., My PLP virtual environment location was /anaconda3/envs/PLP/bin/python so I added:\
+Sys.setenv(PATH = paste("/anaconda3/envs/PLP/bin", Sys.getenv("PATH"), sep=":"))
 
 # Acknowledgments
 
@@ -146,5 +131,4 @@ citation("PatientLevelPrediction")
 
 [Reps JM, Schuemie MJ, Suchard MA, Ryan PB, Rijnbeek PR. Design and implementation of a standardized framework to generate and evaluate patient-level prediction models using observational healthcare data. J Am Med Inform Assoc. 2018;25(8):969-975.](http://dx.doi.org/10.1093/jamia/ocy032)
 
-
-This work is supported in part through the National Science Foundation grant IIS 1251151.
\ No newline at end of file
+This work is supported in part through the National Science Foundation grant IIS 1251151.
diff --git a/vignettes/PatientLevelPrediction.rmd b/vignettes/PatientLevelPrediction.rmd
deleted file mode 100644
index 3df52bbd6..000000000
--- a/vignettes/PatientLevelPrediction.rmd
+++ /dev/null
@@ -1,165 +0,0 @@
----
-title: "Quick Install Guide"
-author: "Jenna Reps, Martijn J. Schuemie, Patrick B. Ryan, Peter R. Rijnbeek"
-date: '`r Sys.Date()`'
-header-includes:
-    - \usepackage{fancyhdr}
-    - \pagestyle{fancy}
-    - \fancyhead{}
-    - \fancyhead[CO,CE]{Installation Guide}
-    - \fancyfoot[CO,CE]{PatientLevelPrediction Package Version `r    utils::packageVersion("PatientLevelPrediction")`}
-    - \fancyfoot[LE,RO]{\thepage}
-    - \renewcommand{\headrulewidth}{0.4pt}
-    - \renewcommand{\footrulewidth}{0.4pt}
-output:
-  pdf_document:
-    includes:
-      in_header: preamble.tex
-    number_sections: yes
-    toc: yes
-  word_document:
-    toc: yes
-  html_document:
-    number_sections: yes
-    toc: yes
----
-<!--
-%\VignetteEngine{knitr}
-%\VignetteIndexEntry{Quick Install Guide}
--->
-
-# Quick Install Guide
-***
-
-## Intalling the R package
-The preferred way to install the package is by using drat, which will automatically install the latest release and all the latest dependencies.
-If the drat code fails or you do not want the official release you could use devtools to install the bleading edge version of the package (latest master).
-Note that the latest master could contain bugs, please report them to us if you experience problems.
-
-To install using drat run:
-```{r, echo = TRUE, message = FALSE, warning = FALSE,tidy=FALSE,eval=FALSE}
-install.packages("drat")
-drat::addRepo("OHDSI")
-install.packages("PatientLevelPrediction")
-```
-
-To install using devtools run:
-```{r, echo = TRUE, message = FALSE, warning = FALSE,tidy=FALSE,eval=FALSE}
-install.packages('devtools')
-devtools::install_github("OHDSI/FeatureExtraction")
-devtools::install_github('ohdsi/PatientLevelPrediction')
-```
-
-When installing using devtools make sure to close any other Rstudio sessions that are using PatientLevelPrediction or any dependency.  Keeping Rstudio sessions open can cause locks that prevent the package installing.
-
-## Setting up Python 
-Many of the classifiers in PatientLevelPrediction use python.  To use the python classifiers you need to install and set up the a python environment in R.  We used the reticulate package:  
-
-```{r, echo = TRUE, message = FALSE, warning = FALSE,tidy=FALSE,eval=FALSE}
-library(PatientLevelPrediction)
-reticulate::install_miniconda()
-configurePython(envname='r-reticulate', envtype='conda')
-
-```
-
-To add the R keras interface, in Rstudio run:
-
-```{r, echo = TRUE, message = FALSE, warning = FALSE,tidy=FALSE,eval=FALSE}
-devtools::install_github("rstudio/keras")
-library(keras)
-install_keras()
-```
-
-Some of the less frequently used classifiers are considered optional and are not installed by default. To install then, run:
-
-For GBM survival:
-```{r, echo = TRUE, message = FALSE, warning = FALSE,tidy=FALSE,eval=FALSE}
-reticulate::conda_install(envname='r-reticulate', packages = c('scikit-survival'), forge = TRUE, pip = FALSE, pip_ignore_installed = TRUE, conda = "auto", channel = 'sebp')
-
-```
-
-For any of the torch models:
-```{r, echo = TRUE, message = FALSE, warning = FALSE,tidy=FALSE,eval=FALSE}
-reticulate::conda_install(envname='r-reticulate', packages = c('pytorch', 'torchvision', 'cpuonly'), forge = TRUE, pip = FALSE, channel = 'pytorch', pip_ignore_installed = TRUE, conda = 'auto')
-
-```
-
-
-## Testing the PatientLevelPrediction Installation
-To test whether the package is installed correctly run:
-```{r eval=FALSE}
-library(PatientLevelPrediction)
-library(DatabaseConnector)
-connectionDetails <- createConnectionDetails(dbms = 'sql_server', 
-                                             user = 'username', 
-                                             password = 'hidden', 
-                                             server = 'your server', 
-                                             port = 'your port')
-PatientLevelPrediction::checkPlpInstallation(connectionDetails = connectionDetails, 
-                                             python = T)
-```
-To test the installation (excluding python) run:
-
-```{r eval=FALSE}
-library(PatientLevelPrediction)
-library(DatabaseConnector)
-connectionDetails <- createConnectionDetails(dbms = 'sql_server', 
-                                           user = 'username', 
-                                           password = 'hidden', 
-                                           server = 'your server', 
-                                           port = 'your port')
-PatientLevelPrediction::checkPlpInstallation(connectionDetails = connectionDetails, 
-                                             python = F)
-```
-The check can take a while to run since it will build the following models in sequence on simulated data: Logistic Regression, RandomForest, MLP, AdaBoost, Decision Tree, Naive Bayes, KNN, Gradient Boosting. Moreover, it will test the database connection.
-
-
-# Common issues
-## python environment Mac/linux users: 
-to make sure R uses the r-reticulate python environment you may need to edit your .Rprofile with the location of the python binary for the PLP environment.    Edit the .Rprofile by running: 
-```{r eval=FALSE}
-usethis::edit_r_profile()
-``` 
-and add  
-```{r eval=FALSE} 
-Sys.setenv(PATH = paste("your python bin location", Sys.getenv("PATH"), sep=":"))
-``` 
-to the file then save.  Where your python bin location is the location returned by 
-```{r eval=FALSE} 
-reticulate::conda_list() 
-``` 
-e.g., My PLP virtual environment location was /anaconda3/envs/PLP/bin/python so I added:  
-Sys.setenv(PATH = paste("/anaconda3/envs/PLP/bin", Sys.getenv("PATH"), sep=":"))
-
-
-# Old Instructions
-
-## To configure python via anaconda
-* Close your RStudio
-* Install python 3.7 using anaconda (https://www.anaconda.com/download) [make sure you pick the correct operating system] and note the installation location. Anaconda should update you path variable with the python binary.
-* Open a new Rstudio and check whether your python is configured correctly by running:
-```{r eval=FALSE} 
-system("python --version") 
-```
-If set up correctly you should see "Python 3.x.x :: Anaconda, Inc." returned.
-
-* If not set up correctly then:
-    + Windows users: make sure your anaconda python binary is in the System PATH environmental variable: go to my computer -> system properties -> advanced system settings Then at the bottom right you’ll see a button: Environmental Variables, clicking on that will enable you to edit the PATH variable. Add the following Anaconda locations to your path: `D:\Anaconda3;D:\Anaconda3\Scripts;D:\Anaconda3\Library\bin` (this assumes your installation was done to `D:\Anaconda3`).
-    + Mac/Linux users: edit the bash profile to add python in the Path by running in the terminal: touch ~/.bash_profile; open ~/.bash_profile; and adding in the location of python in the PATH variable. Unfortunately, you also need to make an edit to the .Rprofile for R to get the correct PATH. To do this open the .Rprofile by running: 
-```{r eval=FALSE} 
-  usethis::edit_r_profile()
-``` 
-and in this file add 
-```{r eval=FALSE} 
-Sys.setenv(PATH = paste("your python bin location", Sys.getenv("PATH"), sep=":"))
-``` 
-* After editing your Path or .Rprofile open a new Rstudio session and test that python is correctly set up via 
-```{r eval=FALSE} 
-system("python --version")
-```
-
-
-
-
-
-
diff --git a/vignettes/Videos.rmd b/vignettes/Videos.rmd
index 410697d7f..5f278c92b 100644
--- a/vignettes/Videos.rmd
+++ b/vignettes/Videos.rmd
@@ -23,184 +23,45 @@ output:
     number_sections: yes
     toc: yes
 ---
+
+```{=html}
 <!--
-%\VignetteEngine{knitr}
+%\VignetteEngine{knitr::knitr}
 %\VignetteIndexEntry{Demo Videos}
 -->
-
-
+```
 ## What is a cohort table?
 
-<table>
-<tr>
-<th>
-Click To Launch
-</th>
-<th>
-Description of Demo
-</th>
-</tr>
-<tr>
-<td width="40%">
-<a href="https://youtu.be/GY2ZTcizY90" target="_blank"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" 
-alt="Video Vignette PLP Package" width="240" height="180" border="10" /></a>
-</td>
-<td width="60%">
-Learn what a cohort table looks like and what columns are required.
-</td>
-</tr>
-</table>
+| Click To Launch                                                                                                                              | Description of Demo                                                 |
+|-----------------------------------------------|------------------------|
+| [![Video Vignette PLP Package](http://img.youtube.com/vi/BEukCbT8UoA/0.jpg){alt="Video Vignette PLP Package"}](https://youtu.be/GY2ZTcizY90) | Learn what a cohort table looks like and what columns are required. |
 
 ## Setting up a connection between your database and R
 
-<table>
-<tr>
-<th>
-Click To Launch
-</th>
-<th>
-Description of Demo
-</th>
-</tr>
-<tr>
-<td width="40%">
-<a href="https://youtu.be/8F2X5SKN64w" target="_blank"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" 
-alt="Video Vignette PLP Package" width="240" height="180" border="10" /></a>
-</td>
-<td width="60%">
-Learn how to configure the connection to your OMOP CDM data from R using the OHDSI DatabaseConnector package.
-</td>
-</tr>
-</table>
+| Click To Launch                                                                                                                              | Description of Demo                                                                                           |
+|----------------------------------------|--------------------------------|
+| [![Video Vignette PLP Package](http://img.youtube.com/vi/BEukCbT8UoA/0.jpg){alt="Video Vignette PLP Package"}](https://youtu.be/8F2X5SKN64w) | Learn how to configure the connection to your OMOP CDM data from R using the OHDSI DatabaseConnector package. |
 
-
-  
 ## Running a single PatientLevelPrediction model
-<table>
-<tr>
-<th>
-Click To Launch
-</th>
-<th>
-Description of Demo
-</th>
-</tr>
-<tr>
-<td width="40%">
-<a href="https://youtu.be/W9uf1IIhDQs" target="_blank"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" 
-alt="Video Vignette PLP Package" width="240" height="180" border="10" /></a>
-</td>
-<td width="60%">
-Learn how to develop and validate a single PatientLevelPrediction model.
-</td>
-</tr>
-</table>
-  
-  
+
+| Click To Launch                                                                                                                              | Description of Demo                                                      |
+|-----------------------------------------------|-------------------------|
+| [![Video Vignette PLP Package](http://img.youtube.com/vi/7AraOsTynD4/0.jpg){alt="Video Vignette PLP Package"}](https://youtu.be/7AraOsTynD4) | Learn how to develop and validate a single PatientLevelPrediction model. |
+
 ## Running multiple PatientLevelPrediction models study
-<table>
-<tr>
-<th>
-Click To Launch
-</th>
-<th>
-Description of Demo
-</th>
-</tr>
-<tr>
-<td width="40%">
-<a href="https://youtu.be/Ppsn_08z5FE" target="_blank"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" 
-alt="Video Vignette PLP Package" width="240" height="180" border="10" /></a>
-</td>
-<td width="60%">
-Learn how to develop and validate multiple PatientLevelPrediction models.
-</td>
-</tr>
-</table>
 
-  
-  
-## Designing a study in Atlas
-<table>
-<tr>
-<th>
-Click To Launch
-</th>
-<th>
-Description of Demo
-</th>
-</tr>
-<tr>
-<td width="40%">
-<a href="https://youtu.be/HqUdO3zx3DI" target="_blank"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" 
-alt="Video Vignette PLP Package" width="240" height="180" border="10" /></a>
-</td>
-<td width="60%">
-Learn how to design a multiple or single PatientLevelPrediction study using Atlas.  Atlas creates an R package that just needs to be built and then you're on your way to developing multiple models!
-</td>
-</tr>
-</table>
-  
-## Building and running the Atlas study 
-<table>
-<tr>
-<th>
-Click To Launch
-</th>
-<th>
-Description of Demo
-</th>
-</tr>
-<tr>
-<td width="40%">
-<a href="https://youtu.be/qESMHZnpHJQ" target="_blank"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" 
-alt="Video Vignette PLP Package" width="240" height="180" border="10" /></a>
-</td>
-<td width="60%">
-Learn how to build the R package generated by Atlas and how to then run the study.
-</td>
-</tr>
-</table>  
+| Click To Launch                                                                                                                              | Description of Demo                                                       |
+|-----------------------------------------------|-------------------------|
+| [![Video Vignette PLP Package](http://img.youtube.com/vi/7wUilx580PE/0.jpg){alt="Video Vignette PLP Package"}](https://youtu.be/7wUilx580PE) | Learn how to develop and validate multiple PatientLevelPrediction models. |
 
 ## Exploring the results in the shiny app
-<table>
-<tr>
-<th>
-Click To Launch
-</th>
-<th>
-Description of Demo
-</th>
-</tr>
-<tr>
-<td width="40%">
-<a href="https://youtu.be/StpV40yl1UE" target="_blank"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" 
-alt="Video Vignette PLP Package" width="240" height="180" border="10" /></a>
-</td>
-<td width="60%">
-Learn how to interactively explore the model performance and model via the shiny apps viewPlp() and viewMultiplePlp()
-</td>
-</tr>
-</table>  
-  
+
+| Click To Launch                                                                                                                              | Description of Demo                                                                                                   |
+|---------------------------------------|---------------------------------|
+| [![Video Vignette PLP Package](http://img.youtube.com/vi/BulmuH32y_Y/0.jpg){alt="Video Vignette PLP Package"}](https://youtu.be/BulmuH32y_Y) | Learn how to interactively explore the model performance and model via the shiny apps viewPlp() and viewMultiplePlp() |
+
 ## Validating existing models on OMOP CDM data
-<table>
-<tr>
-<th>
-Click To Launch
-</th>
-<th>
-Description of Demo
-</th>
-</tr>
-<tr>
-<td width="40%">
-<a href="https://youtu.be/oBsfg9hfrpI" target="_blank"><img src="http://img.youtube.com/vi/BEukCbT8UoA/0.jpg" 
-alt="Video Vignette PLP Package" width="240" height="180" border="10" /></a>
-</td>
-<td width="60%">
-This demo shows how you can add any existing score or logistic model and valdiate the model on new OMOP CDM data.  This is useful for benchmarking when developing new models or to perform extensive external validation of a model across the OHDSI network.
-</td>
-</tr>
-</table>  
-    
\ No newline at end of file
+
+| Click To Launch                                                                                                                              | Description of Demo                                                                                                                                                                                                                                           |
+|--------------------------|----------------------------------------------|
+| [![Video Vignette PLP Package](http://img.youtube.com/vi/BEukCbT8UoA/0.jpg){alt="Video Vignette PLP Package"}](https://youtu.be/oBsfg9hfrpI) | This demo shows how you can add any existing score or logistic model and validate the model on new OMOP CDM data. This is useful for benchmarking when developing new models or to perform extensive external validation of a model across the OHDSI network. |
diff --git a/vignettes/arch1.png b/vignettes/arch1.png
deleted file mode 100644
index e4846e56f..000000000
Binary files a/vignettes/arch1.png and /dev/null differ
diff --git a/vignettes/atlasdownload1.png b/vignettes/atlasdownload1.png
deleted file mode 100644
index ef6559fa9..000000000
Binary files a/vignettes/atlasdownload1.png and /dev/null differ
diff --git a/vignettes/atlasdownload1.webp b/vignettes/atlasdownload1.webp
new file mode 100644
index 000000000..6cac340ed
Binary files /dev/null and b/vignettes/atlasdownload1.webp differ
diff --git a/vignettes/atlasdownload2.png b/vignettes/atlasdownload2.png
deleted file mode 100644
index 619f8c799..000000000
Binary files a/vignettes/atlasdownload2.png and /dev/null differ
diff --git a/vignettes/atlasdownload2.webp b/vignettes/atlasdownload2.webp
new file mode 100644
index 000000000..452c5ca21
Binary files /dev/null and b/vignettes/atlasdownload2.webp differ
diff --git a/vignettes/atlasplp1.png b/vignettes/atlasplp1.png
deleted file mode 100644
index 4b21b2143..000000000
Binary files a/vignettes/atlasplp1.png and /dev/null differ
diff --git a/vignettes/atlasplp1.webp b/vignettes/atlasplp1.webp
new file mode 100644
index 000000000..71a3c1ce9
Binary files /dev/null and b/vignettes/atlasplp1.webp differ
diff --git a/vignettes/atlasplp2.png b/vignettes/atlasplp2.png
deleted file mode 100644
index 6bc7b93ad..000000000
Binary files a/vignettes/atlasplp2.png and /dev/null differ
diff --git a/vignettes/atlasplp2.webp b/vignettes/atlasplp2.webp
new file mode 100644
index 000000000..668202de4
Binary files /dev/null and b/vignettes/atlasplp2.webp differ
diff --git a/vignettes/atlasplp3.png b/vignettes/atlasplp3.png
deleted file mode 100644
index 0911b31ea..000000000
Binary files a/vignettes/atlasplp3.png and /dev/null differ
diff --git a/vignettes/atlasplp3.webp b/vignettes/atlasplp3.webp
new file mode 100644
index 000000000..523d0143c
Binary files /dev/null and b/vignettes/atlasplp3.webp differ
diff --git a/vignettes/atlasplp4.png b/vignettes/atlasplp4.png
deleted file mode 100644
index b5db1b153..000000000
Binary files a/vignettes/atlasplp4.png and /dev/null differ
diff --git a/vignettes/atlasplp4.webp b/vignettes/atlasplp4.webp
new file mode 100644
index 000000000..d90a43ea2
Binary files /dev/null and b/vignettes/atlasplp4.webp differ
diff --git a/vignettes/atlasplp5.png b/vignettes/atlasplp5.png
deleted file mode 100644
index df1c2eb15..000000000
Binary files a/vignettes/atlasplp5.png and /dev/null differ
diff --git a/vignettes/cirenn.png b/vignettes/cirenn.png
deleted file mode 100644
index f4e8ed054..000000000
Binary files a/vignettes/cirenn.png and /dev/null differ
diff --git a/vignettes/cnn_lstm.png b/vignettes/cnn_lstm.png
deleted file mode 100644
index a16e1417d..000000000
Binary files a/vignettes/cnn_lstm.png and /dev/null differ
diff --git a/vignettes/cnn_mlf2.png b/vignettes/cnn_mlf2.png
deleted file mode 100644
index 2b69c159b..000000000
Binary files a/vignettes/cnn_mlf2.png and /dev/null differ
diff --git a/vignettes/conv_arch1.png b/vignettes/conv_arch1.png
deleted file mode 100644
index 5970b3f1c..000000000
Binary files a/vignettes/conv_arch1.png and /dev/null differ
diff --git a/vignettes/conv_arch2.png b/vignettes/conv_arch2.png
deleted file mode 100644
index a51ccf08e..000000000
Binary files a/vignettes/conv_arch2.png and /dev/null differ
diff --git a/vignettes/covcnn2.png b/vignettes/covcnn2.png
deleted file mode 100644
index 0734a49eb..000000000
Binary files a/vignettes/covcnn2.png and /dev/null differ
diff --git a/vignettes/demographicSummary.png b/vignettes/demographicSummary.png
deleted file mode 100644
index 8ceafbee8..000000000
Binary files a/vignettes/demographicSummary.png and /dev/null differ
diff --git a/vignettes/demographicSummary.webp b/vignettes/demographicSummary.webp
new file mode 100644
index 000000000..7d0437deb
Binary files /dev/null and b/vignettes/demographicSummary.webp differ
diff --git a/vignettes/example1/ATLAS_O.PNG b/vignettes/example1/ATLAS_O.PNG
deleted file mode 100644
index 3cda2abf7..000000000
Binary files a/vignettes/example1/ATLAS_O.PNG and /dev/null differ
diff --git a/vignettes/example1/ATLAS_O.webp b/vignettes/example1/ATLAS_O.webp
new file mode 100644
index 000000000..85e63dc9e
Binary files /dev/null and b/vignettes/example1/ATLAS_O.webp differ
diff --git a/vignettes/example1/ATLAS_T.PNG b/vignettes/example1/ATLAS_T.PNG
deleted file mode 100644
index 8be57dc9e..000000000
Binary files a/vignettes/example1/ATLAS_T.PNG and /dev/null differ
diff --git a/vignettes/example1/ATLAS_T.webp b/vignettes/example1/ATLAS_T.webp
new file mode 100644
index 000000000..df3a8245f
Binary files /dev/null and b/vignettes/example1/ATLAS_T.webp differ
diff --git a/vignettes/example2/aceinhibitors.png b/vignettes/example2/aceinhibitors.png
deleted file mode 100644
index ce5148f1d..000000000
Binary files a/vignettes/example2/aceinhibitors.png and /dev/null differ
diff --git a/vignettes/example2/aceinhibitors.webp b/vignettes/example2/aceinhibitors.webp
new file mode 100644
index 000000000..564f8af77
Binary files /dev/null and b/vignettes/example2/aceinhibitors.webp differ
diff --git a/vignettes/example2/angioedema.png b/vignettes/example2/angioedema.png
deleted file mode 100644
index 3adc8dcc9..000000000
Binary files a/vignettes/example2/angioedema.png and /dev/null differ
diff --git a/vignettes/example2/angioedema.webp b/vignettes/example2/angioedema.webp
new file mode 100644
index 000000000..8c728ce50
Binary files /dev/null and b/vignettes/example2/angioedema.webp differ
diff --git a/vignettes/generalizability.png b/vignettes/generalizability.png
deleted file mode 100644
index b476ea71f..000000000
Binary files a/vignettes/generalizability.png and /dev/null differ
diff --git a/vignettes/generalizability.webp b/vignettes/generalizability.webp
new file mode 100644
index 000000000..ba6d14de4
Binary files /dev/null and b/vignettes/generalizability.webp differ
diff --git a/vignettes/lstm_last.png b/vignettes/lstm_last.png
deleted file mode 100644
index 3e6fc16e5..000000000
Binary files a/vignettes/lstm_last.png and /dev/null differ
diff --git a/vignettes/popdef1.png b/vignettes/popdef1.png
deleted file mode 100644
index 3d654fe7d..000000000
Binary files a/vignettes/popdef1.png and /dev/null differ
diff --git a/vignettes/popdef1.webp b/vignettes/popdef1.webp
new file mode 100644
index 000000000..83ef7afd6
Binary files /dev/null and b/vignettes/popdef1.webp differ
diff --git a/vignettes/popdef2.png b/vignettes/popdef2.png
deleted file mode 100644
index a596e188d..000000000
Binary files a/vignettes/popdef2.png and /dev/null differ
diff --git a/vignettes/popdef2.webp b/vignettes/popdef2.webp
new file mode 100644
index 000000000..31887dd1b
Binary files /dev/null and b/vignettes/popdef2.webp differ
diff --git a/vignettes/popdef3.png b/vignettes/popdef3.png
deleted file mode 100644
index 34527ef9f..000000000
Binary files a/vignettes/popdef3.png and /dev/null differ
diff --git a/vignettes/popdef3.webp b/vignettes/popdef3.webp
new file mode 100644
index 000000000..8b409ed49
Binary files /dev/null and b/vignettes/popdef3.webp differ
diff --git a/vignettes/popdef4.png b/vignettes/popdef4.png
deleted file mode 100644
index 35d4949a5..000000000
Binary files a/vignettes/popdef4.png and /dev/null differ
diff --git a/vignettes/popdef4.webp b/vignettes/popdef4.webp
new file mode 100644
index 000000000..2709497e7
Binary files /dev/null and b/vignettes/popdef4.webp differ
diff --git a/vignettes/popdef5.png b/vignettes/popdef5.png
deleted file mode 100644
index f6315b8a8..000000000
Binary files a/vignettes/popdef5.png and /dev/null differ
diff --git a/vignettes/popdef5.webp b/vignettes/popdef5.webp
new file mode 100644
index 000000000..748b8901b
Binary files /dev/null and b/vignettes/popdef5.webp differ
diff --git a/vignettes/popdef6.png b/vignettes/popdef6.png
deleted file mode 100644
index 96a8abd1f..000000000
Binary files a/vignettes/popdef6.png and /dev/null differ
diff --git a/vignettes/popdef6.webp b/vignettes/popdef6.webp
new file mode 100644
index 000000000..583dc9fba
Binary files /dev/null and b/vignettes/popdef6.webp differ
diff --git a/vignettes/precisionRecall.png b/vignettes/precisionRecall.png
deleted file mode 100644
index 1f1d0f154..000000000
Binary files a/vignettes/precisionRecall.png and /dev/null differ
diff --git a/vignettes/precisionRecall.webp b/vignettes/precisionRecall.webp
new file mode 100644
index 000000000..af6b0cfe5
Binary files /dev/null and b/vignettes/precisionRecall.webp differ
diff --git a/vignettes/predictionDistribution.png b/vignettes/predictionDistribution.png
deleted file mode 100644
index 87bafc361..000000000
Binary files a/vignettes/predictionDistribution.png and /dev/null differ
diff --git a/vignettes/predictionDistribution.webp b/vignettes/predictionDistribution.webp
new file mode 100644
index 000000000..c7756d788
Binary files /dev/null and b/vignettes/predictionDistribution.webp differ
diff --git a/vignettes/preferencePDF.png b/vignettes/preferencePDF.png
deleted file mode 100644
index 3b3528452..000000000
Binary files a/vignettes/preferencePDF.png and /dev/null differ
diff --git a/vignettes/preferencePDF.webp b/vignettes/preferencePDF.webp
new file mode 100644
index 000000000..189a356be
Binary files /dev/null and b/vignettes/preferencePDF.webp differ
diff --git a/vignettes/problems.png b/vignettes/problems.png
deleted file mode 100644
index 931efa6d6..000000000
Binary files a/vignettes/problems.png and /dev/null differ
diff --git a/vignettes/problems.webp b/vignettes/problems.webp
new file mode 100644
index 000000000..5c1c27bb4
Binary files /dev/null and b/vignettes/problems.webp differ
diff --git a/vignettes/shinyroc.PNG b/vignettes/shinyroc.PNG
deleted file mode 100644
index 579fab31f..000000000
Binary files a/vignettes/shinyroc.PNG and /dev/null differ
diff --git a/vignettes/shinyroc.webp b/vignettes/shinyroc.webp
new file mode 100644
index 000000000..a11724623
Binary files /dev/null and b/vignettes/shinyroc.webp differ
diff --git a/vignettes/shinysummary.PNG b/vignettes/shinysummary.PNG
deleted file mode 100644
index 75cec2430..000000000
Binary files a/vignettes/shinysummary.PNG and /dev/null differ
diff --git a/vignettes/shinysummary.webp b/vignettes/shinysummary.webp
new file mode 100644
index 000000000..0d256ade1
Binary files /dev/null and b/vignettes/shinysummary.webp differ
diff --git a/vignettes/sparseCalibration.png b/vignettes/sparseCalibration.png
deleted file mode 100644
index d6e34c0cf..000000000
Binary files a/vignettes/sparseCalibration.png and /dev/null differ
diff --git a/vignettes/sparseCalibration.webp b/vignettes/sparseCalibration.webp
new file mode 100644
index 000000000..043019e5b
Binary files /dev/null and b/vignettes/sparseCalibration.webp differ
diff --git a/vignettes/sparseROC.png b/vignettes/sparseROC.png
deleted file mode 100644
index 8a4b13cec..000000000
Binary files a/vignettes/sparseROC.png and /dev/null differ
diff --git a/vignettes/sparseROC.webp b/vignettes/sparseROC.webp
new file mode 100644
index 000000000..2ea3ea56f
Binary files /dev/null and b/vignettes/sparseROC.webp differ
diff --git a/vignettes/studydesign.png b/vignettes/studydesign.png
deleted file mode 100644
index 453f4aadd..000000000
Binary files a/vignettes/studydesign.png and /dev/null differ
diff --git a/vignettes/studydesign.webp b/vignettes/studydesign.webp
new file mode 100644
index 000000000..28717c7d2
Binary files /dev/null and b/vignettes/studydesign.webp differ
diff --git a/vignettes/variableScatterplot.png b/vignettes/variableScatterplot.png
deleted file mode 100644
index bdcf0df4a..000000000
Binary files a/vignettes/variableScatterplot.png and /dev/null differ
diff --git a/vignettes/variableScatterplot.webp b/vignettes/variableScatterplot.webp
new file mode 100644
index 000000000..de6f8999d
Binary files /dev/null and b/vignettes/variableScatterplot.webp differ