Skip to content

Commit

Permalink
Use httptest package, remove .dry_run arguments
Browse files Browse the repository at this point in the history
  • Loading branch information
wilkox committed Mar 19, 2024
1 parent df91c9f commit 1406986
Show file tree
Hide file tree
Showing 20 changed files with 305 additions and 56 deletions.
3 changes: 2 additions & 1 deletion DESCRIPTION
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
Package: GPTscreenR
Title: GPT-assisted source screening for scoping reviews
Version: 0.0.3.9002
Version: 0.0.3.9003
Authors@R:
person("David", "Wilkins", , "[email protected]", role = c("aut", "cre"))
Description: GPT-assisted source screening for scoping reviews.
Expand All @@ -11,6 +11,7 @@ RoxygenNote: 7.3.1
URL: https://github.com/wilkox/GPTscreenR
BugReports: https://github.com/wilkox/GPTscreenR/issues
Suggests:
httptest,
knitr,
rmarkdown,
testthat (>= 3.0.0)
Expand Down
2 changes: 2 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@

- Users can now set the model to use with the `OPENAI_MODEL` environmental
variable
- Now using the httptest package for testing, all `.dry_run` arguments have
been removed

# GPTscreenR 0.0.3

Expand Down
36 changes: 11 additions & 25 deletions R/screen.R
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,9 @@
#' @param abstract The abstract of the article to be screened, a character
#' vector of length 1
#' @param .verbose If FALSE, progress messages will be suppressed
#' @param .dry_run If TRUE, calls to the GPT API will be skipped
#'
#' @export
screen_source <- function(review_description, title, abstract, .verbose = TRUE, .dry_run = FALSE) {
screen_source <- function(review_description, title, abstract, .verbose = TRUE) {

# Validate arguments
validate_screening_param <- function(param, name) {
Expand Down Expand Up @@ -49,10 +48,7 @@ screen_source <- function(review_description, title, abstract, .verbose = TRUE,
role = "system",
content = "You must work step by step. FIRST, generate a numbered list of criteria that must be met for a source to be included."
)
if (! .dry_run) conversation <- complete_GPT_tryCatch(
conversation,
.dry_run = .dry_run
)
conversation <- complete_GPT_tryCatch(conversation)

# Instruct GPT to compare source against criteria
if (.verbose) { cli::cli_progress_step("GPT comparing source against criteria") }
Expand All @@ -61,10 +57,7 @@ screen_source <- function(review_description, title, abstract, .verbose = TRUE,
role = "system",
content = "NEXT, for each numbered criterion, decide whether the criterion is TRUE or FALSE for the source. It is normal for the title and abstract to not have enough information to make a clear decision for every statement. For these situations, give your best guess. After giving your response of TRUE or FALSE, give a one sentence explanation for your response."
)
if (! .dry_run) conversation <- complete_GPT_tryCatch(
conversation,
.dry_run = .dry_run
)
conversation <- complete_GPT_tryCatch(conversation)

# Instruct GPT to give its final response
if (.verbose) { cli::cli_progress_step("GPT giving its final recommendation") }
Expand All @@ -73,15 +66,10 @@ screen_source <- function(review_description, title, abstract, .verbose = TRUE,
role = "system",
content = "FINALLY, consider your decisions on whether the source meets the conclusion criteria. Respond with a single word, either INCLUDE or EXCLUDE, representing your recommendation on whether the source meets the inclusion criteria. Do not write anything other than INCLUDE or EXCLUDE."
)
if (! .dry_run) conversation <- complete_GPT_tryCatch(
conversation,
.dry_run = .dry_run
)
conversation <- complete_GPT_tryCatch(conversation)

# Check recommendation is in required format
if (.dry_run) {
GPT_includes <- NA
} else if (! stringr::str_detect(last_message(conversation),
if (! stringr::str_detect(last_message(conversation),
"(INCLUDE|EXCLUDE)")) {
if (.verbose) cli::cli_alert_warning(c("GPT's recommendation could not be parsed"))
GPT_includes <- NA
Expand Down Expand Up @@ -146,10 +134,9 @@ review_description <- function(objective = NULL, population = NULL, concept = NU
#' @param cache_file A file in which the sources list cache is kept (as RDS),
#' either fs::path() or character vector of length 1
#' @param .verbose If FALSE, progress messages will be suppressed
#' @param .dry_run If TRUE, calls to the GPT API will be skipped
#'
#' @export
screen_sources <- function(sources, review_description, n = NULL, random = TRUE, cache_file = fs::path("sources_cache.rds"), .verbose = TRUE, .dry_run = FALSE) {
screen_sources <- function(sources, review_description, n = NULL, random = TRUE, cache_file = fs::path("sources_cache.rds"), .verbose = TRUE) {

if (.verbose) cli::cli_h1("Preparing to screen sources")

Expand Down Expand Up @@ -194,7 +181,7 @@ screen_sources <- function(sources, review_description, n = NULL, random = TRUE,

# If cache file already exists, load and use it
cache_file <- fs::path(cache_file)
if (fs::file_exists(cache_file) & ! .dry_run) {
if (fs::file_exists(cache_file)) {
if (.verbose) cli::cli_alert_info("Loading cached results from {cache_file}")
cached_sources <- readRDS(cache_file)
sources <- dplyr::left_join(sources, cached_sources)
Expand All @@ -218,7 +205,7 @@ screen_sources <- function(sources, review_description, n = NULL, random = TRUE,
}

if (.verbose) cli::cli_alert_info("Creating cache file {cache_file}")
if (! .dry_run) saveRDS(sources, cache_file)
saveRDS(sources, cache_file)

}

Expand Down Expand Up @@ -249,21 +236,20 @@ screen_sources <- function(sources, review_description, n = NULL, random = TRUE,
# Screen the source and parse the response
response <- screen_source(review_description,
title = sources$title[next_i], abstract = sources$abstract[next_i],
.verbose = .verbose, .dry_run = .dry_run)
if (is.na(response$GPT_includes) & ! .dry_run & .verbose) {
.verbose = .verbose)
if (is.na(response$GPT_includes) & .verbose) {
cli::cli_alert_warning(c("GPT's recommendation could not be parsed. Discarding this response."))
next
}
sources$conversation[[next_i]] <- response$conversation
if (.dry_run) response$GPT_includes <- TRUE
sources$GPT_includes[next_i] <- response$GPT_includes
if (.verbose) cli::cli_alert_info(stringr::str_c(
"GPT recommends ",
ifelse(response$GPT_includes, "inclusion", "exclusion")
))

# Sync the sources list to the cache file
if (! .dry_run) saveRDS(sources, cache_file)
saveRDS(sources, cache_file)
}

# Report on current state
Expand Down
10 changes: 1 addition & 9 deletions man/screen_source.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 1 addition & 4 deletions man/screen_sources.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

23 changes: 23 additions & 0 deletions tests/testthat/api.openai.com/v1/chat/completions-1d7d6c-POST.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"id": "chatcmpl-94KDWKpIsk9bftjZN7dpypNY1MtBv",
"object": "chat.completion",
"created": 1710818054,
"model": "gpt-4-0613",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "1. TRUE: The study involves nursing home residents, who by definition are likely to be aged 65 or older. The abstract does not specify the age of participants, but usually, nursing homes are inhabited by elderly individuals.\n2. TRUE: The study's population are residents of nursing homes which fit the definition of residential aged care facility provided in the criteria.\n3. TRUE: Alpaca therapy is the central focus of the study conducted in these nursing homes and is evaluated in comparison to a control condition (television viewing lounge).\n4. TRUE: The study evaluates the impact of therapy alpaca program on depression, anxiety, and stress which are aspects of mental health and emotional well-being.\n5. TRUE: This study is a primary research project based on a randomized control trial intended to evaluate the role of therapy alpacas in enhancing mental health and emotional well-being in nursing home residents."
},
"logprobs": null,
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 813,
"completion_tokens": 180,
"total_tokens": 993
},
"system_fingerprint": null
}
23 changes: 23 additions & 0 deletions tests/testthat/api.openai.com/v1/chat/completions-3a0bf8-POST.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"id": "chatcmpl-94KDdjGJG4lDyNeXDWI8WShICmx7n",
"object": "chat.completion",
"created": 1710818061,
"model": "gpt-4-0613",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "INCLUDE"
},
"logprobs": null,
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 1051,
"completion_tokens": 1,
"total_tokens": 1052
},
"system_fingerprint": null
}
23 changes: 23 additions & 0 deletions tests/testthat/api.openai.com/v1/chat/completions-514df9-POST.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"id": "chatcmpl-94KEmrrYlksj7lcvjxzNZNxjy9Pjl",
"object": "chat.completion",
"created": 1710818132,
"model": "gpt-4-0613",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "1. TRUE - The study focuses on nursing home residents who are presumably over the age of 65.\n2. TRUE - The study population lives in nursing homes, which are a type of residential aged care facility that provides care and supports daily living activities, as well as nursing services.\n3. TRUE - It is implied by \"nursing home\" that the care delivered is long-term and based on support for daily living and nursing services. The abstract doesn't directly state that the setting is not a hospital, hospice, or independent living village, but these are usually distinct from nursing homes.\n4. TRUE - The study clearly states that therapy alpacas were used as an intervention in the study.\n5. TRUE - The abstract clearly states that the impact on depression, anxiety, and stress (which are aspects of mental health and emotional well-being) was evaluated."
},
"logprobs": null,
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 796,
"completion_tokens": 175,
"total_tokens": 971
},
"system_fingerprint": null
}
23 changes: 23 additions & 0 deletions tests/testthat/api.openai.com/v1/chat/completions-6bebc8-POST.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"id": "chatcmpl-94KDiAiHIlHBb8HWuTGwCv1hgr14g",
"object": "chat.completion",
"created": 1710818066,
"model": "gpt-4-0613",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "1. True - Although the abstract does not explicitly state the age of the nursing home residents, we can reasonably infer that the elderly living in such facilities are predominantly 65 years and older.\n \n2. True - The research is conducted within nursing homes which, per the study description, could be classified as residential aged care facilities as they provide support for activities of daily living and nursing support.\n\n3. True - The abstract clearly mentions the utilization of therapy alpacas in the residential programme.\n \n4. True - Outcomes observed in this study involve reduction in depression, anxiety, and stress, which are aspects of mental health and emotional well-being as required by the criteria.\n \n5. True - This publication refers to a randomized control trial, which classifies it as a primary research source."
},
"logprobs": null,
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 804,
"completion_tokens": 159,
"total_tokens": 963
},
"system_fingerprint": null
}
23 changes: 23 additions & 0 deletions tests/testthat/api.openai.com/v1/chat/completions-6d6d70-POST.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"id": "chatcmpl-94KDxxfEmaBdFXWbxT8wcoJWeGGkh",
"object": "chat.completion",
"created": 1710818081,
"model": "gpt-4-0613",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "1. TRUE - The source focuses on using therapy alpacas as a non-pharmacological intervention to manage depression, anxiety, and stress.\n2. TRUE - The population targeted for this study are nursing home residents.\n3. TRUE - The source presents original research in the form of a randomized control trial.\n4. TRUE - The study design reported in the source is a randomized control trial.\n5. UNKNOWN - The abstract does not provide information on where the source was published.\n6. TRUE - The source discusses its findings and provides substantial data for the same.\n7. TRUE - The intervention of therapy alpacas appears to be feasible in a nursing home environment, as suggested by the successful implementation in the study.\n8. TRUE - The source provides a clear account of the effectiveness of therapy alpacas in reducing depression, anxiety, and stress.\n9. TRUE - The source includes quantitative data on the reduction of depression, anxiety, and stress as measured by the mean DASS-21 scores.\n10. TRUE - The source is written in English."
},
"logprobs": null,
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 644,
"completion_tokens": 212,
"total_tokens": 856
},
"system_fingerprint": null
}
23 changes: 23 additions & 0 deletions tests/testthat/api.openai.com/v1/chat/completions-c593d1-POST.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"id": "chatcmpl-94KEglysgX08fJxdjQcK8B02gEPG1",
"object": "chat.completion",
"created": 1710818126,
"model": "gpt-4-0613",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "INCLUDE"
},
"logprobs": null,
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 994,
"completion_tokens": 1,
"total_tokens": 995
},
"system_fingerprint": null
}
23 changes: 23 additions & 0 deletions tests/testthat/api.openai.com/v1/chat/completions-ce1e77-POST.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"id": "chatcmpl-94KDn7YOcBAV2efEf2Muk2el7iTaE",
"object": "chat.completion",
"created": 1710818071,
"model": "gpt-4-0613",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "INCLUDE"
},
"logprobs": null,
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 1021,
"completion_tokens": 1,
"total_tokens": 1022
},
"system_fingerprint": null
}
23 changes: 23 additions & 0 deletions tests/testthat/api.openai.com/v1/chat/completions-cfc308-POST.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"id": "chatcmpl-94KE4MrXRhZVPz67xVRayXBr2cK1t",
"object": "chat.completion",
"created": 1710818088,
"model": "gpt-4-0613",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "INCLUDE"
},
"logprobs": null,
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 914,
"completion_tokens": 1,
"total_tokens": 915
},
"system_fingerprint": null
}
23 changes: 23 additions & 0 deletions tests/testthat/api.openai.com/v1/chat/completions-dd5499-POST.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"id": "chatcmpl-94KEYOaSri7y3wzXVWwrPY6lsGMe8",
"object": "chat.completion",
"created": 1710818118,
"model": "gpt-4-0613",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "1. TRUE - The abstract describes a randomized control trial which qualifies as primary research.\n2. TRUE - Though the term \"elderly\" is not explicitly used, it is mentioned that the participants are nursing home residents, a population where the majority are usually over 65.\n3. TRUE - The abstract mentions nursing homes, which fall into the definition of residential-aged care facilities providing daily living and nursing support.\n4. TRUE - The study revolves around the introduction of therapy alpacas for residents, which directly addresses the concept outlined in the inclusion criteria.\n5. TRUE - The abstract states that depression, anxiety, and stress levels of nursing home residents were evaluated and significantly reduced through therapy with alpacas, thus focusing on emotional well-being and mental health."
},
"logprobs": null,
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 784,
"completion_tokens": 152,
"total_tokens": 936
},
"system_fingerprint": null
}
Loading

0 comments on commit 1406986

Please sign in to comment.