From c9e9ca8c273fbba6ee7febfc600c6c183a74fd01 Mon Sep 17 00:00:00 2001 From: Richard Martin-Nielsen Date: Mon, 27 Sep 2021 20:15:59 +0300 Subject: [PATCH 1/5] Add tests for json_reader function and sample json data --- tests/testthat/custom_data/mtcars.json | 1 + tests/testthat/test-json_reader.R | 36 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 tests/testthat/custom_data/mtcars.json create mode 100644 tests/testthat/test-json_reader.R diff --git a/tests/testthat/custom_data/mtcars.json b/tests/testthat/custom_data/mtcars.json new file mode 100644 index 00000000..bd79a009 --- /dev/null +++ b/tests/testthat/custom_data/mtcars.json @@ -0,0 +1 @@ +[{"mpg":21,"cyl":6,"disp":160,"hp":110,"drat":3.9,"wt":2.62,"qsec":16.46,"vs":0,"am":1,"gear":4,"carb":4},{"mpg":21,"cyl":6,"disp":160,"hp":110,"drat":3.9,"wt":2.875,"qsec":17.02,"vs":0,"am":1,"gear":4,"carb":4},{"mpg":22.8,"cyl":4,"disp":108,"hp":93,"drat":3.85,"wt":2.32,"qsec":18.61,"vs":1,"am":1,"gear":4,"carb":1},{"mpg":21.4,"cyl":6,"disp":258,"hp":110,"drat":3.08,"wt":3.215,"qsec":19.44,"vs":1,"am":0,"gear":3,"carb":1},{"mpg":18.7,"cyl":8,"disp":360,"hp":175,"drat":3.15,"wt":3.44,"qsec":17.02,"vs":0,"am":0,"gear":3,"carb":2},{"mpg":18.1,"cyl":6,"disp":225,"hp":105,"drat":2.76,"wt":3.46,"qsec":20.22,"vs":1,"am":0,"gear":3,"carb":1}] diff --git a/tests/testthat/test-json_reader.R b/tests/testthat/test-json_reader.R new file mode 100644 index 00000000..84b46284 --- /dev/null +++ b/tests/testthat/test-json_reader.R @@ -0,0 +1,36 @@ +test_path <- "custom_data/mtcars.json" +target <- tibble::as_tibble(head(mtcars)) + +test_that("json_reader can read in a simple dataset", { + test <- json_reader(test_path) + expect_s3_class(test, "tbl_df") + expect_equal( + colnames(test), + colnames(target) + ) + attributes(test)$spec <- NULL + attributes(test)$problems <- NULL + expect_equal( + test, + target + ) +}) + +test_that("json_reader verbosity is controlled as expected", { + expect_gte( + length(capture.output(tmp <- json_reader(test_path, verbose = TRUE), + type = "message" + )), + 1 + ) + expect_equal( + length(capture.output(tmp <- json_reader(test_path, verbose = FALSE), + type = "message" + )), + 0 + ) +}) + +test_that("json_reader fails as expected when given a file that doesn't exist", { + expect_error(json_reader("nonsense.json", verbose = FALSE)) +}) From e48e358004b86eb06b54a1a92b751510fe78dbb4 Mon Sep 17 00:00:00 2001 From: Richard Martin-Nielsen Date: Mon, 27 Sep 2021 20:26:59 +0300 Subject: [PATCH 2/5] Add tests for the download_JSON method --- NAMESPACE | 2 ++ R/test-DataClass.R | 43 ++++++++++++++++++++++++++++++ man/expect_clean_cols.Rd | 1 + man/expect_columns_contain_data.Rd | 1 + man/expect_processed_cols.Rd | 1 + man/test_cleaning.Rd | 1 + man/test_download.Rd | 1 + man/test_download_JSON.Rd | 33 +++++++++++++++++++++++ man/test_processing.Rd | 1 + man/test_return.Rd | 1 + 10 files changed, 85 insertions(+) create mode 100644 man/test_download_JSON.Rd diff --git a/NAMESPACE b/NAMESPACE index 35d1f272..30a995e7 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -41,6 +41,7 @@ export(start_using_memoise) export(stop_using_memoise) export(test_cleaning) export(test_download) +export(test_download_JSON) export(test_processing) export(test_return) importFrom(R6,R6Class) @@ -74,6 +75,7 @@ importFrom(dplyr,pull) importFrom(dplyr,recode) importFrom(dplyr,rename) importFrom(dplyr,select) +importFrom(dplyr,slice_head) importFrom(dplyr,slice_tail) importFrom(dplyr,starts_with) importFrom(dplyr,summarise) diff --git a/R/test-DataClass.R b/R/test-DataClass.R index e6f7f2fd..f0f48282 100644 --- a/R/test-DataClass.R +++ b/R/test-DataClass.R @@ -121,6 +121,49 @@ test_download <- function(DataClass_obj, download, snapshot_path) { } } + +#' Test download method for JSON files works correctly +#' @description Test data can be downloaded if `download = TRUE`, or a requested +#' snapshot file is not found, and store a snap shot in the `snapshot_dir`. If +#' an existing snapshot file is found then load this data to use in future tests +#' @param DataClass_obj The R6Class object to perform checks on. +#' Must be a `DataClass` or `DataClass` child object. +#' @param download Logical check to download or use a snapshot of the data +#' @param snapshot_path character_array the path to save the downloaded +#' snapshot to. +#' @importFrom purrr map walk +#' @importFrom dplyr slice_head +#' @family tests +#' @concept tests +#' @export +test_download_JSON <- function(DataClass_obj, download, snapshot_path) { + if (!file.exists(snapshot_path)) { + download <- TRUE + } + if (download) { + testthat::test_that( + paste0(DataClass_obj$data_name, " downloads sucessfully"), + { + DataClass_obj$download_JSON() + walk(DataClass_obj$data$raw, function(data) { + testthat::expect_s3_class(data, "data.frame") + testthat::expect_true(nrow(data) > 0) + testthat::expect_true(ncol(data) >= 2 + || typeof(data[[1]]) == "list") + }) + } + ) + DataClass_obj$data$raw <- map(vn$data$raw, + slice_head, + n = 2 + ) + + saveRDS(DataClass_obj$data$raw, snapshot_path) + } else { + DataClass_obj$data$raw <- readRDS(snapshot_path) + } +} + #' Test clean method works correctly #' @description Test data can be cleaned properly. The clean method is invoked #' to generate clean data. This data is checked to ensure it is a data.frame, diff --git a/man/expect_clean_cols.Rd b/man/expect_clean_cols.Rd index 4cbfc03e..c5165fef 100644 --- a/man/expect_clean_cols.Rd +++ b/man/expect_clean_cols.Rd @@ -20,6 +20,7 @@ Functions used for testing data is cleaned and processed correctly \code{\link{expect_columns_contain_data}()}, \code{\link{expect_processed_cols}()}, \code{\link{test_cleaning}()}, +\code{\link{test_download_JSON}()}, \code{\link{test_download}()}, \code{\link{test_processing}()}, \code{\link{test_return}()} diff --git a/man/expect_columns_contain_data.Rd b/man/expect_columns_contain_data.Rd index 75439ae5..3c97b657 100644 --- a/man/expect_columns_contain_data.Rd +++ b/man/expect_columns_contain_data.Rd @@ -19,6 +19,7 @@ Functions used for testing data is cleaned and processed correctly \code{\link{expect_clean_cols}()}, \code{\link{expect_processed_cols}()}, \code{\link{test_cleaning}()}, +\code{\link{test_download_JSON}()}, \code{\link{test_download}()}, \code{\link{test_processing}()}, \code{\link{test_return}()} diff --git a/man/expect_processed_cols.Rd b/man/expect_processed_cols.Rd index ad96cf9e..ae79949d 100644 --- a/man/expect_processed_cols.Rd +++ b/man/expect_processed_cols.Rd @@ -23,6 +23,7 @@ Functions used for testing data is cleaned and processed correctly \code{\link{expect_clean_cols}()}, \code{\link{expect_columns_contain_data}()}, \code{\link{test_cleaning}()}, +\code{\link{test_download_JSON}()}, \code{\link{test_download}()}, \code{\link{test_processing}()}, \code{\link{test_return}()} diff --git a/man/test_cleaning.Rd b/man/test_cleaning.Rd index c7f7414d..4194c5a0 100644 --- a/man/test_cleaning.Rd +++ b/man/test_cleaning.Rd @@ -22,6 +22,7 @@ Functions used for testing data is cleaned and processed correctly \code{\link{expect_clean_cols}()}, \code{\link{expect_columns_contain_data}()}, \code{\link{expect_processed_cols}()}, +\code{\link{test_download_JSON}()}, \code{\link{test_download}()}, \code{\link{test_processing}()}, \code{\link{test_return}()} diff --git a/man/test_download.Rd b/man/test_download.Rd index fb9c32ea..1cd01985 100644 --- a/man/test_download.Rd +++ b/man/test_download.Rd @@ -26,6 +26,7 @@ Functions used for testing data is cleaned and processed correctly \code{\link{expect_columns_contain_data}()}, \code{\link{expect_processed_cols}()}, \code{\link{test_cleaning}()}, +\code{\link{test_download_JSON}()}, \code{\link{test_processing}()}, \code{\link{test_return}()} } diff --git a/man/test_download_JSON.Rd b/man/test_download_JSON.Rd new file mode 100644 index 00000000..8d05564f --- /dev/null +++ b/man/test_download_JSON.Rd @@ -0,0 +1,33 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/test-DataClass.R +\name{test_download_JSON} +\alias{test_download_JSON} +\title{Test download method for JSON files works correctly} +\usage{ +test_download_JSON(DataClass_obj, download, snapshot_path) +} +\arguments{ +\item{DataClass_obj}{The R6Class object to perform checks on. +Must be a \code{DataClass} or \code{DataClass} child object.} + +\item{download}{Logical check to download or use a snapshot of the data} + +\item{snapshot_path}{character_array the path to save the downloaded +snapshot to.} +} +\description{ +Test data can be downloaded if \code{download = TRUE}, or a requested +snapshot file is not found, and store a snap shot in the \code{snapshot_dir}. If +an existing snapshot file is found then load this data to use in future tests +} +\seealso{ +Functions used for testing data is cleaned and processed correctly +\code{\link{expect_clean_cols}()}, +\code{\link{expect_columns_contain_data}()}, +\code{\link{expect_processed_cols}()}, +\code{\link{test_cleaning}()}, +\code{\link{test_download}()}, +\code{\link{test_processing}()}, +\code{\link{test_return}()} +} +\concept{tests} diff --git a/man/test_processing.Rd b/man/test_processing.Rd index 6b7ebb31..3b698d3c 100644 --- a/man/test_processing.Rd +++ b/man/test_processing.Rd @@ -25,6 +25,7 @@ Functions used for testing data is cleaned and processed correctly \code{\link{expect_columns_contain_data}()}, \code{\link{expect_processed_cols}()}, \code{\link{test_cleaning}()}, +\code{\link{test_download_JSON}()}, \code{\link{test_download}()}, \code{\link{test_return}()} } diff --git a/man/test_return.Rd b/man/test_return.Rd index 69715cf9..02f66f34 100644 --- a/man/test_return.Rd +++ b/man/test_return.Rd @@ -22,6 +22,7 @@ Functions used for testing data is cleaned and processed correctly \code{\link{expect_columns_contain_data}()}, \code{\link{expect_processed_cols}()}, \code{\link{test_cleaning}()}, +\code{\link{test_download_JSON}()}, \code{\link{test_download}()}, \code{\link{test_processing}()} } From 8790ebfa98866e53395a37a71fbd8897c9764a88 Mon Sep 17 00:00:00 2001 From: Richard Martin-Nielsen Date: Sun, 3 Oct 2021 17:48:12 +0300 Subject: [PATCH 3/5] Fixing reference to non-existent dataframe --- R/test-DataClass.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/test-DataClass.R b/R/test-DataClass.R index f0f48282..6a5a8bf0 100644 --- a/R/test-DataClass.R +++ b/R/test-DataClass.R @@ -153,7 +153,7 @@ test_download_JSON <- function(DataClass_obj, download, snapshot_path) { }) } ) - DataClass_obj$data$raw <- map(vn$data$raw, + DataClass_obj$data$raw <- map(DataClass_obj$data$raw, slice_head, n = 2 ) From e1afd3188e63dc99da1792f26da7e25501db584a Mon Sep 17 00:00:00 2001 From: Richard Martin-Nielsen Date: Thu, 2 Dec 2021 19:18:25 +0200 Subject: [PATCH 4/5] Update NEWS with JSON tests --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index 4a270315..6ba4cd6a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -12,6 +12,8 @@ This release is currently under development * Change the data source for Switzerland to draw data from the Swiss Federal Office of Public Health (FOPH) * Updated the package logo to include the newly supported data sets. * Reduced the number of package dependencies (@bisaloo and @RichardMN) +* Added tests for JSON download code (@RichardMN). + ## Bug fixes - Fixed a bug in the data sourced from Germany so that instead of treating it as a line list of individuals it is treated as a relatively finely resolved count data which needs to be summed up (by @sbfnk). From f9672ac7f3805b9c8d116d28d9e51d841504f1ea Mon Sep 17 00:00:00 2001 From: Hugo Gruson Date: Fri, 4 Feb 2022 17:08:14 +0100 Subject: [PATCH 5/5] Use less strict tests for JSON input data Since we cannot assume it will always be rectangular before the cleaning step --- R/test-DataClass.R | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/R/test-DataClass.R b/R/test-DataClass.R index 6a5a8bf0..ac9867ed 100644 --- a/R/test-DataClass.R +++ b/R/test-DataClass.R @@ -146,10 +146,8 @@ test_download_JSON <- function(DataClass_obj, download, snapshot_path) { { DataClass_obj$download_JSON() walk(DataClass_obj$data$raw, function(data) { - testthat::expect_s3_class(data, "data.frame") - testthat::expect_true(nrow(data) > 0) - testthat::expect_true(ncol(data) >= 2 - || typeof(data[[1]]) == "list") + testthat::expect_true(length(data) > 0) + testthat::expect_false(is.null(data)) }) } )