From 3ea8bc9dee175d90dde993fe8dc3052ecdeb15a8 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Thu, 18 Jun 2020 15:28:32 +0200 Subject: [PATCH 01/36] generate_habitatcaves: RStudio proj with workflow --- .gitignore | 2 + .../10_generate_habitatcaves.Rmd | 110 ++++++++++++++++++ src/generate_habitatcaves/99_sessioninfo.Rmd | 18 +++ .../generate_habitatcaves.Rproj | 18 +++ src/generate_habitatcaves/index.Rmd | 47 ++++++++ 5 files changed, 195 insertions(+) create mode 100644 src/generate_habitatcaves/10_generate_habitatcaves.Rmd create mode 100644 src/generate_habitatcaves/99_sessioninfo.Rmd create mode 100644 src/generate_habitatcaves/generate_habitatcaves.Rproj create mode 100644 src/generate_habitatcaves/index.Rmd diff --git a/.gitignore b/.gitignore index 47753f6..7f83697 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ src/**/*.md _book* !_bookdown.yml src/private_code.R +*.gpkg +src/generate_habitatcaves/data/ diff --git a/src/generate_habitatcaves/10_generate_habitatcaves.Rmd b/src/generate_habitatcaves/10_generate_habitatcaves.Rmd new file mode 100644 index 0000000..81dd623 --- /dev/null +++ b/src/generate_habitatcaves/10_generate_habitatcaves.Rmd @@ -0,0 +1,110 @@ +# Making the data source + +We will create a 'raw' data source `habitatcaves` by cleaning a precursor dataset. + +```{r} +dir.create("data") +``` + + +## Load draft dataset + +```{r} +drive_ls(as_id("1tJLjAlVbcNHcP4bgp7zRLS9e0KB9vyMs")) %>% + {map2(.$name, .$id, function(name, id) { + drive_download(as_id(id), + path = file.path(tempdir(), name), + overwrite = TRUE) + })} %>% + invisible() +``` + + +```{r paged.print = FALSE} +habitatcaves <- read_sf(file.path(tempdir(), "8310_v2018_RAPHAB2019.shp"), + crs = 31370) +``` + +```{r paged.print = FALSE} +habitatcaves +``` + +## First standardization steps + +From common values of variables `id_old1` and `id_old2` in the below standardized dataset, we will derive a `unit_id` variable that represents somehow interconnected systems: + +```{r paged.print = FALSE} +(habitatcaves <- + habitatcaves %>% + select(id_old1 = Id, + id_old2 = nr, + name = Naam, + type = HT8310, + source = Bron)) +``` + +Writing a derived dataset that contains the auto-generated `unit_id`: + +```{r} +habitatcaves %>% + st_drop_geometry %>% + count(id_old1, id_old2) %>% + filter(n > 1, id_old1 > 0 | id_old2 > 0) %>% + mutate(unit_id = 1:nrow(.)) %>% + select(-n) %>% + right_join(habitatcaves, by = c("id_old1", "id_old2")) %>% + select(-contains("old")) %>% + st_write("data/habitatcaves1.gpkg", + delete_dsn = TRUE) +``` + +## Manual updates + +At this stage, `habitatcaves1.gpkg` has been copied to `habitatcaves2.gpkg` which was subsequently edited in **QGIS** 3.12: + +- updated a truncated value for `source`; +- capitalized a lowercase name in `source`; +- added a few extra `unit_id` values for adjacent polygons. + +## Final standardization and writing the resulting data source + +Reading `habitatcaves2.gpkg` and turning it into a standardized data source: + +```{r paged.print = FALSE} +habitatcaves <- read_sf("data/habitatcaves2.gpkg") +# for spatial sorting: +centr <- + st_centroid(habitatcaves) %>% + st_coordinates() +read_sf("data/habitatcaves2.gpkg") %>% + # filter(!st_is_empty(geom)) %>% + mutate(x = centr[,"X"], + y = centr[,"Y"]) %>% + arrange(unit_id, x, y) %>% + mutate(polygon_id = 1:nrow(.), + unit_id = ifelse(is.na(unit_id), + 100 + polygon_id, + unit_id) %>% + as.integer, + code_orig = type, + type = ifelse(str_detect(type, "8310"), + "8310", + NA)) %>% + select(polygon_id, + unit_id, + name, + code_orig, + type, + source) %>% + st_write("data/habitatcaves3.gpkg", + delete_dsn = TRUE) +``` + +# Checks on the data source + +```{r paged.print = FALSE} +read_sf("data/habitatcaves3.gpkg") %>% + st_drop_geometry %>% + count(source) +``` + diff --git a/src/generate_habitatcaves/99_sessioninfo.Rmd b/src/generate_habitatcaves/99_sessioninfo.Rmd new file mode 100644 index 0000000..98e55f2 --- /dev/null +++ b/src/generate_habitatcaves/99_sessioninfo.Rmd @@ -0,0 +1,18 @@ +# Used environment + +```{r session-info, results = "asis", echo=FALSE} +si <- devtools::session_info() +p <- si$platform %>% + do.call(what = "c") +sprintf("- **%s**:\n %s\n", names(p), p) %>% + cat() +``` + +```{r results = "asis", echo=FALSE} +si$packages %>% + as_tibble %>% + select(package, loadedversion, date, source) %>% +pander::pandoc.table(caption = "(\\#tab:sessioninfo)Loaded R packages", + split.table = Inf) +``` + diff --git a/src/generate_habitatcaves/generate_habitatcaves.Rproj b/src/generate_habitatcaves/generate_habitatcaves.Rproj new file mode 100644 index 0000000..cde2e35 --- /dev/null +++ b/src/generate_habitatcaves/generate_habitatcaves.Rproj @@ -0,0 +1,18 @@ +Version: 1.0 + +RestoreWorkspace: Default +SaveWorkspace: Default +AlwaysSaveHistory: Default + +EnableCodeIndexing: Yes +UseSpacesForTab: Yes +NumSpacesForTab: 2 +Encoding: UTF-8 + +RnwWeave: knitr +LaTeX: XeLaTeX + +AutoAppendNewline: Yes +StripTrailingWhitespace: Yes + +BuildType: Website diff --git a/src/generate_habitatcaves/index.Rmd b/src/generate_habitatcaves/index.Rmd new file mode 100644 index 0000000..49931d8 --- /dev/null +++ b/src/generate_habitatcaves/index.Rmd @@ -0,0 +1,47 @@ +--- +title: "Preparing the raw data source habitatcaves" +# subtitle: "x" +date: "`r lubridate::now()`" +link-citations: true +linkcolor: link.colour +citecolor: link.colour +urlcolor: link.colour +geometry: margin=1in +mainfont: "Calibri" +fontsize: 11pt +documentclass: "article" +# csl: ../inbo.csl +# bibliography: ../references.bib +site: bookdown::bookdown_site +output: + bookdown::html_document2: + keep_md: TRUE + number_sections: yes + fig_caption: yes + df_print: paged + toc: TRUE + toc_float: + collapsed: FALSE + smooth_scroll: FALSE + includes: + in_header: ../header.html +--- + +```{r setup, include=FALSE} +library(sf) +library(dplyr) +library(stringr) +library(knitr) +library(googledrive) +opts_chunk$set( + echo = TRUE, + dpi = 300 +) +``` + +**Note: this is a bookdown project, supposed to be run from within the `src/generate_habitatcaves` subfolder. You can use the `generate_habitatcaves.Rproj` RStudio project file in this subfolder to run it.** + + + + + From b0fa3fff9095f8259e13a9460bf1801d1d78db9a Mon Sep 17 00:00:00 2001 From: florisvdh Date: Thu, 18 Jun 2020 15:29:07 +0200 Subject: [PATCH 02/36] generate_habitatcaves: implement renv --- src/generate_habitatcaves/.Rprofile | 1 + src/generate_habitatcaves/renv.lock | 693 ++++++++++++++++++++ src/generate_habitatcaves/renv/.gitignore | 3 + src/generate_habitatcaves/renv/activate.R | 341 ++++++++++ src/generate_habitatcaves/renv/settings.dcf | 6 + 5 files changed, 1044 insertions(+) create mode 100644 src/generate_habitatcaves/.Rprofile create mode 100644 src/generate_habitatcaves/renv.lock create mode 100644 src/generate_habitatcaves/renv/.gitignore create mode 100644 src/generate_habitatcaves/renv/activate.R create mode 100644 src/generate_habitatcaves/renv/settings.dcf diff --git a/src/generate_habitatcaves/.Rprofile b/src/generate_habitatcaves/.Rprofile new file mode 100644 index 0000000..81b960f --- /dev/null +++ b/src/generate_habitatcaves/.Rprofile @@ -0,0 +1 @@ +source("renv/activate.R") diff --git a/src/generate_habitatcaves/renv.lock b/src/generate_habitatcaves/renv.lock new file mode 100644 index 0000000..304379a --- /dev/null +++ b/src/generate_habitatcaves/renv.lock @@ -0,0 +1,693 @@ +{ + "R": { + "Version": "3.6.3", + "Repositories": [ + { + "Name": "RStudio", + "URL": "http://cloud.r-project.org" + }, + { + "Name": "INLA", + "URL": "https://inla.r-inla-download.org/R/stable" + }, + { + "Name": "inbo", + "URL": "https://inbo.github.io/drat" + } + ] + }, + "Packages": { + "BH": { + "Package": "BH", + "Version": "1.72.0-3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "8f9ce74c6417d61f0782cbae5fd2b7b0" + }, + "DBI": { + "Package": "DBI", + "Version": "1.1.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "4744be45519d675af66c28478720fce5" + }, + "DT": { + "Package": "DT", + "Version": "0.13", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "a16cb2832248c7cff5a6f505e2aea45b" + }, + "KernSmooth": { + "Package": "KernSmooth", + "Version": "2.23-17", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "bbff70c8c0357b5b88238c83f680fcd3" + }, + "MASS": { + "Package": "MASS", + "Version": "7.3-51.6", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "1dad32ac9dbd8057167b2979fb932ff7" + }, + "R6": { + "Package": "R6", + "Version": "2.4.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "292b54f8f4b94669b08f94e5acce6be2" + }, + "Rcpp": { + "Package": "Rcpp", + "Version": "1.0.4.6", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "e652f23d8b1807cc975c51410d05b72f" + }, + "askpass": { + "Package": "askpass", + "Version": "1.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "e8a22846fff485f0be3770c2da758713" + }, + "assertthat": { + "Package": "assertthat", + "Version": "0.2.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "50c838a310445e954bc13f26f26a6ecf" + }, + "backports": { + "Package": "backports", + "Version": "1.1.8", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "3ef0eac19317fd03c0c854aed581d473" + }, + "base64enc": { + "Package": "base64enc", + "Version": "0.1-3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "543776ae6848fde2f48ff3816d0628bc" + }, + "bookdown": { + "Package": "bookdown", + "Version": "0.19", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "4f669b7b4f236538e6afc311f32d2c93" + }, + "brew": { + "Package": "brew", + "Version": "1.0-6", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "92a5f887f9ae3035ac7afde22ba73ee9" + }, + "callr": { + "Package": "callr", + "Version": "3.4.3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "643163a00cb536454c624883a10ae0bc" + }, + "class": { + "Package": "class", + "Version": "7.3-17", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "9267f5dab59a4ef44229858a142bded1" + }, + "classInt": { + "Package": "classInt", + "Version": "0.4-3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "17bdfa3c51df4a6c82484d13b11fb380" + }, + "cli": { + "Package": "cli", + "Version": "2.0.2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "ff0becff7bfdfe3f75d29aff8f3172dd" + }, + "clipr": { + "Package": "clipr", + "Version": "0.7.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "08cf4045c149a0f0eaf405324c7495bd" + }, + "commonmark": { + "Package": "commonmark", + "Version": "1.7", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "0f22be39ec1d141fd03683c06f3a6e67" + }, + "covr": { + "Package": "covr", + "Version": "3.5.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "cbc6df1ef6ee576f844f973c1fc04ab4" + }, + "crayon": { + "Package": "crayon", + "Version": "1.3.4", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "0d57bc8e27b7ba9e45dba825ebc0de6b" + }, + "crosstalk": { + "Package": "crosstalk", + "Version": "1.1.0.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "ae55f5d7c02f0ab43c58dd050694f2b4" + }, + "curl": { + "Package": "curl", + "Version": "4.3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "2b7d10581cc730804e9ed178c8374bd6" + }, + "desc": { + "Package": "desc", + "Version": "1.2.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "6c8fe8fa26a23b79949375d372c7b395" + }, + "devtools": { + "Package": "devtools", + "Version": "2.3.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "d55dae3aa077e25128115a81e974f318" + }, + "digest": { + "Package": "digest", + "Version": "0.6.25", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "f697db7d92b7028c4b3436e9603fb636" + }, + "dplyr": { + "Package": "dplyr", + "Version": "1.0.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "4011f62581a34080e44105d4aa05a97f" + }, + "e1071": { + "Package": "e1071", + "Version": "1.7-3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "728e85416ffc90743054c1ac04486d69" + }, + "ellipsis": { + "Package": "ellipsis", + "Version": "0.3.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "fd2844b3a43ae2d27e70ece2df1b4e2a" + }, + "evaluate": { + "Package": "evaluate", + "Version": "0.14", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "ec8ca05cffcc70569eaaad8469d2a3a7" + }, + "fansi": { + "Package": "fansi", + "Version": "0.4.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "7fce217eaaf8016e72065e85c73027b5" + }, + "fs": { + "Package": "fs", + "Version": "1.4.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "780713bd2f53156fae001443dcdbdcd5" + }, + "gargle": { + "Package": "gargle", + "Version": "0.5.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "aaacaf8b0ec3dfe45df9eb6bc040db44" + }, + "generics": { + "Package": "generics", + "Version": "0.0.2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "b8cff1d1391fd1ad8b65877f4c7f2e53" + }, + "gh": { + "Package": "gh", + "Version": "1.1.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "89ea5998938d1ad55f035c8a86f96b74" + }, + "git2r": { + "Package": "git2r", + "Version": "0.27.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "531a82d1beed1f545beb25f4f5945bf7" + }, + "glue": { + "Package": "glue", + "Version": "1.4.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "f43e0d5e85ccb0a4045670c0607ee504" + }, + "googledrive": { + "Package": "googledrive", + "Version": "1.0.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "79ba5d18133290a69b7c135dc3dfef1a" + }, + "highr": { + "Package": "highr", + "Version": "0.8", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "4dc5bb88961e347a0f4d8aad597cbfac" + }, + "htmltools": { + "Package": "htmltools", + "Version": "0.5.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "7d651b7131794fe007b1ad6f21aaa401" + }, + "htmlwidgets": { + "Package": "htmlwidgets", + "Version": "1.5.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "41bace23583fbc25089edae324de2dc3" + }, + "httr": { + "Package": "httr", + "Version": "1.4.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "7146fea4685b4252ebf478978c75f597" + }, + "ini": { + "Package": "ini", + "Version": "0.3.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "6154ec2223172bce8162d4153cda21f7" + }, + "jsonlite": { + "Package": "jsonlite", + "Version": "1.6.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "84b0ee361e2f78d6b7d670db9471c0c5" + }, + "knitr": { + "Package": "knitr", + "Version": "1.28", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "915a6f0134cdbdf016d7778bc80b2eda" + }, + "later": { + "Package": "later", + "Version": "1.1.0.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "d0a62b247165aabf397fded504660d8a" + }, + "lazyeval": { + "Package": "lazyeval", + "Version": "0.2.2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "d908914ae53b04d4c0c0fd72ecc35370" + }, + "lifecycle": { + "Package": "lifecycle", + "Version": "0.2.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "361811f31f71f8a617a9a68bf63f1f42" + }, + "lubridate": { + "Package": "lubridate", + "Version": "1.7.9", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "fc1c91e2e8d9e1fc932e75aa1ed989b7" + }, + "magrittr": { + "Package": "magrittr", + "Version": "1.5", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "1bb58822a20301cee84a41678e25d9b7" + }, + "markdown": { + "Package": "markdown", + "Version": "1.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "61e4a10781dd00d7d81dd06ca9b94e95" + }, + "memoise": { + "Package": "memoise", + "Version": "1.1.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "58baa74e4603fcfb9a94401c58c8f9b1" + }, + "mime": { + "Package": "mime", + "Version": "0.9", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "e87a35ec73b157552814869f45a63aa3" + }, + "openssl": { + "Package": "openssl", + "Version": "1.4.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "49f7258fd86ebeaea1df24d9ded00478" + }, + "pander": { + "Package": "pander", + "Version": "0.6.3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "121198ce37b5a6b5227edc5a38223da4" + }, + "pillar": { + "Package": "pillar", + "Version": "1.4.4", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "b01d2494d0e2b6b02fae40e1543fbcb0" + }, + "pkgbuild": { + "Package": "pkgbuild", + "Version": "1.0.8", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "8d8b5e29223aabb829246397299f0592" + }, + "pkgconfig": { + "Package": "pkgconfig", + "Version": "2.0.3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "01f28d4278f15c76cddbea05899c5d6f" + }, + "pkgload": { + "Package": "pkgload", + "Version": "1.1.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "b6b150cd4709e0c0c9b5d51ac4376282" + }, + "praise": { + "Package": "praise", + "Version": "1.0.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "a555924add98c99d2f411e37e7d25e9f" + }, + "prettyunits": { + "Package": "prettyunits", + "Version": "1.1.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "95ef9167b75dde9d2ccc3c7528393e7e" + }, + "processx": { + "Package": "processx", + "Version": "3.4.2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "20a082f2bde0ffcd8755779fd476a274" + }, + "promises": { + "Package": "promises", + "Version": "1.1.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "a8730dcbdd19f9047774909f0ec214a4" + }, + "ps": { + "Package": "ps", + "Version": "1.3.3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "425d938eb9c02906a8ac98c0c2a306b5" + }, + "purrr": { + "Package": "purrr", + "Version": "0.3.4", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "97def703420c8ab10d8f0e6c72101e02" + }, + "rcmdcheck": { + "Package": "rcmdcheck", + "Version": "1.3.3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "ed95895886dab6d2a584da45503555da" + }, + "rematch2": { + "Package": "rematch2", + "Version": "2.1.2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "76c9e04c712a05848ae7a23d2f170a40" + }, + "remotes": { + "Package": "remotes", + "Version": "2.1.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "57c3009534f805f0f6476ffee68483cc" + }, + "renv": { + "Package": "renv", + "Version": "0.10.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "0d3f1c2b63e70b3d918246b4e2ca59b7" + }, + "rex": { + "Package": "rex", + "Version": "1.2.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "093584b944440c5cd07a696b3c8e0e4c" + }, + "rlang": { + "Package": "rlang", + "Version": "0.4.6", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "aa263e3ce17b177c49e0daade2ee3cdc" + }, + "rmarkdown": { + "Package": "rmarkdown", + "Version": "2.2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "aa82ab08b203d5a32a5b5f404e80c399" + }, + "roxygen2": { + "Package": "roxygen2", + "Version": "7.1.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "f173062c04dc8e91d7376d914df6efee" + }, + "rprojroot": { + "Package": "rprojroot", + "Version": "1.3-2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "f6a407ae5dd21f6f80a6708bbb6eb3ae" + }, + "rstudioapi": { + "Package": "rstudioapi", + "Version": "0.11", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "33a5b27a03da82ac4b1d43268f80088a" + }, + "rversions": { + "Package": "rversions", + "Version": "2.0.2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "0ec41191f744d0f5afad8c6f35cc36e4" + }, + "sessioninfo": { + "Package": "sessioninfo", + "Version": "1.1.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "308013098befe37484df72c39cf90d6e" + }, + "sf": { + "Package": "sf", + "Version": "0.9-3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "f16ad0ac903df6727dbfb2e30eafcc26" + }, + "stringi": { + "Package": "stringi", + "Version": "1.4.6", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "e99d8d656980d2dd416a962ae55aec90" + }, + "stringr": { + "Package": "stringr", + "Version": "1.4.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "0759e6b6c0957edb1311028a49a35e76" + }, + "sys": { + "Package": "sys", + "Version": "3.3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "507f3116a38d37ad330a038b3be07b66" + }, + "testthat": { + "Package": "testthat", + "Version": "2.3.2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "0829b987b8961fb07f3b1b64a2fbc495" + }, + "tibble": { + "Package": "tibble", + "Version": "3.0.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "1c61e4cad000e03b1bd687db16a75926" + }, + "tidyselect": { + "Package": "tidyselect", + "Version": "1.1.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "6ea435c354e8448819627cf686f66e0a" + }, + "tinytex": { + "Package": "tinytex", + "Version": "0.23", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "a8c653ef8ce67683f588e6088ed45572" + }, + "units": { + "Package": "units", + "Version": "0.6-7", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "4a3df844d6d35ca2ba2b7ba95446b955" + }, + "usethis": { + "Package": "usethis", + "Version": "1.6.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "e1985f5a9985fea2e338fa7eb99018ca" + }, + "utf8": { + "Package": "utf8", + "Version": "1.1.4", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "4a5081acfb7b81a572e4384a7aaf2af1" + }, + "uuid": { + "Package": "uuid", + "Version": "0.1-4", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "e4169eb989a5d03ccb6b628cad1b1b50" + }, + "vctrs": { + "Package": "vctrs", + "Version": "0.3.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "1739235995f08583db4095a28c357207" + }, + "whisker": { + "Package": "whisker", + "Version": "0.4", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "ca970b96d894e90397ed20637a0c1bbe" + }, + "withr": { + "Package": "withr", + "Version": "2.2.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "ecd17882a0b4419545691e095b74ee89" + }, + "xfun": { + "Package": "xfun", + "Version": "0.14", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "60e146e3aba37a547e12306dc3da77a6" + }, + "xml2": { + "Package": "xml2", + "Version": "1.3.2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "d4d71a75dd3ea9eb5fa28cc21f9585e2" + }, + "xopen": { + "Package": "xopen", + "Version": "1.0.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "6c85f015dee9cc7710ddd20f86881f58" + }, + "yaml": { + "Package": "yaml", + "Version": "2.2.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "2826c5d9efb0a88f657c7a679c7106db" + } + } +} diff --git a/src/generate_habitatcaves/renv/.gitignore b/src/generate_habitatcaves/renv/.gitignore new file mode 100644 index 0000000..82740ba --- /dev/null +++ b/src/generate_habitatcaves/renv/.gitignore @@ -0,0 +1,3 @@ +library/ +python/ +staging/ diff --git a/src/generate_habitatcaves/renv/activate.R b/src/generate_habitatcaves/renv/activate.R new file mode 100644 index 0000000..c2fe5e9 --- /dev/null +++ b/src/generate_habitatcaves/renv/activate.R @@ -0,0 +1,341 @@ + +local({ + + # the requested version of renv + version <- "0.10.0" + + # the project directory + project <- getwd() + + # avoid recursion + if (!is.na(Sys.getenv("RENV_R_INITIALIZING", unset = NA))) + return(invisible(TRUE)) + + # signal that we're loading renv during R startup + Sys.setenv("RENV_R_INITIALIZING" = "true") + on.exit(Sys.unsetenv("RENV_R_INITIALIZING"), add = TRUE) + + # signal that we've consented to use renv + options(renv.consent = TRUE) + + # load the 'utils' package eagerly -- this ensures that renv shims, which + # mask 'utils' packages, will come first on the search path + library(utils, lib.loc = .Library) + + # check to see if renv has already been loaded + if ("renv" %in% loadedNamespaces()) { + + # if renv has already been loaded, and it's the requested version of renv, + # nothing to do + spec <- .getNamespaceInfo(.getNamespace("renv"), "spec") + if (identical(spec[["version"]], version)) + return(invisible(TRUE)) + + # otherwise, unload and attempt to load the correct version of renv + unloadNamespace("renv") + + } + + # load bootstrap tools + bootstrap <- function(version, library) { + + # fix up repos + repos <- getOption("repos") + on.exit(options(repos = repos), add = TRUE) + repos[repos == "@CRAN@"] <- "https://cloud.r-project.org" + options(repos = repos) + + # attempt to download renv + tarball <- tryCatch(renv_bootstrap_download(version), error = identity) + if (inherits(tarball, "error")) + stop("failed to download renv ", version) + + # now attempt to install + status <- tryCatch(renv_bootstrap_install(version, tarball, library), error = identity) + if (inherits(status, "error")) + stop("failed to install renv ", version) + + } + + renv_bootstrap_download_impl <- function(url, destfile) { + + mode <- "wb" + + # https://bugs.r-project.org/bugzilla/show_bug.cgi?id=17715 + fixup <- + Sys.info()[["sysname"]] == "Windows" && + substring(url, 1L, 5L) == "file:" + + if (fixup) + mode <- "w+b" + + download.file( + url = url, + destfile = destfile, + mode = mode, + quiet = TRUE + ) + + } + + renv_bootstrap_download <- function(version) { + + methods <- list( + renv_bootstrap_download_cran_latest, + renv_bootstrap_download_cran_archive, + renv_bootstrap_download_github + ) + + for (method in methods) { + path <- tryCatch(method(version), error = identity) + if (is.character(path) && file.exists(path)) + return(path) + } + + stop("failed to download renv ", version) + + } + + renv_bootstrap_download_cran_latest <- function(version) { + + # check for renv on CRAN matching this version + db <- as.data.frame(available.packages(), stringsAsFactors = FALSE) + if (!"renv" %in% rownames(db)) + stop("renv is not available on your declared package repositories") + + entry <- db["renv", ] + if (!identical(entry$Version, version)) + stop("renv is not available on your declared package repositories") + + message("* Downloading renv ", version, " from CRAN ... ", appendLF = FALSE) + + info <- tryCatch( + download.packages("renv", destdir = tempdir()), + condition = identity + ) + + if (inherits(info, "condition")) { + message("FAILED") + return(FALSE) + } + + message("OK") + info[1, 2] + + } + + renv_bootstrap_download_cran_archive <- function(version) { + + name <- sprintf("renv_%s.tar.gz", version) + repos <- getOption("repos") + urls <- file.path(repos, "src/contrib/Archive/renv", name) + destfile <- file.path(tempdir(), name) + + message("* Downloading renv ", version, " from CRAN archive ... ", appendLF = FALSE) + + for (url in urls) { + + status <- tryCatch( + renv_bootstrap_download_impl(url, destfile), + condition = identity + ) + + if (identical(status, 0L)) { + message("OK") + return(destfile) + } + + } + + message("FAILED") + return(FALSE) + + } + + renv_bootstrap_download_github <- function(version) { + + enabled <- Sys.getenv("RENV_BOOTSTRAP_FROM_GITHUB", unset = "TRUE") + if (!identical(enabled, "TRUE")) + return(FALSE) + + # prepare download options + pat <- Sys.getenv("GITHUB_PAT") + if (nzchar(Sys.which("curl")) && nzchar(pat)) { + fmt <- "--location --fail --header \"Authorization: token %s\"" + extra <- sprintf(fmt, pat) + saved <- options("download.file.method", "download.file.extra") + options(download.file.method = "curl", download.file.extra = extra) + on.exit(do.call(base::options, saved), add = TRUE) + } else if (nzchar(Sys.which("wget")) && nzchar(pat)) { + fmt <- "--header=\"Authorization: token %s\"" + extra <- sprintf(fmt, pat) + saved <- options("download.file.method", "download.file.extra") + options(download.file.method = "wget", download.file.extra = extra) + on.exit(do.call(base::options, saved), add = TRUE) + } + + message("* Downloading renv ", version, " from GitHub ... ", appendLF = FALSE) + + url <- file.path("https://api.github.com/repos/rstudio/renv/tarball", version) + name <- sprintf("renv_%s.tar.gz", version) + destfile <- file.path(tempdir(), name) + + status <- tryCatch( + renv_bootstrap_download_impl(url, destfile), + condition = identity + ) + + if (!identical(status, 0L)) { + message("FAILED") + return(FALSE) + } + + message("Done!") + return(destfile) + + } + + renv_bootstrap_install <- function(version, tarball, library) { + + # attempt to install it into project library + message("* Installing renv ", version, " ... ", appendLF = FALSE) + dir.create(library, showWarnings = FALSE, recursive = TRUE) + + # invoke using system2 so we can capture and report output + bin <- R.home("bin") + exe <- if (Sys.info()[["sysname"]] == "Windows") "R.exe" else "R" + r <- file.path(bin, exe) + args <- c("--vanilla", "CMD", "INSTALL", "-l", shQuote(library), shQuote(tarball)) + output <- system2(r, args, stdout = TRUE, stderr = TRUE) + message("Done!") + + # check for successful install + status <- attr(output, "status") + if (is.numeric(status) && !identical(status, 0L)) { + header <- "Error installing renv:" + lines <- paste(rep.int("=", nchar(header)), collapse = "") + text <- c(header, lines, output) + writeLines(text, con = stderr()) + } + + status + + } + + renv_bootstrap_prefix <- function() { + + # construct version prefix + version <- paste(R.version$major, R.version$minor, sep = ".") + prefix <- paste("R", numeric_version(version)[1, 1:2], sep = "-") + + # include SVN revision for development versions of R + # (to avoid sharing platform-specific artefacts with released versions of R) + devel <- + identical(R.version[["status"]], "Under development (unstable)") || + identical(R.version[["nickname"]], "Unsuffered Consequences") + + if (devel) + prefix <- paste(prefix, R.version[["svn rev"]], sep = "-r") + + # build list of path components + components <- c(prefix, R.version$platform) + + # include prefix if provided by user + prefix <- Sys.getenv("RENV_PATHS_PREFIX") + if (nzchar(prefix)) + components <- c(prefix, components) + + # build prefix + paste(components, collapse = "/") + + } + + renv_bootstrap_library_root <- function(project) { + + path <- Sys.getenv("RENV_PATHS_LIBRARY", unset = NA) + if (!is.na(path)) + return(path) + + path <- Sys.getenv("RENV_PATHS_LIBRARY_ROOT", unset = NA) + if (!is.na(path)) + return(file.path(path, basename(project))) + + file.path(project, "renv/library") + + } + + renv_bootstrap_validate_version <- function(version) { + + loadedversion <- utils::packageDescription("renv", fields = "Version") + if (version == loadedversion) + return(TRUE) + + # assume four-component versions are from GitHub; three-component + # versions are from CRAN + components <- strsplit(loadedversion, "[.-]")[[1]] + remote <- if (length(components) == 4L) + paste("rstudio/renv", loadedversion, sep = "@") + else + paste("renv", loadedversion, sep = "@") + + fmt <- paste( + "renv %1$s was loaded from project library, but renv %2$s is recorded in lockfile.", + "Use `renv::record(\"%3$s\")` to record this version in the lockfile.", + "Use `renv::restore(packages = \"renv\")` to install renv %2$s into the project library.", + sep = "\n" + ) + + msg <- sprintf(fmt, loadedversion, version, remote) + warning(msg, call. = FALSE) + + FALSE + + } + + renv_bootstrap_load <- function(project, libpath, version) { + + # try to load renv from the project library + if (!requireNamespace("renv", lib.loc = libpath, quietly = TRUE)) + return(FALSE) + + # warn if the version of renv loaded does not match + renv_bootstrap_validate_version(version) + + # load the project + renv::load(project) + + TRUE + + } + + # construct path to library root + root <- renv_bootstrap_library_root(project) + + # construct library prefix for platform + prefix <- renv_bootstrap_prefix() + + # construct full libpath + libpath <- file.path(root, prefix) + + # attempt to load + if (renv_bootstrap_load(project, libpath, version)) + return(TRUE) + + # load failed; attempt to bootstrap + bootstrap(version, libpath) + + # try again to load + if (requireNamespace("renv", lib.loc = libpath, quietly = TRUE)) { + message("Successfully installed and loaded renv ", version, ".") + return(renv::load()) + } + + # failed to download or load renv; warn the user + msg <- c( + "Failed to find an renv installation: the project will not be loaded.", + "Use `renv::activate()` to re-initialize the project." + ) + + warning(paste(msg, collapse = "\n"), call. = FALSE) + +}) diff --git a/src/generate_habitatcaves/renv/settings.dcf b/src/generate_habitatcaves/renv/settings.dcf new file mode 100644 index 0000000..11a53ea --- /dev/null +++ b/src/generate_habitatcaves/renv/settings.dcf @@ -0,0 +1,6 @@ +external.libraries: +ignored.packages: +package.dependency.fields: Imports, Depends, LinkingTo +snapshot.type: implicit +use.cache: TRUE +vcs.ignore.library: TRUE From 539b52212417398e6bd148bd18e9b6439e932c8e Mon Sep 17 00:00:00 2001 From: florisvdh Date: Thu, 18 Jun 2020 16:11:11 +0200 Subject: [PATCH 03/36] generate_habitatcaves: update renv for gdrive auth --- .../10_generate_habitatcaves.Rmd | 1 + src/generate_habitatcaves/renv.lock | 98 +++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/src/generate_habitatcaves/10_generate_habitatcaves.Rmd b/src/generate_habitatcaves/10_generate_habitatcaves.Rmd index 81dd623..b1a9b08 100644 --- a/src/generate_habitatcaves/10_generate_habitatcaves.Rmd +++ b/src/generate_habitatcaves/10_generate_habitatcaves.Rmd @@ -10,6 +10,7 @@ dir.create("data") ## Load draft dataset ```{r} +drive_auth(email = TRUE) drive_ls(as_id("1tJLjAlVbcNHcP4bgp7zRLS9e0KB9vyMs")) %>% {map2(.$name, .$id, function(name, id) { drive_download(as_id(id), diff --git a/src/generate_habitatcaves/renv.lock b/src/generate_habitatcaves/renv.lock index 304379a..6cc4bcb 100644 --- a/src/generate_habitatcaves/renv.lock +++ b/src/generate_habitatcaves/renv.lock @@ -52,6 +52,13 @@ "Repository": "CRAN", "Hash": "1dad32ac9dbd8057167b2979fb932ff7" }, + "Matrix": { + "Package": "Matrix", + "Version": "1.2-18", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "08588806cba69f04797dab50627428ed" + }, "R6": { "Package": "R6", "Version": "2.4.1", @@ -101,6 +108,13 @@ "Repository": "CRAN", "Hash": "4f669b7b4f236538e6afc311f32d2c93" }, + "boot": { + "Package": "boot", + "Version": "1.3-25", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "bd51734a754b6c2baf28b2d1ebc11e91" + }, "brew": { "Package": "brew", "Version": "1.0-6", @@ -143,6 +157,20 @@ "Repository": "CRAN", "Hash": "08cf4045c149a0f0eaf405324c7495bd" }, + "cluster": { + "Package": "cluster", + "Version": "2.1.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "db63a44aab5aadcb6bf2f129751d129a" + }, + "codetools": { + "Package": "codetools", + "Version": "0.2-16", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "89cf4b8207269ccf82fbeb6473fd662b" + }, "commonmark": { "Package": "commonmark", "Version": "1.7", @@ -234,6 +262,13 @@ "Repository": "CRAN", "Hash": "7fce217eaaf8016e72065e85c73027b5" }, + "foreign": { + "Package": "foreign", + "Version": "0.8-76", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "649b9c7a979d50d869578c73fed072cd" + }, "fs": { "Package": "fs", "Version": "1.4.1", @@ -339,6 +374,13 @@ "Repository": "CRAN", "Hash": "d0a62b247165aabf397fded504660d8a" }, + "lattice": { + "Package": "lattice", + "Version": "0.20-41", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "fbd9285028b0263d76d18c95ae51a53d" + }, "lazyeval": { "Package": "lazyeval", "Version": "0.2.2", @@ -381,6 +423,13 @@ "Repository": "CRAN", "Hash": "58baa74e4603fcfb9a94401c58c8f9b1" }, + "mgcv": { + "Package": "mgcv", + "Version": "1.8-31", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "4bb7e0c4f3557583e1e8d3c9ffb8ba5c" + }, "mime": { "Package": "mime", "Version": "0.9", @@ -388,6 +437,20 @@ "Repository": "CRAN", "Hash": "e87a35ec73b157552814869f45a63aa3" }, + "nlme": { + "Package": "nlme", + "Version": "3.1-148", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "662f52871983ff3e3ef042c62de126df" + }, + "nnet": { + "Package": "nnet", + "Version": "7.3-14", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "0d87e50e11394a7151a28873637d799a" + }, "openssl": { "Package": "openssl", "Version": "1.4.1", @@ -528,6 +591,13 @@ "Repository": "CRAN", "Hash": "f173062c04dc8e91d7376d914df6efee" }, + "rpart": { + "Package": "rpart", + "Version": "4.1-15", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "9787c1fcb680e655d062e7611cadf78e" + }, "rprojroot": { "Package": "rprojroot", "Version": "1.3-2", @@ -549,6 +619,20 @@ "Repository": "CRAN", "Hash": "0ec41191f744d0f5afad8c6f35cc36e4" }, + "rvest": { + "Package": "rvest", + "Version": "0.3.5", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "6a20c2cdf133ebc7ac45888c9ccc052b" + }, + "selectr": { + "Package": "selectr", + "Version": "0.4-2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "3838071b66e0c566d55cc26bd6e27bf4" + }, "sessioninfo": { "Package": "sessioninfo", "Version": "1.1.1", @@ -563,6 +647,13 @@ "Repository": "CRAN", "Hash": "f16ad0ac903df6727dbfb2e30eafcc26" }, + "spatial": { + "Package": "spatial", + "Version": "7.3-12", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "58a02ce0150652b96c044bc67a0df2e5" + }, "stringi": { "Package": "stringi", "Version": "1.4.6", @@ -577,6 +668,13 @@ "Repository": "CRAN", "Hash": "0759e6b6c0957edb1311028a49a35e76" }, + "survival": { + "Package": "survival", + "Version": "3.2-3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "3cc6154c577a82f06250254db30a4bfb" + }, "sys": { "Package": "sys", "Version": "3.3", From 5d86864db85808fc4ba5bca3106d561caa6eccb1 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Thu, 18 Jun 2020 16:11:54 +0200 Subject: [PATCH 04/36] generate_habitatcaves: use gdrive for intermediate result --- .../10_generate_habitatcaves.Rmd | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/generate_habitatcaves/10_generate_habitatcaves.Rmd b/src/generate_habitatcaves/10_generate_habitatcaves.Rmd index b1a9b08..b76c69d 100644 --- a/src/generate_habitatcaves/10_generate_habitatcaves.Rmd +++ b/src/generate_habitatcaves/10_generate_habitatcaves.Rmd @@ -67,12 +67,21 @@ At this stage, `habitatcaves1.gpkg` has been copied to `habitatcaves2.gpkg` whic - capitalized a lowercase name in `source`; - added a few extra `unit_id` values for adjacent polygons. +```{r eval=FALSE} +# Saving/updating the manually crafted habitatcaves2.gpkg in Google Drive: +drive_update(media = "data/habitatcaves2.gpkg", + file = as_id("14MGdxHtxe8VGaCu70Y8Pmc0jusZgpym9")) +``` + ## Final standardization and writing the resulting data source Reading `habitatcaves2.gpkg` and turning it into a standardized data source: ```{r paged.print = FALSE} -habitatcaves <- read_sf("data/habitatcaves2.gpkg") +drive_download(as_id("1aM3hZqEp3ax66PCrhuyjwBZKd3EcALUS"), + path = file.path(tempdir(), "habitatcaves2.gpkg"), + overwrite = TRUE) +habitatcaves <- read_sf(file.path(tempdir(), "habitatcaves2.gpkg")) # for spatial sorting: centr <- st_centroid(habitatcaves) %>% From 6ec003b62f047302ff1747d9ad018532018819b6 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Tue, 30 Jun 2020 09:36:16 +0200 Subject: [PATCH 05/36] generate_habitatquarries: folder & file renaming --- .gitignore | 2 +- .../.Rprofile | 0 .../10_generate_habitatquarries.Rmd} | 0 .../99_sessioninfo.Rmd | 0 .../generate_habitatquarries.Rproj} | 0 .../index.Rmd | 0 .../renv.lock | 0 .../renv/.gitignore | 0 .../renv/activate.R | 0 .../renv/settings.dcf | 0 10 files changed, 1 insertion(+), 1 deletion(-) rename src/{generate_habitatcaves => generate_habitatquarries}/.Rprofile (100%) rename src/{generate_habitatcaves/10_generate_habitatcaves.Rmd => generate_habitatquarries/10_generate_habitatquarries.Rmd} (100%) rename src/{generate_habitatcaves => generate_habitatquarries}/99_sessioninfo.Rmd (100%) rename src/{generate_habitatcaves/generate_habitatcaves.Rproj => generate_habitatquarries/generate_habitatquarries.Rproj} (100%) rename src/{generate_habitatcaves => generate_habitatquarries}/index.Rmd (100%) rename src/{generate_habitatcaves => generate_habitatquarries}/renv.lock (100%) rename src/{generate_habitatcaves => generate_habitatquarries}/renv/.gitignore (100%) rename src/{generate_habitatcaves => generate_habitatquarries}/renv/activate.R (100%) rename src/{generate_habitatcaves => generate_habitatquarries}/renv/settings.dcf (100%) diff --git a/.gitignore b/.gitignore index 7f83697..4649c05 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,4 @@ _book* !_bookdown.yml src/private_code.R *.gpkg -src/generate_habitatcaves/data/ +src/generate_habitatquarries/data/ diff --git a/src/generate_habitatcaves/.Rprofile b/src/generate_habitatquarries/.Rprofile similarity index 100% rename from src/generate_habitatcaves/.Rprofile rename to src/generate_habitatquarries/.Rprofile diff --git a/src/generate_habitatcaves/10_generate_habitatcaves.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd similarity index 100% rename from src/generate_habitatcaves/10_generate_habitatcaves.Rmd rename to src/generate_habitatquarries/10_generate_habitatquarries.Rmd diff --git a/src/generate_habitatcaves/99_sessioninfo.Rmd b/src/generate_habitatquarries/99_sessioninfo.Rmd similarity index 100% rename from src/generate_habitatcaves/99_sessioninfo.Rmd rename to src/generate_habitatquarries/99_sessioninfo.Rmd diff --git a/src/generate_habitatcaves/generate_habitatcaves.Rproj b/src/generate_habitatquarries/generate_habitatquarries.Rproj similarity index 100% rename from src/generate_habitatcaves/generate_habitatcaves.Rproj rename to src/generate_habitatquarries/generate_habitatquarries.Rproj diff --git a/src/generate_habitatcaves/index.Rmd b/src/generate_habitatquarries/index.Rmd similarity index 100% rename from src/generate_habitatcaves/index.Rmd rename to src/generate_habitatquarries/index.Rmd diff --git a/src/generate_habitatcaves/renv.lock b/src/generate_habitatquarries/renv.lock similarity index 100% rename from src/generate_habitatcaves/renv.lock rename to src/generate_habitatquarries/renv.lock diff --git a/src/generate_habitatcaves/renv/.gitignore b/src/generate_habitatquarries/renv/.gitignore similarity index 100% rename from src/generate_habitatcaves/renv/.gitignore rename to src/generate_habitatquarries/renv/.gitignore diff --git a/src/generate_habitatcaves/renv/activate.R b/src/generate_habitatquarries/renv/activate.R similarity index 100% rename from src/generate_habitatcaves/renv/activate.R rename to src/generate_habitatquarries/renv/activate.R diff --git a/src/generate_habitatcaves/renv/settings.dcf b/src/generate_habitatquarries/renv/settings.dcf similarity index 100% rename from src/generate_habitatcaves/renv/settings.dcf rename to src/generate_habitatquarries/renv/settings.dcf From 19c771d683aaec5feb384bea3a5b43746d7010e7 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Tue, 30 Jun 2020 09:36:37 +0200 Subject: [PATCH 06/36] generate_habitatquarries: renaming in Rmd files --- .../10_generate_habitatquarries.Rmd | 36 +++++++++---------- src/generate_habitatquarries/index.Rmd | 4 +-- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index b76c69d..814634e 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -1,6 +1,6 @@ # Making the data source -We will create a 'raw' data source `habitatcaves` by cleaning a precursor dataset. +We will create a 'raw' data source `habitatquarries` by cleaning a precursor dataset. ```{r} dir.create("data") @@ -22,12 +22,12 @@ drive_ls(as_id("1tJLjAlVbcNHcP4bgp7zRLS9e0KB9vyMs")) %>% ```{r paged.print = FALSE} -habitatcaves <- read_sf(file.path(tempdir(), "8310_v2018_RAPHAB2019.shp"), +habitatquarries <- read_sf(file.path(tempdir(), "8310_v2018_RAPHAB2019.shp"), crs = 31370) ``` ```{r paged.print = FALSE} -habitatcaves +habitatquarries ``` ## First standardization steps @@ -35,8 +35,8 @@ habitatcaves From common values of variables `id_old1` and `id_old2` in the below standardized dataset, we will derive a `unit_id` variable that represents somehow interconnected systems: ```{r paged.print = FALSE} -(habitatcaves <- - habitatcaves %>% +(habitatquarries <- + habitatquarries %>% select(id_old1 = Id, id_old2 = nr, name = Naam, @@ -47,46 +47,46 @@ From common values of variables `id_old1` and `id_old2` in the below standardize Writing a derived dataset that contains the auto-generated `unit_id`: ```{r} -habitatcaves %>% +habitatquarries %>% st_drop_geometry %>% count(id_old1, id_old2) %>% filter(n > 1, id_old1 > 0 | id_old2 > 0) %>% mutate(unit_id = 1:nrow(.)) %>% select(-n) %>% - right_join(habitatcaves, by = c("id_old1", "id_old2")) %>% + right_join(habitatquarries, by = c("id_old1", "id_old2")) %>% select(-contains("old")) %>% - st_write("data/habitatcaves1.gpkg", + st_write("data/habitatquarries1.gpkg", delete_dsn = TRUE) ``` ## Manual updates -At this stage, `habitatcaves1.gpkg` has been copied to `habitatcaves2.gpkg` which was subsequently edited in **QGIS** 3.12: +At this stage, `habitatquarries1.gpkg` has been copied to `habitatquarries2.gpkg` which was subsequently edited in **QGIS** 3.12: - updated a truncated value for `source`; - capitalized a lowercase name in `source`; - added a few extra `unit_id` values for adjacent polygons. ```{r eval=FALSE} -# Saving/updating the manually crafted habitatcaves2.gpkg in Google Drive: -drive_update(media = "data/habitatcaves2.gpkg", +# Saving/updating the manually crafted habitatquarries2.gpkg in Google Drive: +drive_update(media = "data/habitatquarries2.gpkg", file = as_id("14MGdxHtxe8VGaCu70Y8Pmc0jusZgpym9")) ``` ## Final standardization and writing the resulting data source -Reading `habitatcaves2.gpkg` and turning it into a standardized data source: +Reading `habitatquarries2.gpkg` and turning it into a standardized data source: ```{r paged.print = FALSE} drive_download(as_id("1aM3hZqEp3ax66PCrhuyjwBZKd3EcALUS"), - path = file.path(tempdir(), "habitatcaves2.gpkg"), + path = file.path(tempdir(), "habitatquarries2.gpkg"), overwrite = TRUE) -habitatcaves <- read_sf(file.path(tempdir(), "habitatcaves2.gpkg")) +habitatquarries <- read_sf(file.path(tempdir(), "habitatquarries2.gpkg")) # for spatial sorting: centr <- - st_centroid(habitatcaves) %>% + st_centroid(habitatquarries) %>% st_coordinates() -read_sf("data/habitatcaves2.gpkg") %>% +read_sf("data/habitatquarries2.gpkg") %>% # filter(!st_is_empty(geom)) %>% mutate(x = centr[,"X"], y = centr[,"Y"]) %>% @@ -106,14 +106,14 @@ read_sf("data/habitatcaves2.gpkg") %>% code_orig, type, source) %>% - st_write("data/habitatcaves3.gpkg", + st_write("data/habitatquarries3.gpkg", delete_dsn = TRUE) ``` # Checks on the data source ```{r paged.print = FALSE} -read_sf("data/habitatcaves3.gpkg") %>% +read_sf("data/habitatquarries3.gpkg") %>% st_drop_geometry %>% count(source) ``` diff --git a/src/generate_habitatquarries/index.Rmd b/src/generate_habitatquarries/index.Rmd index 49931d8..5759f5b 100644 --- a/src/generate_habitatquarries/index.Rmd +++ b/src/generate_habitatquarries/index.Rmd @@ -1,5 +1,5 @@ --- -title: "Preparing the raw data source habitatcaves" +title: "Preparing the raw data source habitatquarries" # subtitle: "x" date: "`r lubridate::now()`" link-citations: true @@ -39,7 +39,7 @@ opts_chunk$set( ) ``` -**Note: this is a bookdown project, supposed to be run from within the `src/generate_habitatcaves` subfolder. You can use the `generate_habitatcaves.Rproj` RStudio project file in this subfolder to run it.** +**Note: this is a bookdown project, supposed to be run from within the `src/generate_habitatquarries` subfolder. You can use the `generate_habitatquarries.Rproj` RStudio project file in this subfolder to run it.** From 79bcf41ea244e3fb2057912cb7f8a256a1b034e7 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Tue, 30 Jun 2020 16:48:42 +0200 Subject: [PATCH 07/36] Update gitignore --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 4649c05..dad80d1 100644 --- a/.gitignore +++ b/.gitignore @@ -15,5 +15,5 @@ src/**/*.md _book* !_bookdown.yml src/private_code.R -*.gpkg -src/generate_habitatquarries/data/ +*.gpkg* +*.qgs From b1f00eb14c7e2feb2c5cbf9b62742d418b1247f4 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Tue, 30 Jun 2020 17:31:38 +0200 Subject: [PATCH 08/36] generate_habitatquarries: fix missing package --- src/generate_habitatquarries/index.Rmd | 1 + 1 file changed, 1 insertion(+) diff --git a/src/generate_habitatquarries/index.Rmd b/src/generate_habitatquarries/index.Rmd index 5759f5b..9b909c8 100644 --- a/src/generate_habitatquarries/index.Rmd +++ b/src/generate_habitatquarries/index.Rmd @@ -31,6 +31,7 @@ output: library(sf) library(dplyr) library(stringr) +library(purrr) library(knitr) library(googledrive) opts_chunk$set( From 7236c8ce83d5ccac07fe005fb9db11f05c361783 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Tue, 30 Jun 2020 17:33:10 +0200 Subject: [PATCH 09/36] generate_habitatquarries: restrict to Flemish habitat observation --- .../10_generate_habitatquarries.Rmd | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index 814634e..4b02a04 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -96,16 +96,25 @@ read_sf("data/habitatquarries2.gpkg") %>% 100 + polygon_id, unit_id) %>% as.integer, - code_orig = type, - type = ifelse(str_detect(type, "8310"), + code_orig = ifelse(str_detect(type, "WAL|NL"), + NA_character_, + type), + is_habitat = case_when( + str_detect(type, "WAL|NL") ~ NA, + str_detect(type, "8310") ~ TRUE, + TRUE ~ FALSE + ), + type = ifelse(str_detect(type, "8310") & + !str_detect(type, "WAL|NL"), "8310", - NA)) %>% + NA_character_)) %>% select(polygon_id, unit_id, name, code_orig, type, - source) %>% + is_habitat, + extra_reference = source) %>% st_write("data/habitatquarries3.gpkg", delete_dsn = TRUE) ``` From 83835a61c24339b3501afe2169ca2af030923445 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Tue, 30 Jun 2020 20:30:59 +0200 Subject: [PATCH 10/36] generate_habitatquarries: bugfix GDrive id --- src/generate_habitatquarries/10_generate_habitatquarries.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index 4b02a04..a2d1a85 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -70,7 +70,7 @@ At this stage, `habitatquarries1.gpkg` has been copied to `habitatquarries2.gpkg ```{r eval=FALSE} # Saving/updating the manually crafted habitatquarries2.gpkg in Google Drive: drive_update(media = "data/habitatquarries2.gpkg", - file = as_id("14MGdxHtxe8VGaCu70Y8Pmc0jusZgpym9")) + file = as_id("1aM3hZqEp3ax66PCrhuyjwBZKd3EcALUS")) ``` ## Final standardization and writing the resulting data source From fa1ae5a80bd3fb207b282616f45754d49ed49898 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Tue, 30 Jun 2020 20:33:14 +0200 Subject: [PATCH 11/36] generate_habitatquarries: bugfix: use downloaded file --- src/generate_habitatquarries/10_generate_habitatquarries.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index a2d1a85..6a388ef 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -86,7 +86,7 @@ habitatquarries <- read_sf(file.path(tempdir(), "habitatquarries2.gpkg")) centr <- st_centroid(habitatquarries) %>% st_coordinates() -read_sf("data/habitatquarries2.gpkg") %>% +habitatquarries %>% # filter(!st_is_empty(geom)) %>% mutate(x = centr[,"X"], y = centr[,"Y"]) %>% From abddf5a85bcc050dfcfc954e1816e39c17680ce8 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Tue, 30 Jun 2020 20:57:12 +0200 Subject: [PATCH 12/36] generate_habitatquarries: more stages done in intermediate --- .../10_generate_habitatquarries.Rmd | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index 6a388ef..7007186 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -61,11 +61,15 @@ habitatquarries %>% ## Manual updates -At this stage, `habitatquarries1.gpkg` has been copied to `habitatquarries2.gpkg` which was subsequently edited in **QGIS** 3.12: +At this stage, `habitatquarries1.gpkg` has been copied to `habitatquarries2.gpkg` which was subsequently edited in **QGIS** 3.14: - updated a truncated value for `source`; - capitalized a lowercase name in `source`; -- added a few extra `unit_id` values for adjacent polygons. +- added a few extra `unit_id` values for adjacent polygons and interconnected quarries; +- deleted 2 rows without geometry; +- updated the value of `type` for one polygon; +- updated the value of `name` for many polygons; +- updated the value of `source` for many polygons. ```{r eval=FALSE} # Saving/updating the manually crafted habitatquarries2.gpkg in Google Drive: From acad7fdfa49b76494dfbb1112446702b07595c91 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Tue, 30 Jun 2020 20:57:59 +0200 Subject: [PATCH 13/36] generate_habitatquarries: write spatial layer of endresult --- .../10_generate_habitatquarries.Rmd | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index 7007186..d23f472 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -91,7 +91,6 @@ centr <- st_centroid(habitatquarries) %>% st_coordinates() habitatquarries %>% - # filter(!st_is_empty(geom)) %>% mutate(x = centr[,"X"], y = centr[,"Y"]) %>% arrange(unit_id, x, y) %>% @@ -111,7 +110,8 @@ habitatquarries %>% type = ifelse(str_detect(type, "8310") & !str_detect(type, "WAL|NL"), "8310", - NA_character_)) %>% + NA_character_), + source = ifelse(source == "", NA, source)) %>% select(polygon_id, unit_id, name, @@ -119,15 +119,15 @@ habitatquarries %>% type, is_habitat, extra_reference = source) %>% - st_write("data/habitatquarries3.gpkg", + st_write("data/habitatquarries.gpkg", delete_dsn = TRUE) ``` # Checks on the data source ```{r paged.print = FALSE} -read_sf("data/habitatquarries3.gpkg") %>% +read_sf("data/habitatquarries.gpkg") %>% st_drop_geometry %>% - count(source) + count(extra_reference) ``` From 6d5edba21fedef2b0992198e14301a570617ff04 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Wed, 1 Jul 2020 09:12:06 +0200 Subject: [PATCH 14/36] generate_habitatquarries: use rprojroot for stability --- .../10_generate_habitatquarries.Rmd | 13 ++++++++----- src/generate_habitatquarries/index.Rmd | 1 + 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index d23f472..0fcffd9 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -3,7 +3,9 @@ We will create a 'raw' data source `habitatquarries` by cleaning a precursor dataset. ```{r} -dir.create("data") +local_root <- find_root(has_file("generate_habitatquarries.Rproj")) +datapath <- file.path(local_root, "data") +if (!dir.exists(datapath)) dir.create(datapath) ``` @@ -55,7 +57,8 @@ habitatquarries %>% select(-n) %>% right_join(habitatquarries, by = c("id_old1", "id_old2")) %>% select(-contains("old")) %>% - st_write("data/habitatquarries1.gpkg", + st_write(file.path(datapath, + "habitatquarries1.gpkg"), delete_dsn = TRUE) ``` @@ -73,7 +76,7 @@ At this stage, `habitatquarries1.gpkg` has been copied to `habitatquarries2.gpkg ```{r eval=FALSE} # Saving/updating the manually crafted habitatquarries2.gpkg in Google Drive: -drive_update(media = "data/habitatquarries2.gpkg", +drive_update(media = file.path(datapath, "habitatquarries2.gpkg"), file = as_id("1aM3hZqEp3ax66PCrhuyjwBZKd3EcALUS")) ``` @@ -119,14 +122,14 @@ habitatquarries %>% type, is_habitat, extra_reference = source) %>% - st_write("data/habitatquarries.gpkg", + st_write(file.path(datapath, "habitatquarries.gpkg"), delete_dsn = TRUE) ``` # Checks on the data source ```{r paged.print = FALSE} -read_sf("data/habitatquarries.gpkg") %>% +read_sf(file.path(datapath, "habitatquarries.gpkg")) %>% st_drop_geometry %>% count(extra_reference) ``` diff --git a/src/generate_habitatquarries/index.Rmd b/src/generate_habitatquarries/index.Rmd index 9b909c8..4ffeda6 100644 --- a/src/generate_habitatquarries/index.Rmd +++ b/src/generate_habitatquarries/index.Rmd @@ -33,6 +33,7 @@ library(dplyr) library(stringr) library(purrr) library(knitr) +library(rprojroot) library(googledrive) opts_chunk$set( echo = TRUE, From c3833c8a5106a830fcadbab1dc82285e92c2b351 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Wed, 1 Jul 2020 10:33:18 +0200 Subject: [PATCH 15/36] generate_habitatquarries: set chunks to eval=FALSE --- .../10_generate_habitatquarries.Rmd | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index 0fcffd9..00d412d 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -11,7 +11,10 @@ if (!dir.exists(datapath)) dir.create(datapath) ## Load draft dataset -```{r} +This code is no longer executed because of the manual steps undertaken afterwards. +Go to [manual-updates]. + +```{r eval=FALSE} drive_auth(email = TRUE) drive_ls(as_id("1tJLjAlVbcNHcP4bgp7zRLS9e0KB9vyMs")) %>% {map2(.$name, .$id, function(name, id) { @@ -23,12 +26,12 @@ drive_ls(as_id("1tJLjAlVbcNHcP4bgp7zRLS9e0KB9vyMs")) %>% ``` -```{r paged.print = FALSE} +```{r paged.print = FALSE, eval=FALSE} habitatquarries <- read_sf(file.path(tempdir(), "8310_v2018_RAPHAB2019.shp"), crs = 31370) ``` -```{r paged.print = FALSE} +```{r paged.print = FALSE, eval=FALSE} habitatquarries ``` @@ -36,7 +39,7 @@ habitatquarries From common values of variables `id_old1` and `id_old2` in the below standardized dataset, we will derive a `unit_id` variable that represents somehow interconnected systems: -```{r paged.print = FALSE} +```{r paged.print = FALSE, eval=FALSE} (habitatquarries <- habitatquarries %>% select(id_old1 = Id, @@ -48,7 +51,7 @@ From common values of variables `id_old1` and `id_old2` in the below standardize Writing a derived dataset that contains the auto-generated `unit_id`: -```{r} +```{r eval=FALSE} habitatquarries %>% st_drop_geometry %>% count(id_old1, id_old2) %>% From d8bd0d053cf5a65df35063be8467b52a1f5788d9 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Wed, 1 Jul 2020 10:35:48 +0200 Subject: [PATCH 16/36] generate_habitatquarries: write bibliography into GPKG --- .gitignore | 1 + .../10_generate_habitatquarries.Rmd | 44 +++++++++ .../data/habitatquarries.bib | 96 +++++++++++++++++++ src/generate_habitatquarries/index.Rmd | 1 + 4 files changed, 142 insertions(+) create mode 100644 src/generate_habitatquarries/data/habitatquarries.bib diff --git a/.gitignore b/.gitignore index dad80d1..79e86de 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ _book* src/private_code.R *.gpkg* *.qgs +*.bak diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index 00d412d..584acbc 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -129,6 +129,50 @@ habitatquarries %>% delete_dsn = TRUE) ``` +# Adding the bibliography + +## Creating a dataframe from the bibtex file + +```{r} +refs <- bib2df(file.path(datapath, "habitatquarries.bib")) +``` + + +```{r} +refs2 <- + refs %>% + map_lgl(function(col) any(!is.na(col))) %>% + refs[,.] %>% + `colnames<-`(tolower(colnames(.))) %>% + mutate(author = map_chr(author, + function(x) paste(x, collapse = " and "))) +``` + + +```{r} +glimpse(refs2) +``` + +Suggestion for making a function to read back as bibtex: + +```{r eval=FALSE} +refs2 %>% + mutate(author = str_split(author, " and ")) %>% + `colnames<-`(toupper(colnames(.))) %>% + df2bib +``` + +## Adding the dataframe + +```{r} +refs2 %>% + st_write(file.path(datapath, "habitatquarries.gpkg"), + layer = "extra_references", + delete_layer = TRUE) +``` + + + # Checks on the data source ```{r paged.print = FALSE} diff --git a/src/generate_habitatquarries/data/habitatquarries.bib b/src/generate_habitatquarries/data/habitatquarries.bib new file mode 100644 index 0000000..05a54b5 --- /dev/null +++ b/src/generate_habitatquarries/data/habitatquarries.bib @@ -0,0 +1,96 @@ +% This file was created with JabRef 2.10. +% Encoding: UTF8 + + +@Book{de_haan_inventarisatie_2018, + Title = {Inventarisatie en waardering van de mergelgroeve ‘{Grote} {Berg}’ te {Zussen}}, + Author = {De Haan, A. and Lahaye, M.}, + Publisher = {Agentschap Onroerend Erfgoed}, + Year = {2018}, + + Address = {Brussels}, + Series = {Onderzoeksrapport van het {Agentschap} {Onroerend} {Erfgoed}}, + + Url = {https://www.vlaanderen.be/publicaties/inventarisatie-en-waardering-van-de-mergelgroeve-grote-berg-te-zussen-onderzoeksrapport} +} + +@InCollection{dusar_mergelgrotten_2007, + Title = {De {Mergelgrotten} van {Hinnisdael} te {Vechmaal}, geologisch erfgoed in de schaduw}, + Author = {Dusar, M and Dreesen, R. and Lagrou, D.}, + Booktitle = {Likona jaarboek}, + Publisher = {Likona (Limburgse Koepel voor Natuurstudie)}, + Year = {2007}, + + Address = {Genk}, + Pages = {6-13}, + Volume = {2006} +} + +@Book{jennekens_pitjesberg_2012, + Title = {De {Pitjesberg}}, + Author = {Jennekens, Peter and Hageman, John}, + Publisher = {Riemster Monumenten en landschappen}, + Year = {2012}, + + Address = {Riemst} +} + +@InCollection{lahaye_groeve_2018, + Title = {Groeve het {Avergat}. {Visie} van de gemeente {Riemst}}, + Author = {Lahaye, Mike}, + Year = {2018}, + + Address = {Riemst}, + Pages = {12}, + + Url = {https://www.riemst.be/sites/default/files/public/_development/5-Wonen/5_7_4_Groeven/Publicaties/visie%20avergat.pdf} +} + +@Book{verhoeven_studieopdracht_2008, + Title = {Studieopdracht naar een archeologische evaluatie van het plateau van {Caestert} ({Riemst}, {Provincie} {Limburg})}, + Author = {Verhoeven, M.P.F.}, + Publisher = {RAAP Archeologisch Adviesbureau}, + Year = {2008}, + + Address = {Weert}, + Number = {1769}, + Series = {{RAAP}-rapport}, + + Url = {https://oar.onroerenderfgoed.be/publicaties/STUA/3/STUA003-001.pdf} +} + +@Book{walschot_over_2010, + Title = {Over groeve {De} {Keel}. {De} boven- en ondergrondse geschiedenis van een kalksteengroeve en haar omgeving}, + Author = {Walschot, Luck}, + Publisher = {Stichting Ondergrondse Werken}, + Year = {2010}, + + ISBN = {97890090257174} +} + + +@misc{wikipedia_muizenberg_2019, + title = {Muizenberg ({Limburg})}, + copyright = {Creative Commons Attribution-ShareAlike License}, + url = {https://nl.wikipedia.org/w/index.php?title=Muizenberg_(Limburg)&oldid=53081088}, + abstract = {De Muizenberg is een ondergrondse mergelgroeve gelegen in het zuidelijke deel van de Cannerberg, grotendeels behorende tot de gemeente Riemst (België), op de grens van Nederland en België ten westen van grenspaal 70. Het gangenstelsel had vroeger een eigen ingang aan de weg Kanne-Vroenhoven, eveneens Muizenberg geheten, en ten zuiden van de veldweg genaamd chemin de la Xhavée. De deels ingestorte groeve stond in verbinding met de groeve Boschberg, thans ook Kasteelgroeve of Cannerberggroeve genoemd.}, + language = {nl}, + urldate = {2020-06-30}, + author = {{Wikipedia}}, + month = {jan}, + year = {2019}, + note = {Page Version ID: 53081088} +} + +@misc{limburgs_landschap_natuurgebied_2020, + title = {Natuurgebied {De} {Coolen}}, + url = {https://limburgs-landschap.be/de-coolen/}, + abstract = {De Coolen is een hellingbosje met een middelgrote mergelgroeve. Omwille van de veiligheid en de bescherming van de vleermuizen is de grot niet toegankelijk.}, + language = {nl}, + urldate = {2020-06-30}, + author = {{Limburgs Landschap}}, + month = {apr}, + year = {2020}, + note = {Library Catalog: limburgs-landschap.be} +} + diff --git a/src/generate_habitatquarries/index.Rmd b/src/generate_habitatquarries/index.Rmd index 4ffeda6..6b0dbaf 100644 --- a/src/generate_habitatquarries/index.Rmd +++ b/src/generate_habitatquarries/index.Rmd @@ -35,6 +35,7 @@ library(purrr) library(knitr) library(rprojroot) library(googledrive) +library(bib2df) opts_chunk$set( echo = TRUE, dpi = 300 From 73b51e5935ff7d4cd526ce9e91e382805129a72f Mon Sep 17 00:00:00 2001 From: florisvdh Date: Wed, 1 Jul 2020 11:46:42 +0200 Subject: [PATCH 17/36] generate_habitatquarries: minor updates --- .../10_generate_habitatquarries.Rmd | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index 584acbc..0ca3b9a 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -11,8 +11,8 @@ if (!dir.exists(datapath)) dir.create(datapath) ## Load draft dataset -This code is no longer executed because of the manual steps undertaken afterwards. -Go to [manual-updates]. +_This code is no longer executed because of the manual steps undertaken afterwards._ +_Go directly to [manual updates](#manual-updates)._ ```{r eval=FALSE} drive_auth(email = TRUE) @@ -37,6 +37,9 @@ habitatquarries ## First standardization steps +_This code is no longer executed because of the manual steps undertaken afterwards._ +_Go directly to [manual updates](#manual-updates)._ + From common values of variables `id_old1` and `id_old2` in the below standardized dataset, we will derive a `unit_id` variable that represents somehow interconnected systems: ```{r paged.print = FALSE, eval=FALSE} @@ -133,7 +136,7 @@ habitatquarries %>% ## Creating a dataframe from the bibtex file -```{r} +```{r warning=FALSE} refs <- bib2df(file.path(datapath, "habitatquarries.bib")) ``` @@ -162,7 +165,7 @@ refs2 %>% df2bib ``` -## Adding the dataframe +## Adding the dataframe to the GeoPackage ```{r} refs2 %>% From 18fdb960c3c2700ac8c2c5b53723084febbe635a Mon Sep 17 00:00:00 2001 From: florisvdh Date: Wed, 1 Jul 2020 13:50:06 +0200 Subject: [PATCH 18/36] generate_habitatquarries: explain about BibTeX bib file --- .../10_generate_habitatquarries.Rmd | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index 0ca3b9a..f0dd8c1 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -134,7 +134,11 @@ habitatquarries %>% # Adding the bibliography -## Creating a dataframe from the bibtex file +The literature references have been saved as a BibTeX bibliography file `habitatquarries.bib`, making it usable by most reference management software and R Markdown. + +We will include this information as a table inside the GeoPackage, in order to make the latter self-contained, but we'll do that in a way that it can be read out to a BibTeX file. + +## Creating a dataframe from the BibTeX bibliography file ```{r warning=FALSE} refs <- bib2df(file.path(datapath, "habitatquarries.bib")) @@ -156,7 +160,7 @@ refs2 <- glimpse(refs2) ``` -Suggestion for making a function to read back as bibtex: +Suggestion for making a function to read back as BibTeX bibliography: ```{r eval=FALSE} refs2 %>% From 03337ecbe2c374c0b686ed5c4b88b0f60adfdad9 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Wed, 1 Jul 2020 13:50:55 +0200 Subject: [PATCH 19/36] generate_habitatquarries: section to check & explore the data source --- .../10_generate_habitatquarries.Rmd | 157 +++++++++++++++++- src/generate_habitatquarries/index.Rmd | 1 + 2 files changed, 157 insertions(+), 1 deletion(-) diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index f0dd8c1..52554bf 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -182,9 +182,164 @@ refs2 %>% # Checks on the data source +Checksums: + +```{r} +filepath <- file.path(datapath, "habitatquarries.gpkg") +openssl::md5(filepath) %>% str_c(collapse = '') %>% `names<-`("md5sum") +openssl::sha256(filepath) %>% str_c(collapse = '') %>% `names<-`("sha256sum") +``` + + +Available layers: + ```{r paged.print = FALSE} -read_sf(file.path(datapath, "habitatquarries.gpkg")) %>% +st_layers(filepath) +``` + +## Geospatial layer + +Reading from file: + +```{r paged.print = FALSE} +habitatquarries_test <- + read_sf(filepath, + layer = "habitatquarries") +``` + +Overview of contents: + +```{r paged.print = FALSE} +habitatquarries_test %>% + print(n = Inf) +``` + +All attributes: + +```{r} +habitatquarries_test %>% + st_drop_geometry +``` +Number of unique combinations of `unit_id` and `name`: + +```{r} +habitatquarries_test %>% + st_drop_geometry %>% + count(unit_id, name) %>% + kable +``` + +Number of unique combinations of `type` and `is_habitat`: + +```{r} +habitatquarries_test %>% + st_drop_geometry %>% + count(type, is_habitat) +``` +Zooming in on non-TRUE cases of `is_habitat` shows that this coincides with two possible values of `code_orig`: + +```{r} +habitatquarries_test %>% + st_drop_geometry %>% + filter(!is_habitat | is.na(is_habitat)) %>% + count(code_orig, type, is_habitat) +``` + +Hence, when `is_habitat = FALSE` we know that no habitat is present. +When unknown (outside of Flanders), it is set as `NA`. + +Occurrence of different references: + +```{r paged.print = FALSE} +habitatquarries_test %>% st_drop_geometry %>% count(extra_reference) ``` +```{r warning=FALSE} +provinces_path <- find_root_file("n2khab_data/10_raw/provinces", + criterion = has_dir("n2khab_data")) +provinces <- + read_sf(provinces_path, crs = 31370) +bbox1 <- st_bbox(habitatquarries_test) +``` + +Overview map: + +```{r} +ggplot() + + geom_sf(data = provinces, fill = "white", colour = "grey70") + + geom_sf(data = habitatquarries_test, + colour = NA, + aes(fill = factor(unit_id))) + + coord_sf(datum = st_crs(31370), + xlim = bbox1$xlim + c(-2e3, 2e3), + ylim = bbox1$ylim + c(-2e3, 2e3)) + + theme_bw() + + theme(legend.position = "none") +``` + +Zoomed in on the eastern part: + +```{r} +zoom <- coord_sf(datum = st_crs(31370), + xlim = c(234e3, 244e3), + ylim = c(163e3, 169.2e3)) +ggplot() + + geom_sf(data = provinces, fill = "white", colour = "grey70") + + geom_sf(data = habitatquarries_test, + colour = NA, + aes(fill = factor(unit_id))) + + zoom + + geom_sf_text(data = habitatquarries_test, + aes(label = unit_id), + size = 3) + + theme_bw() + + theme(legend.position = "none", + axis.title = element_blank()) +``` + +With the combined values shown of `code_orig`, `is_habitat` and `type`: + +```{r} +ggplot() + + geom_sf(data = provinces, fill = "white", colour = "grey70") + + geom_sf(data = + habitatquarries_test %>% + mutate(`code_orig:is_habitat:type` = paste(code_orig, + is_habitat, + type, + sep = ":")), + colour = NA, + aes(fill = `code_orig:is_habitat:type`)) + + zoom + + theme_bw() + + theme(legend.position = "bottom") +``` + + +## Table with extra references + +Reading from file: + +```{r paged.print = FALSE} +extra_references <- + read_sf(filepath, + layer = "extra_references") +``` + +Overview of structure: + +```{r paged.print = FALSE} +extra_references %>% + glimpse() +``` + +Closer inspection: + +```{r} +extra_references +``` + +The above table can be converted back into a BibTeX bibliography file, using code as shown above. + diff --git a/src/generate_habitatquarries/index.Rmd b/src/generate_habitatquarries/index.Rmd index 6b0dbaf..dbb13a0 100644 --- a/src/generate_habitatquarries/index.Rmd +++ b/src/generate_habitatquarries/index.Rmd @@ -36,6 +36,7 @@ library(knitr) library(rprojroot) library(googledrive) library(bib2df) +library(ggplot2) opts_chunk$set( echo = TRUE, dpi = 300 From 11f7ca469618ffe09ebdf61e10e9b1ceb9ee8421 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Wed, 1 Jul 2020 11:46:42 +0200 Subject: [PATCH 20/36] generate_habitatquarries: bookdown administration --- src/generate_habitatquarries/_bookdown.yml | 4 ++++ src/generate_habitatquarries/index.Rmd | 1 + 2 files changed, 5 insertions(+) create mode 100644 src/generate_habitatquarries/_bookdown.yml diff --git a/src/generate_habitatquarries/_bookdown.yml b/src/generate_habitatquarries/_bookdown.yml new file mode 100644 index 0000000..c137ede --- /dev/null +++ b/src/generate_habitatquarries/_bookdown.yml @@ -0,0 +1,4 @@ +book_filename: "generating_habitatquarries.Rmd" +new_session: FALSE +rmd_files: # specifies the order of Rmd files; NOT needed when you use index.Rmd and an alphabetical order for other Rmd files + # - index.Rmd diff --git a/src/generate_habitatquarries/index.Rmd b/src/generate_habitatquarries/index.Rmd index dbb13a0..e2955cb 100644 --- a/src/generate_habitatquarries/index.Rmd +++ b/src/generate_habitatquarries/index.Rmd @@ -15,6 +15,7 @@ documentclass: "article" site: bookdown::bookdown_site output: bookdown::html_document2: + code_folding: show keep_md: TRUE number_sections: yes fig_caption: yes From 4ef14662d68de61f64ae755bb9b10b901fdc7f17 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Wed, 1 Jul 2020 14:10:18 +0200 Subject: [PATCH 21/36] generate_habitatquarries: add sf_extSoftVersion() to env section --- src/generate_habitatquarries/99_sessioninfo.Rmd | 1 + 1 file changed, 1 insertion(+) diff --git a/src/generate_habitatquarries/99_sessioninfo.Rmd b/src/generate_habitatquarries/99_sessioninfo.Rmd index 98e55f2..c06f9d8 100644 --- a/src/generate_habitatquarries/99_sessioninfo.Rmd +++ b/src/generate_habitatquarries/99_sessioninfo.Rmd @@ -4,6 +4,7 @@ si <- devtools::session_info() p <- si$platform %>% do.call(what = "c") +if ("sf" %in% si$packages$package) p <- c(p, sf_extSoftVersion()) sprintf("- **%s**:\n %s\n", names(p), p) %>% cat() ``` From 47503f903d7c7dd39e528bb1270c5911bd7702ca Mon Sep 17 00:00:00 2001 From: florisvdh Date: Wed, 1 Jul 2020 14:17:46 +0200 Subject: [PATCH 22/36] generate_habitatquarries: update renv.lock --- src/generate_habitatquarries/renv.lock | 112 ++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 2 deletions(-) diff --git a/src/generate_habitatquarries/renv.lock b/src/generate_habitatquarries/renv.lock index 6cc4bcb..1065c65 100644 --- a/src/generate_habitatquarries/renv.lock +++ b/src/generate_habitatquarries/renv.lock @@ -66,6 +66,13 @@ "Repository": "CRAN", "Hash": "292b54f8f4b94669b08f94e5acce6be2" }, + "RColorBrewer": { + "Package": "RColorBrewer", + "Version": "1.1-2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "e031418365a7f7a766181ab5a41a5716" + }, "Rcpp": { "Package": "Rcpp", "Version": "1.0.4.6", @@ -101,6 +108,18 @@ "Repository": "CRAN", "Hash": "543776ae6848fde2f48ff3816d0628bc" }, + "bib2df": { + "Package": "bib2df", + "Version": "1.1.2", + "Source": "GitHub", + "RemoteType": "github", + "RemoteHost": "api.github.com", + "RemoteRepo": "bib2df", + "RemoteUsername": "ropensci", + "RemoteRef": "v1.1.2", + "RemoteSha": "ad349fef53d26434c6e270bc386ae18f16b80522", + "Hash": "a335b9407f154160cc3e04c0ec091b4f" + }, "bookdown": { "Package": "bookdown", "Version": "0.19", @@ -171,6 +190,13 @@ "Repository": "CRAN", "Hash": "89cf4b8207269ccf82fbeb6473fd662b" }, + "colorspace": { + "Package": "colorspace", + "Version": "1.4-1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "6b436e95723d1f0e861224dd9b094dfb" + }, "commonmark": { "Package": "commonmark", "Version": "1.7", @@ -262,6 +288,13 @@ "Repository": "CRAN", "Hash": "7fce217eaaf8016e72065e85c73027b5" }, + "farver": { + "Package": "farver", + "Version": "2.0.3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "dad6793a5a1f73c8e91f1a1e3e834b05" + }, "foreign": { "Package": "foreign", "Version": "0.8-76", @@ -290,6 +323,13 @@ "Repository": "CRAN", "Hash": "b8cff1d1391fd1ad8b65877f4c7f2e53" }, + "ggplot2": { + "Package": "ggplot2", + "Version": "3.3.2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "4ded8b439797f7b1693bd3d238d0106b" + }, "gh": { "Package": "gh", "Version": "1.1.0", @@ -318,6 +358,13 @@ "Repository": "CRAN", "Hash": "79ba5d18133290a69b7c135dc3dfef1a" }, + "gtable": { + "Package": "gtable", + "Version": "0.3.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "ac5c6baf7822ce8732b343f14c072c4d" + }, "highr": { "Package": "highr", "Version": "0.8", @@ -346,6 +393,13 @@ "Repository": "CRAN", "Hash": "7146fea4685b4252ebf478978c75f597" }, + "humaniformat": { + "Package": "humaniformat", + "Version": "0.6.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "d521cf9db39ca79250a00029661fb7cd" + }, "ini": { "Package": "ini", "Version": "0.3.1", @@ -353,6 +407,20 @@ "Repository": "CRAN", "Hash": "6154ec2223172bce8162d4153cda21f7" }, + "isoband": { + "Package": "isoband", + "Version": "0.2.2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "6e58bd3d6b3dd82a944cd6f05ade228f" + }, + "iterators": { + "Package": "iterators", + "Version": "1.0.12", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "117128f48662573ff4c4e72608b9e202" + }, "jsonlite": { "Package": "jsonlite", "Version": "1.6.1", @@ -367,6 +435,13 @@ "Repository": "CRAN", "Hash": "915a6f0134cdbdf016d7778bc80b2eda" }, + "labeling": { + "Package": "labeling", + "Version": "0.3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "73832978c1de350df58108c745ed0e3e" + }, "later": { "Package": "later", "Version": "1.1.0.1", @@ -437,6 +512,13 @@ "Repository": "CRAN", "Hash": "e87a35ec73b157552814869f45a63aa3" }, + "munsell": { + "Package": "munsell", + "Version": "0.5.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "6dfe8bf774944bd5595785e3229d8771" + }, "nlme": { "Package": "nlme", "Version": "3.1-148", @@ -626,6 +708,13 @@ "Repository": "CRAN", "Hash": "6a20c2cdf133ebc7ac45888c9ccc052b" }, + "scales": { + "Package": "scales", + "Version": "1.1.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "6f76f71042411426ec8df6c54f34e6dd" + }, "selectr": { "Package": "selectr", "Version": "0.4-2", @@ -642,10 +731,22 @@ }, "sf": { "Package": "sf", - "Version": "0.9-3", + "Version": "0.9-5", + "Source": "GitHub", + "RemoteType": "github", + "RemoteHost": "api.github.com", + "RemoteRepo": "sf", + "RemoteUsername": "r-spatial", + "RemoteRef": "master", + "RemoteSha": "e0debeb55be814bc6d2f8d0903b17ce287bdcc6f", + "Hash": "25e78c6be953746f6dd1e2ffbbd978a4" + }, + "sp": { + "Package": "sp", + "Version": "1.4-2", "Source": "Repository", "Repository": "CRAN", - "Hash": "f16ad0ac903df6727dbfb2e30eafcc26" + "Hash": "3290eebc34ba4df5e213878d54c1e623" }, "spatial": { "Package": "spatial", @@ -745,6 +846,13 @@ "Repository": "CRAN", "Hash": "1739235995f08583db4095a28c357207" }, + "viridisLite": { + "Package": "viridisLite", + "Version": "0.3.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "ce4f6271baa94776db692f1cb2055bee" + }, "whisker": { "Package": "whisker", "Version": "0.4", From 0c16969518823d1771a2a804d1f1ed5db0ac198b Mon Sep 17 00:00:00 2001 From: florisvdh Date: Wed, 1 Jul 2020 15:27:09 +0200 Subject: [PATCH 23/36] generate_habitatquarries: drop variables 'type' and 'is_habitat' * They overcomplicates the data source: 'code_orig', 'type' and 'is_habitat' are more appropriate to have as standardized, derived variables in the resulting output of a function in n2khab. 'code_orig' has been renamed as 'habitattype', in accordance with the approach followed for the raw data source 'habitatsprings'. Such variables are more convenient to have in 'processed' data sources than in a 'raw' data source. --- .../10_generate_habitatquarries.Rmd | 41 ++++--------------- 1 file changed, 8 insertions(+), 33 deletions(-) diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index 52554bf..eefc9a1 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -108,25 +108,14 @@ habitatquarries %>% 100 + polygon_id, unit_id) %>% as.integer, - code_orig = ifelse(str_detect(type, "WAL|NL"), + habitattype = ifelse(str_detect(type, "WAL|NL"), NA_character_, type), - is_habitat = case_when( - str_detect(type, "WAL|NL") ~ NA, - str_detect(type, "8310") ~ TRUE, - TRUE ~ FALSE - ), - type = ifelse(str_detect(type, "8310") & - !str_detect(type, "WAL|NL"), - "8310", - NA_character_), source = ifelse(source == "", NA, source)) %>% select(polygon_id, unit_id, name, - code_orig, - type, - is_habitat, + habitattype, extra_reference = source) %>% st_write(file.path(datapath, "habitatquarries.gpkg"), delete_dsn = TRUE) @@ -229,23 +218,14 @@ habitatquarries_test %>% kable ``` -Number of unique combinations of `type` and `is_habitat`: +Number of unique combinations of `habitattype`: ```{r} habitatquarries_test %>% st_drop_geometry %>% - count(type, is_habitat) + count(habitattype) ``` -Zooming in on non-TRUE cases of `is_habitat` shows that this coincides with two possible values of `code_orig`: - -```{r} -habitatquarries_test %>% - st_drop_geometry %>% - filter(!is_habitat | is.na(is_habitat)) %>% - count(code_orig, type, is_habitat) -``` - -Hence, when `is_habitat = FALSE` we know that no habitat is present. +When `habitattype = "gh"` we know that no habitat is present. When unknown (outside of Flanders), it is set as `NA`. Occurrence of different references: @@ -299,19 +279,14 @@ ggplot() + axis.title = element_blank()) ``` -With the combined values shown of `code_orig`, `is_habitat` and `type`: +With the values shown of `habitattype`: ```{r} ggplot() + geom_sf(data = provinces, fill = "white", colour = "grey70") + - geom_sf(data = - habitatquarries_test %>% - mutate(`code_orig:is_habitat:type` = paste(code_orig, - is_habitat, - type, - sep = ":")), + geom_sf(data = habitatquarries_test, colour = NA, - aes(fill = `code_orig:is_habitat:type`)) + + aes(fill = `habitattype`)) + zoom + theme_bw() + theme(legend.position = "bottom") From 4f41a46fc2cf80c26b8cfd9fd8e210ef28af1f18 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Thu, 2 Jul 2020 08:58:29 +0200 Subject: [PATCH 24/36] generate_habitatquarries: upgrade bib2df (renv) --- src/generate_habitatquarries/renv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/generate_habitatquarries/renv.lock b/src/generate_habitatquarries/renv.lock index 1065c65..31edece 100644 --- a/src/generate_habitatquarries/renv.lock +++ b/src/generate_habitatquarries/renv.lock @@ -116,9 +116,9 @@ "RemoteHost": "api.github.com", "RemoteRepo": "bib2df", "RemoteUsername": "ropensci", - "RemoteRef": "v1.1.2", - "RemoteSha": "ad349fef53d26434c6e270bc386ae18f16b80522", - "Hash": "a335b9407f154160cc3e04c0ec091b4f" + "RemoteRef": "master", + "RemoteSha": "04b2a2394dd743370a09b47e5c18dfd580efa74e", + "Hash": "d3934c6c0b404526704635dce522222a" }, "bookdown": { "Package": "bookdown", From 0b31d841f43c039a72f424b30c351c045fa1ca2f Mon Sep 17 00:00:00 2001 From: florisvdh Date: Tue, 15 Sep 2020 10:40:14 +0200 Subject: [PATCH 25/36] generate_habitatquarries: renv upgrade & simplified snapshot --- src/generate_habitatquarries/renv.lock | 88 +------------------- src/generate_habitatquarries/renv/activate.R | 22 +++-- 2 files changed, 17 insertions(+), 93 deletions(-) diff --git a/src/generate_habitatquarries/renv.lock b/src/generate_habitatquarries/renv.lock index 31edece..a3ee843 100644 --- a/src/generate_habitatquarries/renv.lock +++ b/src/generate_habitatquarries/renv.lock @@ -127,13 +127,6 @@ "Repository": "CRAN", "Hash": "4f669b7b4f236538e6afc311f32d2c93" }, - "boot": { - "Package": "boot", - "Version": "1.3-25", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "bd51734a754b6c2baf28b2d1ebc11e91" - }, "brew": { "Package": "brew", "Version": "1.0-6", @@ -176,20 +169,6 @@ "Repository": "CRAN", "Hash": "08cf4045c149a0f0eaf405324c7495bd" }, - "cluster": { - "Package": "cluster", - "Version": "2.1.0", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "db63a44aab5aadcb6bf2f129751d129a" - }, - "codetools": { - "Package": "codetools", - "Version": "0.2-16", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "89cf4b8207269ccf82fbeb6473fd662b" - }, "colorspace": { "Package": "colorspace", "Version": "1.4-1", @@ -295,13 +274,6 @@ "Repository": "CRAN", "Hash": "dad6793a5a1f73c8e91f1a1e3e834b05" }, - "foreign": { - "Package": "foreign", - "Version": "0.8-76", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "649b9c7a979d50d869578c73fed072cd" - }, "fs": { "Package": "fs", "Version": "1.4.1", @@ -414,13 +386,6 @@ "Repository": "CRAN", "Hash": "6e58bd3d6b3dd82a944cd6f05ade228f" }, - "iterators": { - "Package": "iterators", - "Version": "1.0.12", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "117128f48662573ff4c4e72608b9e202" - }, "jsonlite": { "Package": "jsonlite", "Version": "1.6.1", @@ -526,13 +491,6 @@ "Repository": "CRAN", "Hash": "662f52871983ff3e3ef042c62de126df" }, - "nnet": { - "Package": "nnet", - "Version": "7.3-14", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "0d87e50e11394a7151a28873637d799a" - }, "openssl": { "Package": "openssl", "Version": "1.4.1", @@ -640,10 +598,10 @@ }, "renv": { "Package": "renv", - "Version": "0.10.0", + "Version": "0.12.0", "Source": "Repository", "Repository": "CRAN", - "Hash": "0d3f1c2b63e70b3d918246b4e2ca59b7" + "Hash": "7340c71f46a0fd16506cfa804e224e44" }, "rex": { "Package": "rex", @@ -673,13 +631,6 @@ "Repository": "CRAN", "Hash": "f173062c04dc8e91d7376d914df6efee" }, - "rpart": { - "Package": "rpart", - "Version": "4.1-15", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "9787c1fcb680e655d062e7611cadf78e" - }, "rprojroot": { "Package": "rprojroot", "Version": "1.3-2", @@ -701,13 +652,6 @@ "Repository": "CRAN", "Hash": "0ec41191f744d0f5afad8c6f35cc36e4" }, - "rvest": { - "Package": "rvest", - "Version": "0.3.5", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "6a20c2cdf133ebc7ac45888c9ccc052b" - }, "scales": { "Package": "scales", "Version": "1.1.1", @@ -715,13 +659,6 @@ "Repository": "CRAN", "Hash": "6f76f71042411426ec8df6c54f34e6dd" }, - "selectr": { - "Package": "selectr", - "Version": "0.4-2", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "3838071b66e0c566d55cc26bd6e27bf4" - }, "sessioninfo": { "Package": "sessioninfo", "Version": "1.1.1", @@ -741,20 +678,6 @@ "RemoteSha": "e0debeb55be814bc6d2f8d0903b17ce287bdcc6f", "Hash": "25e78c6be953746f6dd1e2ffbbd978a4" }, - "sp": { - "Package": "sp", - "Version": "1.4-2", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "3290eebc34ba4df5e213878d54c1e623" - }, - "spatial": { - "Package": "spatial", - "Version": "7.3-12", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "58a02ce0150652b96c044bc67a0df2e5" - }, "stringi": { "Package": "stringi", "Version": "1.4.6", @@ -769,13 +692,6 @@ "Repository": "CRAN", "Hash": "0759e6b6c0957edb1311028a49a35e76" }, - "survival": { - "Package": "survival", - "Version": "3.2-3", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "3cc6154c577a82f06250254db30a4bfb" - }, "sys": { "Package": "sys", "Version": "3.3", diff --git a/src/generate_habitatquarries/renv/activate.R b/src/generate_habitatquarries/renv/activate.R index c2fe5e9..ff7e655 100644 --- a/src/generate_habitatquarries/renv/activate.R +++ b/src/generate_habitatquarries/renv/activate.R @@ -2,7 +2,7 @@ local({ # the requested version of renv - version <- "0.10.0" + version <- "0.12.0" # the project directory project <- getwd() @@ -39,8 +39,12 @@ local({ # load bootstrap tools bootstrap <- function(version, library) { + # read repos (respecting override if set) + repos <- Sys.getenv("RENV_CONFIG_REPOS_OVERRIDE", unset = NA) + if (is.na(repos)) + repos <- getOption("repos") + # fix up repos - repos <- getOption("repos") on.exit(options(repos = repos), add = TRUE) repos[repos == "@CRAN@"] <- "https://cloud.r-project.org" options(repos = repos) @@ -100,12 +104,12 @@ local({ # check for renv on CRAN matching this version db <- as.data.frame(available.packages(), stringsAsFactors = FALSE) - if (!"renv" %in% rownames(db)) - stop("renv is not available on your declared package repositories") - entry <- db["renv", ] - if (!identical(entry$Version, version)) - stop("renv is not available on your declared package repositories") + entry <- db[db$Package %in% "renv" & db$Version %in% version, ] + if (nrow(entry) == 0) { + fmt <- "renv %s is not available from your declared package repositories" + stop(sprintf(fmt, version)) + } message("* Downloading renv ", version, " from CRAN ... ", appendLF = FALSE) @@ -324,6 +328,10 @@ local({ # load failed; attempt to bootstrap bootstrap(version, libpath) + # exit early if we're just testing bootstrap + if (!is.na(Sys.getenv("RENV_BOOTSTRAP_INSTALL_ONLY", unset = NA))) + return(TRUE) + # try again to load if (requireNamespace("renv", lib.loc = libpath, quietly = TRUE)) { message("Successfully installed and loaded renv ", version, ".") From f064f0ddb71600e1737b4fcd797709e0eba002e0 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Tue, 15 Sep 2020 15:48:31 +0200 Subject: [PATCH 26/36] update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 79e86de..dfb025d 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ src/private_code.R *.gpkg* *.qgs *.bak +*.qgz From 3e2994b57ec7da3871e5033a3af52d5c4520062a Mon Sep 17 00:00:00 2001 From: florisvdh Date: Thu, 24 Sep 2020 10:59:25 +0200 Subject: [PATCH 27/36] generate_habitatquarries: script comparing prelim. & further amended file --- .../compare_old_new_endresult.R | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 src/generate_habitatquarries/compare_old_new_endresult.R diff --git a/src/generate_habitatquarries/compare_old_new_endresult.R b/src/generate_habitatquarries/compare_old_new_endresult.R new file mode 100644 index 0000000..b74b529 --- /dev/null +++ b/src/generate_habitatquarries/compare_old_new_endresult.R @@ -0,0 +1,134 @@ +# Huidige afspraken: +# 1. Steven werkt shapefile verder af: +# - opsplitsen grensoverschrijdend 8310 +# - nummering van polygon_id: opnieuw maken (1 tot 44), op basis van gesorteerd naamveld +# - nummering van unit_id: kan worden hernummerd door: +# 1. gelijkstellen aan polygon_id +# 2. mergen van sommige nummers waar meerdere polygon_id's één populatie-eenheid zijn. Dus die grensoverschrijdende. +# 2. Floris zet shapefile om naar geopackage (kolomnamen goedzetten, lege strings 'missing' maken, bibliografie als een niet-spatiale tabel toevoegen) +# 3. Steven publiceert op Zenodo + + + +filepath <- file.path(datapath, "habitatquarries.gpkg") +habitatquarries_test <- + read_sf(filepath, + layer = "habitatquarries") + +filepath_new <- "/media/floris/DATA/PROJECTS/09685_NatuurlijkMilieu/160 Bewerkingen en resultaat/Dataketen/habitatquarries_intermediates" +st_layers(filepath_new) +(habitatquarries_new <- + read_sf(filepath_new)) + +# polygon_id + +habitatquarries_test %>% + st_drop_geometry %>% + count(polygon_id) %>% filter(n > 1) + +# non-unique: +habitatquarries_new %>% + st_drop_geometry %>% + count(polygon_id) %>% filter(n > 1) + +# unit_id + +habitatquarries_test %>% + st_drop_geometry %>% + count(unit_id) %>% as.data.frame + +habitatquarries_new %>% + st_drop_geometry %>% + count(unit_id) %>% as.data.frame + + +# name coincidence + +habitatquarries_test %>% + st_drop_geometry %>% + count(name) %>% + full_join(habitatquarries_new %>% + st_drop_geometry %>% count(name), + by = "name") %>% + arrange(name) %>% + View("names") + +# unit_id + name coincidence + +habitatquarries_test %>% + st_drop_geometry %>% + count(name, unit_id) %>% + full_join(habitatquarries_new %>% + st_drop_geometry %>% count(name, unit_id), + by = "name") %>% + arrange(name) %>% + View("unit_names") + +# type + name coincidence + +habitatquarries_test %>% + st_drop_geometry %>% + count(name, unit_id, habitattype) %>% + full_join(habitatquarries_new %>% + st_drop_geometry %>% count(name, unit_id, habitattyp), + by = "name") %>% + arrange(name) %>% + View("habtype_names") + +# extra_reference coincidence + +habitatquarries_test %>% + st_drop_geometry %>% + count(extra_reference) %>% + full_join(habitatquarries_new %>% + st_drop_geometry %>% count(extra_refe), + by = c("extra_reference" = "extra_refe")) %>% + arrange(extra_reference) %>% + View("sources") + + +# missing habtype = outside Flanders? + +provinces_path <- find_root_file("n2khab_data/10_raw/provinces", + criterion = has_dir("n2khab_data")) +provinces <- + read_sf(provinces_path, crs = 31370) +zoom <- coord_sf(datum = st_crs(31370), + xlim = c(234e3, 244e3), + ylim = c(163e3, 169.2e3)) +bbox1 <- st_bbox(habitatquarries_test) +ggplot() + + geom_sf(data = habitatquarries_new %>% filter(habitattyp == "8310"), + colour = "black", + aes(fill = `habitattyp`)) + + geom_sf(data = provinces, fill = NA, colour = "purple") + + zoom + + # coord_sf(datum = st_crs(31370), + # xlim = bbox1$xlim + c(-2e3, 2e3), + # ylim = bbox1$ylim + c(-2e3, 2e3)) + + theme_bw() + + theme(legend.position = "bottom") +ggplot() + + geom_sf(data = habitatquarries_new %>% filter(habitattyp == "gh"), + colour = NA, + aes(fill = `habitattyp`)) + + geom_sf(data = provinces, fill = NA, colour = "purple") + + zoom + + # coord_sf(datum = st_crs(31370), + # xlim = bbox1$xlim + c(-2e3, 2e3), + # ylim = bbox1$ylim + c(-2e3, 2e3)) + + theme_bw() + + theme(legend.position = "bottom") + +ggplot() + + geom_sf(data = habitatquarries_new %>% filter(is.na(habitattyp)), + colour = NA, + fill = "grey70") + + geom_sf(data = provinces, fill = NA, colour = "purple") + + zoom + + # coord_sf(datum = st_crs(31370), + # xlim = bbox1$xlim + c(-2e3, 2e3), + # ylim = bbox1$ylim + c(-2e3, 2e3)) + + theme_bw() + + theme(legend.position = "bottom") + From 33dae137772d38296b48d3c6129991119b840ff9 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Thu, 24 Sep 2020 17:49:43 +0200 Subject: [PATCH 28/36] generate_habitatquarries: comparing prelim. & 2nd amended file --- .../compare_old_new_endresult.R | 93 ++++++++++++++++++- 1 file changed, 91 insertions(+), 2 deletions(-) diff --git a/src/generate_habitatquarries/compare_old_new_endresult.R b/src/generate_habitatquarries/compare_old_new_endresult.R index b74b529..c4dc086 100644 --- a/src/generate_habitatquarries/compare_old_new_endresult.R +++ b/src/generate_habitatquarries/compare_old_new_endresult.R @@ -10,12 +10,22 @@ -filepath <- file.path(datapath, "habitatquarries.gpkg") +filepath <- file.path(datapath, "habitatquarries_previous.gpkg") habitatquarries_test <- read_sf(filepath, layer = "habitatquarries") -filepath_new <- "/media/floris/DATA/PROJECTS/09685_NatuurlijkMilieu/160 Bewerkingen en resultaat/Dataketen/habitatquarries_intermediates" +drive_auth(email = TRUE) +drive_ls(as_id("14MGdxHtxe8VGaCu70Y8Pmc0jusZgpym9")) %>% + filter(str_detect(name, "habitatquarries_v2_20200917")) %>% + {map2(.$name, .$id, function(name, id) { + drive_download(as_id(id), + path = file.path(tempdir(), name), + overwrite = TRUE) + })} %>% + invisible() + +filepath_new <- file.path(tempdir(), "habitatquarries_v2_20200917.shp") st_layers(filepath_new) (habitatquarries_new <- read_sf(filepath_new)) @@ -132,3 +142,82 @@ ggplot() + theme_bw() + theme(legend.position = "bottom") + +# checking reasoning of units with multiple polygons & of unit_id > 100 + +habitatquarries_new %>% + st_drop_geometry %>% + count(unit_id) %>% + filter(n > 1) %>% + semi_join(habitatquarries_new, ., by = "unit_id") %>% + # st_drop_geometry %>% + # count(unit_id, name) + # # A tibble: 7 x 3 + # unit_id name n + # + # 1 100 De Keel 2 + # 2 102 Kleine Keel 2 + # 3 103 Muizenberg 3 + # 4 104 Ternaaien beneden 2 + # 5 105 Ternaaien boven 2 + # 6 106 Caestert 2 + # 7 107 Roosburg - Drie-dagen-berg 2 + ggplot() + + geom_sf(colour = "green", + aes(fill = `habitattyp`)) + + geom_sf(data = provinces, fill = NA, colour = "purple") + + zoom + + # coord_sf(datum = st_crs(31370), + # xlim = bbox1$xlim + c(-2e3, 2e3), + # ylim = bbox1$ylim + c(-2e3, 2e3)) + + theme_bw() + + theme(legend.position = "bottom") + +habitatquarries_new %>% + st_drop_geometry %>% + count(unit_id) %>% + filter(n == 1, unit_id >= 100) %>% + semi_join(habitatquarries_new, ., by = "unit_id") %>% + # st_drop_geometry %>% + # count(unit_id, name) + # # A tibble: 1 x 3 + # unit_id name n + # + # 1 101 De Keel - Balkon 1 + ggplot() + + geom_sf(colour = "green", + aes(fill = `habitattyp`)) + + geom_sf(data = provinces, fill = NA, colour = "purple") + + # zoom + + coord_sf(datum = st_crs(31370), + xlim = c(240e3, 241e3), + ylim = c(168e3, 169e3)) + + theme_bw() + + theme(legend.position = "bottom") + +habitatquarries_new %>% + st_drop_geometry %>% + filter(str_detect(name, "Keel")) + # # A tibble: 5 x 6 + # fid_ polygon_id unit_id name habitattyp extra_refe + # + # 1 0 13 100 De Keel 8310 Walschot 2010 + # 2 0 14 100 De Keel NA Walschot 2010 + # 3 0 18 102 Kleine Keel 8310 Walschot 2010 + # 4 0 19 102 Kleine Keel NA Walschot 2010 + # 5 0 45 101 De Keel - Balkon 8310 Walschot 2010 + + +# how are polygon_ids and unit_ids sorted? + +habitatquarries_new %>% + st_drop_geometry %>% + arrange(name, polygon_id) %>% + View("arranged") + +habitatquarries_new %>% + st_drop_geometry %>% + arrange(unit_id >= 100, name, polygon_id) %>% + select(polygon_id, unit_id, name) %>% + kable + From 0809a336d43f7fe4fca68729a25f91e50735c6dc Mon Sep 17 00:00:00 2001 From: florisvdh Date: Thu, 24 Sep 2020 18:16:07 +0200 Subject: [PATCH 29/36] generate_habitatquarries: drop comparison script --- .../compare_old_new_endresult.R | 223 ------------------ 1 file changed, 223 deletions(-) delete mode 100644 src/generate_habitatquarries/compare_old_new_endresult.R diff --git a/src/generate_habitatquarries/compare_old_new_endresult.R b/src/generate_habitatquarries/compare_old_new_endresult.R deleted file mode 100644 index c4dc086..0000000 --- a/src/generate_habitatquarries/compare_old_new_endresult.R +++ /dev/null @@ -1,223 +0,0 @@ -# Huidige afspraken: -# 1. Steven werkt shapefile verder af: -# - opsplitsen grensoverschrijdend 8310 -# - nummering van polygon_id: opnieuw maken (1 tot 44), op basis van gesorteerd naamveld -# - nummering van unit_id: kan worden hernummerd door: -# 1. gelijkstellen aan polygon_id -# 2. mergen van sommige nummers waar meerdere polygon_id's één populatie-eenheid zijn. Dus die grensoverschrijdende. -# 2. Floris zet shapefile om naar geopackage (kolomnamen goedzetten, lege strings 'missing' maken, bibliografie als een niet-spatiale tabel toevoegen) -# 3. Steven publiceert op Zenodo - - - -filepath <- file.path(datapath, "habitatquarries_previous.gpkg") -habitatquarries_test <- - read_sf(filepath, - layer = "habitatquarries") - -drive_auth(email = TRUE) -drive_ls(as_id("14MGdxHtxe8VGaCu70Y8Pmc0jusZgpym9")) %>% - filter(str_detect(name, "habitatquarries_v2_20200917")) %>% - {map2(.$name, .$id, function(name, id) { - drive_download(as_id(id), - path = file.path(tempdir(), name), - overwrite = TRUE) - })} %>% - invisible() - -filepath_new <- file.path(tempdir(), "habitatquarries_v2_20200917.shp") -st_layers(filepath_new) -(habitatquarries_new <- - read_sf(filepath_new)) - -# polygon_id - -habitatquarries_test %>% - st_drop_geometry %>% - count(polygon_id) %>% filter(n > 1) - -# non-unique: -habitatquarries_new %>% - st_drop_geometry %>% - count(polygon_id) %>% filter(n > 1) - -# unit_id - -habitatquarries_test %>% - st_drop_geometry %>% - count(unit_id) %>% as.data.frame - -habitatquarries_new %>% - st_drop_geometry %>% - count(unit_id) %>% as.data.frame - - -# name coincidence - -habitatquarries_test %>% - st_drop_geometry %>% - count(name) %>% - full_join(habitatquarries_new %>% - st_drop_geometry %>% count(name), - by = "name") %>% - arrange(name) %>% - View("names") - -# unit_id + name coincidence - -habitatquarries_test %>% - st_drop_geometry %>% - count(name, unit_id) %>% - full_join(habitatquarries_new %>% - st_drop_geometry %>% count(name, unit_id), - by = "name") %>% - arrange(name) %>% - View("unit_names") - -# type + name coincidence - -habitatquarries_test %>% - st_drop_geometry %>% - count(name, unit_id, habitattype) %>% - full_join(habitatquarries_new %>% - st_drop_geometry %>% count(name, unit_id, habitattyp), - by = "name") %>% - arrange(name) %>% - View("habtype_names") - -# extra_reference coincidence - -habitatquarries_test %>% - st_drop_geometry %>% - count(extra_reference) %>% - full_join(habitatquarries_new %>% - st_drop_geometry %>% count(extra_refe), - by = c("extra_reference" = "extra_refe")) %>% - arrange(extra_reference) %>% - View("sources") - - -# missing habtype = outside Flanders? - -provinces_path <- find_root_file("n2khab_data/10_raw/provinces", - criterion = has_dir("n2khab_data")) -provinces <- - read_sf(provinces_path, crs = 31370) -zoom <- coord_sf(datum = st_crs(31370), - xlim = c(234e3, 244e3), - ylim = c(163e3, 169.2e3)) -bbox1 <- st_bbox(habitatquarries_test) -ggplot() + - geom_sf(data = habitatquarries_new %>% filter(habitattyp == "8310"), - colour = "black", - aes(fill = `habitattyp`)) + - geom_sf(data = provinces, fill = NA, colour = "purple") + - zoom + - # coord_sf(datum = st_crs(31370), - # xlim = bbox1$xlim + c(-2e3, 2e3), - # ylim = bbox1$ylim + c(-2e3, 2e3)) + - theme_bw() + - theme(legend.position = "bottom") -ggplot() + - geom_sf(data = habitatquarries_new %>% filter(habitattyp == "gh"), - colour = NA, - aes(fill = `habitattyp`)) + - geom_sf(data = provinces, fill = NA, colour = "purple") + - zoom + - # coord_sf(datum = st_crs(31370), - # xlim = bbox1$xlim + c(-2e3, 2e3), - # ylim = bbox1$ylim + c(-2e3, 2e3)) + - theme_bw() + - theme(legend.position = "bottom") - -ggplot() + - geom_sf(data = habitatquarries_new %>% filter(is.na(habitattyp)), - colour = NA, - fill = "grey70") + - geom_sf(data = provinces, fill = NA, colour = "purple") + - zoom + - # coord_sf(datum = st_crs(31370), - # xlim = bbox1$xlim + c(-2e3, 2e3), - # ylim = bbox1$ylim + c(-2e3, 2e3)) + - theme_bw() + - theme(legend.position = "bottom") - - -# checking reasoning of units with multiple polygons & of unit_id > 100 - -habitatquarries_new %>% - st_drop_geometry %>% - count(unit_id) %>% - filter(n > 1) %>% - semi_join(habitatquarries_new, ., by = "unit_id") %>% - # st_drop_geometry %>% - # count(unit_id, name) - # # A tibble: 7 x 3 - # unit_id name n - # - # 1 100 De Keel 2 - # 2 102 Kleine Keel 2 - # 3 103 Muizenberg 3 - # 4 104 Ternaaien beneden 2 - # 5 105 Ternaaien boven 2 - # 6 106 Caestert 2 - # 7 107 Roosburg - Drie-dagen-berg 2 - ggplot() + - geom_sf(colour = "green", - aes(fill = `habitattyp`)) + - geom_sf(data = provinces, fill = NA, colour = "purple") + - zoom + - # coord_sf(datum = st_crs(31370), - # xlim = bbox1$xlim + c(-2e3, 2e3), - # ylim = bbox1$ylim + c(-2e3, 2e3)) + - theme_bw() + - theme(legend.position = "bottom") - -habitatquarries_new %>% - st_drop_geometry %>% - count(unit_id) %>% - filter(n == 1, unit_id >= 100) %>% - semi_join(habitatquarries_new, ., by = "unit_id") %>% - # st_drop_geometry %>% - # count(unit_id, name) - # # A tibble: 1 x 3 - # unit_id name n - # - # 1 101 De Keel - Balkon 1 - ggplot() + - geom_sf(colour = "green", - aes(fill = `habitattyp`)) + - geom_sf(data = provinces, fill = NA, colour = "purple") + - # zoom + - coord_sf(datum = st_crs(31370), - xlim = c(240e3, 241e3), - ylim = c(168e3, 169e3)) + - theme_bw() + - theme(legend.position = "bottom") - -habitatquarries_new %>% - st_drop_geometry %>% - filter(str_detect(name, "Keel")) - # # A tibble: 5 x 6 - # fid_ polygon_id unit_id name habitattyp extra_refe - # - # 1 0 13 100 De Keel 8310 Walschot 2010 - # 2 0 14 100 De Keel NA Walschot 2010 - # 3 0 18 102 Kleine Keel 8310 Walschot 2010 - # 4 0 19 102 Kleine Keel NA Walschot 2010 - # 5 0 45 101 De Keel - Balkon 8310 Walschot 2010 - - -# how are polygon_ids and unit_ids sorted? - -habitatquarries_new %>% - st_drop_geometry %>% - arrange(name, polygon_id) %>% - View("arranged") - -habitatquarries_new %>% - st_drop_geometry %>% - arrange(unit_id >= 100, name, polygon_id) %>% - select(polygon_id, unit_id, name) %>% - kable - From c630d6913ed40109ffa60afbbdf2f8214c755788 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Thu, 24 Sep 2020 17:51:36 +0200 Subject: [PATCH 30/36] generate_habitatquarries: bugfix filehash calculation --- .../10_generate_habitatquarries.Rmd | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index eefc9a1..3c2e322 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -175,8 +175,14 @@ Checksums: ```{r} filepath <- file.path(datapath, "habitatquarries.gpkg") -openssl::md5(filepath) %>% str_c(collapse = '') %>% `names<-`("md5sum") -openssl::sha256(filepath) %>% str_c(collapse = '') %>% `names<-`("sha256sum") +file(filepath) %>% + openssl::md5() %>% + str_c(collapse = '') %>% + `names<-`("md5sum") +file(filepath) %>% + openssl::sha256() %>% + str_c(collapse = '') %>% + `names<-`("sha256sum") ``` From 39f85164b5d75ca52dd59209b9fc1ce3dc0e650a Mon Sep 17 00:00:00 2001 From: florisvdh Date: Thu, 24 Sep 2020 17:51:56 +0200 Subject: [PATCH 31/36] generate_habitatquarries: small text fix --- src/generate_habitatquarries/10_generate_habitatquarries.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index 3c2e322..0120814 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -224,7 +224,7 @@ habitatquarries_test %>% kable ``` -Number of unique combinations of `habitattype`: +Number of unique values of `habitattype`: ```{r} habitatquarries_test %>% From 0bfbf45a0c61182fae5ebdf093f745bc4b42c101 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Thu, 24 Sep 2020 17:53:11 +0200 Subject: [PATCH 32/36] generate_habitatquarries: rewrite file standardization * - this is because more manual steps were undertaken than before - also, the final file destination is now set correctly --- .../10_generate_habitatquarries.Rmd | 69 +++++++++++-------- 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index 0120814..e743449 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -6,6 +6,9 @@ We will create a 'raw' data source `habitatquarries` by cleaning a precursor dat local_root <- find_root(has_file("generate_habitatquarries.Rproj")) datapath <- file.path(local_root, "data") if (!dir.exists(datapath)) dir.create(datapath) +finalpath <- find_root_file("n2khab_data/10_raw/habitatquarries", + criterion = has_dir("n2khab_data")) +if (!dir.exists(finalpath)) dir.create(finalpath) ``` @@ -86,38 +89,42 @@ drive_update(media = file.path(datapath, "habitatquarries2.gpkg"), file = as_id("1aM3hZqEp3ax66PCrhuyjwBZKd3EcALUS")) ``` -## Final standardization and writing the resulting data source +After that, further standardization has been done on the result, in an attempt to create a first version of the final file. +This refers to the state of paragraph \@ref(standardization) at commit 4f41a46 (2020-07-02 08:58:29 +0200). -Reading `habitatquarries2.gpkg` and turning it into a standardized data source: +Then, the file has been reworked manually again, regarding both geometry and non-spatial attributes, according to several conclusions written down in `compare_old_new_endresult.R`. +This mainly had to do with the applied definition of a 'unit'; see the Zenodo metadata for more information. +The result was the shapefile `habitatquarries_v2_20200917.shp`, which will now be handled further by the R code. -```{r paged.print = FALSE} -drive_download(as_id("1aM3hZqEp3ax66PCrhuyjwBZKd3EcALUS"), - path = file.path(tempdir(), "habitatquarries2.gpkg"), +## Final standardization and writing the resulting data source {#standardization} + +Reading `habitatquarries_v2_20200917.shp` and turning it into a standardized data source: + +```{r} +drive_auth(email = TRUE) +drive_ls(as_id("14MGdxHtxe8VGaCu70Y8Pmc0jusZgpym9")) %>% + filter(str_detect(name, "habitatquarries_v2_20200917")) %>% + {map2(.$name, .$id, function(name, id) { + drive_download(as_id(id), + path = file.path(tempdir(), name), overwrite = TRUE) -habitatquarries <- read_sf(file.path(tempdir(), "habitatquarries2.gpkg")) -# for spatial sorting: -centr <- - st_centroid(habitatquarries) %>% - st_coordinates() -habitatquarries %>% - mutate(x = centr[,"X"], - y = centr[,"Y"]) %>% - arrange(unit_id, x, y) %>% - mutate(polygon_id = 1:nrow(.), - unit_id = ifelse(is.na(unit_id), - 100 + polygon_id, - unit_id) %>% - as.integer, - habitattype = ifelse(str_detect(type, "WAL|NL"), - NA_character_, - type), - source = ifelse(source == "", NA, source)) %>% + })} %>% + invisible() +``` + + +```{r paged.print = FALSE} +read_sf(file.path(tempdir(), + "habitatquarries_v2_20200917.shp")) %>% select(polygon_id, unit_id, - name, - habitattype, - extra_reference = source) %>% - st_write(file.path(datapath, "habitatquarries.gpkg"), + name, + habitattype = habitattyp, + extra_reference = extra_refe) %>% + arrange(unit_id >= 100, name, polygon_id) %>% + mutate(polygon_id = as.integer(polygon_id), + unit_id = as.integer(unit_id)) %>% + st_write(file.path(finalpath, "habitatquarries.gpkg"), delete_dsn = TRUE) ``` @@ -162,7 +169,7 @@ refs2 %>% ```{r} refs2 %>% - st_write(file.path(datapath, "habitatquarries.gpkg"), + st_write(file.path(finalpath, "habitatquarries.gpkg"), layer = "extra_references", delete_layer = TRUE) ``` @@ -171,10 +178,14 @@ refs2 %>% # Checks on the data source +```{r} +filepath <- file.path(finalpath, "habitatquarries.gpkg") +``` + + Checksums: ```{r} -filepath <- file.path(datapath, "habitatquarries.gpkg") file(filepath) %>% openssl::md5() %>% str_c(collapse = '') %>% From 4d90115700bc3ae46ddbe4e1f2c7cfdfc135d565 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Fri, 25 Sep 2020 16:13:19 +0200 Subject: [PATCH 33/36] generate_habitatquarries: drop comparison script from text --- src/generate_habitatquarries/10_generate_habitatquarries.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index e743449..9f749c0 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -92,7 +92,7 @@ drive_update(media = file.path(datapath, "habitatquarries2.gpkg"), After that, further standardization has been done on the result, in an attempt to create a first version of the final file. This refers to the state of paragraph \@ref(standardization) at commit 4f41a46 (2020-07-02 08:58:29 +0200). -Then, the file has been reworked manually again, regarding both geometry and non-spatial attributes, according to several conclusions written down in `compare_old_new_endresult.R`. +Then, the file has been reworked manually again, regarding both geometry and non-spatial attributes. This mainly had to do with the applied definition of a 'unit'; see the Zenodo metadata for more information. The result was the shapefile `habitatquarries_v2_20200917.shp`, which will now be handled further by the R code. From 9f09ac03916f3f64fcc0ed1b7a53f74bc147d12f Mon Sep 17 00:00:00 2001 From: florisvdh Date: Fri, 25 Sep 2020 16:32:00 +0200 Subject: [PATCH 34/36] generate_habitatquarries: save attributes as tsv for version control --- .../10_generate_habitatquarries.Rmd | 8 ++++ .../data/habitatquarries.tsv | 46 +++++++++++++++++++ src/generate_habitatquarries/index.Rmd | 1 + src/generate_habitatquarries/renv.lock | 14 ++++++ 4 files changed, 69 insertions(+) create mode 100644 src/generate_habitatquarries/data/habitatquarries.tsv diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index 9f749c0..0d1b2f5 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -213,6 +213,14 @@ habitatquarries_test <- layer = "habitatquarries") ``` +Writing attributes to text file for version control: + +```{r} +habitatquarries_test %>% + st_drop_geometry %>% + write_tsv("data/habitatquarries.tsv") +``` + Overview of contents: ```{r paged.print = FALSE} diff --git a/src/generate_habitatquarries/data/habitatquarries.tsv b/src/generate_habitatquarries/data/habitatquarries.tsv new file mode 100644 index 0000000..c834a87 --- /dev/null +++ b/src/generate_habitatquarries/data/habitatquarries.tsv @@ -0,0 +1,46 @@ +polygon_id unit_id name habitattype extra_reference +4 4 Avergat - Driesberg gh Lahaye 2018 +6 6 Avergat - Grootberg 8310 Lahaye 2018 +5 5 Avergat - Putberg gh Lahaye 2018 +20 20 Coolen 8310 Limburgs Landschap 2020; pers. comm. Veerle Cielen (Limburgs Landschap vzw) +21 21 Coolen gh Limburgs Landschap 2020 +29 29 Groeve Lindestraat 8310 NA +31 31 Grote berg - centraal 8310 De Haan & Lahaye 2018 +37 37 Grote berg - Pauly 8310 De Haan & Lahaye 2018 +24 24 Henisdael - Champignonkuil 8310 Dusar et al. 2007 +34 34 Henisdael - Grote Kuil 8310 Dusar et al. 2007 +36 36 Henisdael - Hussenkuil noord 8310 Dusar et al. 2007 +27 27 Henisdael - Hussenkuil zuid 8310 Dusar et al. 2007 +25 25 Henisdael - Walenkuil 8310 Dusar et al. 2007 +26 26 Henisdael - Waterkuil VII 8310 Dusar et al. 2007 +35 35 Henisdael - Waterkuil VIII 8310 Dusar et al. 2007 +44 44 Jageneau's Huiske 8310 NA +28 28 Janekes Koet 8310 NA +32 32 Juge Zussen 8310 NA +1 1 Kuilen van Valmeer gh NA +2 2 Kuilen van Valmeer gh NA +3 3 Kuilen van Valmeer gh NA +30 30 Pitjesberg - centraal 8310 Jennekens & Hageman 2012 +38 38 Pitjesberg - Tiendenschuur 8310 Jennekens & Hageman 2012 +41 41 Roosburg - Flessenberg 8310 NA +40 40 Roosburg - Koeberg 8310 NA +33 33 Roosburg - Verbiestberg 8310 NA +39 39 Roosburg - Verbiestberg midden 8310 NA +22 22 Werken van Mathuus 8310 Walschot 2010 +23 23 Werken van Mathuus 8310 Walschot 2010 +7 106 Caestert 8310 Verhoeven 2008 +8 106 Caestert NA Verhoeven 2008 +13 100 De Keel 8310 Walschot 2010 +14 100 De Keel NA Walschot 2010 +45 101 De Keel - Balkon 8310 Walschot 2010 +18 102 Kleine Keel 8310 Walschot 2010 +19 102 Kleine Keel NA Walschot 2010 +15 103 Muizenberg gh Walschot 2010; Wikipedia 2019 +16 103 Muizenberg NA Walschot 2010; Wikipedia 2019 +17 103 Muizenberg gh Walschot 2010; Wikipedia 2019 +42 107 Roosburg - Drie-dagen-berg 8310 NA +43 107 Roosburg - Drie-dagen-berg NA NA +11 104 Ternaaien beneden 8310 Verhoeven 2008 +12 104 Ternaaien beneden NA Verhoeven 2008 +9 105 Ternaaien boven 8310 Verhoeven 2008 +10 105 Ternaaien boven NA Verhoeven 2008 diff --git a/src/generate_habitatquarries/index.Rmd b/src/generate_habitatquarries/index.Rmd index e2955cb..f5fab6b 100644 --- a/src/generate_habitatquarries/index.Rmd +++ b/src/generate_habitatquarries/index.Rmd @@ -38,6 +38,7 @@ library(rprojroot) library(googledrive) library(bib2df) library(ggplot2) +library(readr) opts_chunk$set( echo = TRUE, dpi = 300 diff --git a/src/generate_habitatquarries/renv.lock b/src/generate_habitatquarries/renv.lock index a3ee843..82cc818 100644 --- a/src/generate_habitatquarries/renv.lock +++ b/src/generate_habitatquarries/renv.lock @@ -344,6 +344,13 @@ "Repository": "CRAN", "Hash": "4dc5bb88961e347a0f4d8aad597cbfac" }, + "hms": { + "Package": "hms", + "Version": "0.5.3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "726671f634529d470545f9fd1a9d1869" + }, "htmltools": { "Package": "htmltools", "Version": "0.5.0", @@ -582,6 +589,13 @@ "Repository": "CRAN", "Hash": "ed95895886dab6d2a584da45503555da" }, + "readr": { + "Package": "readr", + "Version": "1.3.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "af8ab99cd936773a148963905736907b" + }, "rematch2": { "Package": "rematch2", "Version": "2.1.2", From 4c58308f53214081bc438db6cbf905aa16425149 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Fri, 25 Sep 2020 16:43:44 +0200 Subject: [PATCH 35/36] generate_habitatquarries: add literature reference in bib file --- .../data/habitatquarries.bib | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/generate_habitatquarries/data/habitatquarries.bib b/src/generate_habitatquarries/data/habitatquarries.bib index 05a54b5..50eb373 100644 --- a/src/generate_habitatquarries/data/habitatquarries.bib +++ b/src/generate_habitatquarries/data/habitatquarries.bib @@ -94,3 +94,16 @@ @misc{limburgs_landschap_natuurgebied_2020 note = {Library Catalog: limburgs-landschap.be} } +@article{silvertant_ontstaansgeschiedenis_2003, + title = {De Ontstaansgeschiedenis van de Gangenstelsels Te {{Klein}}-{{Ternaaien}} ({{België}})}, + author = {Silvertant, J.}, + editor = {{Natuurhistorisch Genootschap in Limburg Maastricht NL}}, + year = {2003}, + volume = {92}, + pages = {334--340}, + issn = {0028-1107}, + journal = {Natuurhistorisch Maandblad}, + keywords = {belgië,belgium,geschiedenis,history,marl soils,mergelgronden,mijnbouw,mining,onderzoek,quarries,research,steengroeven}, + number = {12} +} + From c0821ebc07dbd3387994c15e431a1f504fe79d71 Mon Sep 17 00:00:00 2001 From: florisvdh Date: Fri, 25 Sep 2020 16:50:37 +0200 Subject: [PATCH 36/36] generate_habitatquarries: use new version of shapefile --- .../10_generate_habitatquarries.Rmd | 8 ++++---- .../data/habitatquarries.tsv | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd index 0d1b2f5..10c9137 100644 --- a/src/generate_habitatquarries/10_generate_habitatquarries.Rmd +++ b/src/generate_habitatquarries/10_generate_habitatquarries.Rmd @@ -94,16 +94,16 @@ This refers to the state of paragraph \@ref(standardization) at commit 4f41a46 ( Then, the file has been reworked manually again, regarding both geometry and non-spatial attributes. This mainly had to do with the applied definition of a 'unit'; see the Zenodo metadata for more information. -The result was the shapefile `habitatquarries_v2_20200917.shp`, which will now be handled further by the R code. +The result was the shapefile `habitatquarries_v2_20200925.shp`, which will now be handled further by the R code. ## Final standardization and writing the resulting data source {#standardization} -Reading `habitatquarries_v2_20200917.shp` and turning it into a standardized data source: +Reading `habitatquarries_v2_20200925.shp` and turning it into a standardized data source: ```{r} drive_auth(email = TRUE) drive_ls(as_id("14MGdxHtxe8VGaCu70Y8Pmc0jusZgpym9")) %>% - filter(str_detect(name, "habitatquarries_v2_20200917")) %>% + filter(str_detect(name, "habitatquarries_v2_20200925")) %>% {map2(.$name, .$id, function(name, id) { drive_download(as_id(id), path = file.path(tempdir(), name), @@ -115,7 +115,7 @@ drive_ls(as_id("14MGdxHtxe8VGaCu70Y8Pmc0jusZgpym9")) %>% ```{r paged.print = FALSE} read_sf(file.path(tempdir(), - "habitatquarries_v2_20200917.shp")) %>% + "habitatquarries_v2_20200925.shp")) %>% select(polygon_id, unit_id, name, diff --git a/src/generate_habitatquarries/data/habitatquarries.tsv b/src/generate_habitatquarries/data/habitatquarries.tsv index c834a87..dfcc673 100644 --- a/src/generate_habitatquarries/data/habitatquarries.tsv +++ b/src/generate_habitatquarries/data/habitatquarries.tsv @@ -27,9 +27,9 @@ polygon_id unit_id name habitattype extra_reference 33 33 Roosburg - Verbiestberg 8310 NA 39 39 Roosburg - Verbiestberg midden 8310 NA 22 22 Werken van Mathuus 8310 Walschot 2010 -23 23 Werken van Mathuus 8310 Walschot 2010 -7 106 Caestert 8310 Verhoeven 2008 -8 106 Caestert NA Verhoeven 2008 +23 23 Werken van Mathuus - Opkanne IV 8310 Walschot 2010 +7 106 Caestert 8310 Silvertant 2003, Verhoeven 2008 +8 106 Caestert NA Silvertant 2003, Verhoeven 2008 13 100 De Keel 8310 Walschot 2010 14 100 De Keel NA Walschot 2010 45 101 De Keel - Balkon 8310 Walschot 2010 @@ -40,7 +40,7 @@ polygon_id unit_id name habitattype extra_reference 17 103 Muizenberg gh Walschot 2010; Wikipedia 2019 42 107 Roosburg - Drie-dagen-berg 8310 NA 43 107 Roosburg - Drie-dagen-berg NA NA -11 104 Ternaaien beneden 8310 Verhoeven 2008 -12 104 Ternaaien beneden NA Verhoeven 2008 -9 105 Ternaaien boven 8310 Verhoeven 2008 -10 105 Ternaaien boven NA Verhoeven 2008 +11 104 Ternaaien beneden 8310 Silvertant 2003, Verhoeven 2008 +12 104 Ternaaien beneden NA Silvertant 2003, Verhoeven 2008 +9 105 Ternaaien boven 8310 Silvertant 2003, Verhoeven 2008 +10 105 Ternaaien boven NA Silvertant 2003, Verhoeven 2008