Commit f9186509 authored by Conor Anderson's avatar Conor Anderson

More steps toward plumbing

parent d68fc4f0
......@@ -4,3 +4,4 @@ deploy.sh
local_settings.R
packrat/lib*/
packrat/src/
.cache
get_bounds <- function(var) {
if (debug_flag) message(sprintf("Getting model bounds for %s", var))
r <- POST(URLencode(paste0(plumber_address, "/bounds?var=", var)),
config = list(add_headers(accept = "application/json")))
stop_for_status(r)
unserialize(content(r))
}
\ No newline at end of file
......@@ -2,8 +2,7 @@ get_gcm_ts <- function(meta) {
if (debug_flag) {
message("get_gcm_ts")
message(sprintf("Reading files from %s on %i cores with a cache date of %s",
file_dir, no_of_cores, cache_ver))
message(sprintf("Requesting files on %i cores", no_of_cores))
}
# Process each NetCDF file
......
......@@ -5,10 +5,10 @@ get_metadata <- function(choices, coords, baseline = NULL,
if (debug_flag) message("get_metadata")
## FIXME: This is the nuclear option.
if (!st_meta$exists("cache_ver") || st_meta$get("cache_ver") != cache_ver) {
if (!cache$exists("cache_ver") || cache$get("cache_ver") != cache_ver) {
message("Metadata cache is invalid. Deleting.")
st_meta$clear()
st_meta$set("cache_ver", cache_ver)
cache$clear()
cache$set("cache_ver", cache_ver)
}
var <- unique(choices$Variable)
......@@ -42,10 +42,10 @@ get_metadata <- function(choices, coords, baseline = NULL,
filter(cases, Scenario == "historical")
}
if (st_meta$exists(gcm_baseline_key)) {
gcm_baseline_files <- st_meta$get(gcm_baseline_key)
known_shorts <- if (st_meta$exists(baseline_shorts_key)) {
st_meta$get(baseline_shorts_key)
if (cache$exists(gcm_baseline_key)) {
gcm_baseline_files <- cache$get(gcm_baseline_key)
known_shorts <- if (cache$exists(baseline_shorts_key)) {
cache$get(baseline_shorts_key)
} else {
NULL
}
......@@ -69,11 +69,11 @@ get_metadata <- function(choices, coords, baseline = NULL,
distinct(Model, Scenario, Ensemble, Start, End, .keep_all = TRUE)
gcm_baseline_files <- bind_rows(gcm_baseline_files, new_hist_files) %>%
distinct(Model, Scenario, Ensemble, Start, End, .keep_all = TRUE)
st_meta$set(key = gcm_baseline_key, value = gcm_baseline_files)
cache$set(key = gcm_baseline_key, value = gcm_baseline_files)
}
short_cases <- anti_join(short_cases, hist_files %>% dplyr::select(Model, Scenario, Ensemble))
if (nrow(short_cases) > 0) {
st_meta$set(key = baseline_shorts_key,
cache$set(key = baseline_shorts_key,
value = bind_rows(known_shorts, short_cases) %>% distinct)
}
}
......@@ -89,10 +89,10 @@ get_metadata <- function(choices, coords, baseline = NULL,
short_cases <- filter(cases, Scenario != "historical")
if (st_meta$exists(gcm_projection_key)) {
gcm_projection_files <- st_meta$get(gcm_projection_key)
known_shorts <- if (st_meta$exists(projection_shorts_key)) {
st_meta$get(projection_shorts_key)
if (cache$exists(gcm_projection_key)) {
gcm_projection_files <- cache$get(gcm_projection_key)
known_shorts <- if (cache$exists(projection_shorts_key)) {
cache$get(projection_shorts_key)
} else {
NULL
}
......@@ -116,11 +116,11 @@ get_metadata <- function(choices, coords, baseline = NULL,
distinct(Model, Scenario, Ensemble, Start, End, .keep_all = TRUE)
gcm_projection_files <- bind_rows(gcm_projection_files, new_proj_files) %>%
distinct(Model, Scenario, Ensemble, Start, End, .keep_all = TRUE)
st_meta$set(key = gcm_projection_key, value = gcm_projection_files)
cache$set(key = gcm_projection_key, value = gcm_projection_files)
}
short_cases <- anti_join(short_cases, proj_files %>% dplyr::select(Model, Scenario, Ensemble))
if (nrow(short_cases) > 0) {
st_meta$set(key = projection_shorts_key,
cache$set(key = projection_shorts_key,
value = bind_rows(known_shorts, short_cases) %>% distinct)
}
}
......@@ -129,18 +129,7 @@ get_metadata <- function(choices, coords, baseline = NULL,
}
if(!is.null(coords)) {
# It is overkill to do this for each variable, but I'll do it this way to be safe.
if (st_meta$exists(paste0("lookup_table_bounds_", var))) {
cached <- TRUE
bounds_tab <- st_meta$get(paste0("lookup_table_bounds_", var))
} else {
cached <- FALSE
bounds_tab <- lookup_table_bounds(choices)
}
if (!cached) st_meta$set(key = paste0("lookup_table_bounds_", var), value = bounds_tab)
rm(cached)
bounds_tab <- get_bounds(var)
# Convert Western longitudes to degrees East
lon <- ifelse(coords$lon <0, 360 + coords$lon, coords$lon)
......
lookup_table_bounds <- function(choices) {
var <- unique(choices$Variable)
## As far as I can tell, the model bounds never change, regardless of scenario, experiment etc. So we just need one file for each
# Note, we could confirm this if it weren't for https://github.com/tidyverse/dplyr/issues/3088
choices <- choices %>% group_by(Model) %>% summarize(Filename = head(Filenames, 1))
get_bounds <- function(file_dir, var, x) {
nc_nc <- nc_open(file.path(file_dir, var, "verified", x))
lon_bnds <- try(ncvar_get(nc_nc, "lon_bnds"), silent = TRUE)
if (inherits(lon_bnds, "try-error")) {
lon_bnds <- ncvar_get(nc_nc, "lon_bounds")
}
lat_bnds <- try(ncvar_get(nc_nc, "lat_bnds"), silent = TRUE)
if (inherits(lat_bnds, "try-error")) {
lat_bnds <- ncvar_get(nc_nc, "lat_bounds")
}
c(Lat_Bot = list(lat_bnds[1,]), Lat_Top = list(lat_bnds[2,]),
Lon_Lef = list(lon_bnds[1,]), Lon_Rig = list(lon_bnds[2,]))
}
bounds <- choices$Filename %>%
map(~get_bounds(file_dir, var, .)) %>%
map_df(~tibble(Lat_Bot = .[1], Lat_Top = .[2], Lon_Lef = .[3], Lon_Rig = .[4]))
bind_cols(choices %>% dplyr::select(-`Filename`), bounds)
}
......@@ -3,30 +3,21 @@
options(shiny.reactlog = FALSE)
debug_flag = FALSE
## Data location
# Conjuntool assumes that you have amassed all necessary NetCDF files form the ESGF
# These should be sorted into folders by variable, e.g. "tas", "tasmin", etc.
# For now these are hard-coded, so you will need to change the relevant line of code to include other variables
# For more info on the ESGF, see https://esgf-node.llnl.gov/projects/esgf-llnl/
file_dir = ""
## Cache configuration
# A custom cache configuration
cache_root = file.path(file_dir, ".cache")
st_avg = storr::storr_rds(file.path(cache_root, "avg-cache"), default_namespace = "avg")
st_point = storr::storr_rds(file.path(cache_root, "point-cache"), default_namespace = "point")
st_meta = storr::storr_rds(file.path(cache_root, "meta-cache"), default_namespace = "meta")
## Google Maps API key
# Conjuntool uses Google Maps to for geocoding locations. Enter your key here.
# If you don' t have a key, get one at https://developers.google.com/maps/documentation/javascript/get-api-key
google_key = ""
## Cache configuration
# The metadata lookups are still cached locally (for now).
cache = storr::storr_rds(".cache", default_namespace = "meta")
cache_ver = "2018-12-06"
## Plumber Address
# There is a plumber api that interacts with the netCDF files.
# If you want to self-host, change the address here.
plumber_address = "https://api.conr.ca/conjuntool/"
## Number of cores to use
# Conjuntool parallelizes some heavy functions. Set the number of cores to use.
no_of_cores = detectCores() - 1
## Cache version
# A code with which to validate cache entries. Entries that do not match this
# version string will be deleted as they are called (useful for breaking changes).
cache_ver = "2018-10-11"
......@@ -13,11 +13,6 @@ Source: CRAN
Version: 1.0.0
Hash: 6abedd7919c4457604c0aa44529a6683
Package: PCICt
Source: CRAN
Version: 0.5-4.1
Hash: da2eb46879cc528ae62461878f099fc1
Package: R.methodsS3
Source: CRAN
Version: 1.7.1
......@@ -45,6 +40,11 @@ Source: CRAN
Version: 1.1-2
Hash: c0d56cd15034f395874c870141870c25
Package: RNetCDF
Source: CRAN
Version: 1.9-1
Hash: b7713cf85e9ec8f323dc2a8a846cbff9
Package: Rcpp
Source: CRAN
Version: 0.12.18
......@@ -88,9 +88,10 @@ Hash: 931f9972deae0f205e1c78a51f33149b
Package: broom
Source: CRAN
Version: 0.5.0
Hash: 9f383b5f4632f82878222399b5fa7bbd
Requires: backports, dplyr, purrr, reshape2, stringr, tibble, tidyr
Version: 0.5.1
Hash: f170bc989c523e039487511e87c1854f
Requires: backports, dplyr, generics, purrr, reshape2, stringr, tibble,
tidyr
Package: classInt
Source: CRAN
......@@ -104,6 +105,11 @@ Version: 1.0.0
Hash: f4239f89feb7ddc65821e4514e9734ae
Requires: assertthat, crayon
Package: clipr
Source: CRAN
Version: 0.4.1
Hash: caf20ae357bfa2ed50e0e7db267f69ce
Package: colorspace
Source: CRAN
Version: 1.3-2
......@@ -142,8 +148,8 @@ Hash: 1f202ebd812ac7192be3739245746a13
Package: doParallel
Source: CRAN
Version: 1.0.11
Hash: cb602c6ef64eced7c8f32b206d4d74b4
Version: 1.0.14
Hash: 7e564b9d723d8648b18457baba2a1383
Requires: foreach, iterators
Package: dplyr
......@@ -181,6 +187,11 @@ Version: 0.1.7
Hash: 1f6b265b0bebb12742515f8cc5d1b119
Requires: Rcpp, withr
Package: generics
Source: CRAN
Version: 0.0.2
Hash: 0a2498d0288de2a2dabf31438562c2b1
Package: ggplot2
Source: CRAN
Version: 3.0.0
......@@ -195,14 +206,14 @@ Hash: 1fbde6dec830370be696eee8ef31c9e4
Package: googlePolylines
Source: CRAN
Version: 0.6.1
Hash: 970bb2b71cdc7e0fa1ec1fff4a3f2be8
Version: 0.7.2
Hash: 73b527570c776579b98ee7ac13cbf16b
Requires: BH, Rcpp
Package: googleway
Source: CRAN
Version: 2.7.0
Hash: 6708b79f500d459bdf5221d573ac14db
Version: 2.7.1
Hash: 9516778a476949813bef72b6d6d54f4e
Requires: curl, googlePolylines, htmltools, htmlwidgets, jpeg, jqr,
jsonlite, magrittr, scales, shiny, viridisLite
......@@ -287,10 +298,6 @@ GithubRepo: later
GithubUsername: r-lib
GithubRef: master
GithubSha1: b87fa73875b1b41782bc88568d03d92d938cb818
RemoteHost: https://api.github.com
RemoteRepo: later
RemoteUsername: r-lib
RemoteSha: b87fa73875b1b41782bc88568d03d92d938cb818
Package: lazyeval
Source: CRAN
......@@ -317,10 +324,10 @@ Hash: bdc4d48c3135e8f3b399536ddf160df4
Package: mapview
Source: CRAN
Version: 2.4.0
Hash: c45b26801a96d120d0f74cf7c34bd26d
Requires: Rcpp, base64enc, brew, gdalUtils, htmltools, htmlwidgets,
leaflet, png, raster, satellite, scales, sf, sp, svglite, uuid,
Version: 2.6.0
Hash: fad1c2c0fba3dca25eb1e7fe046cb778
Requires: Rcpp, base64enc, brew, htmltools, htmlwidgets, leaflet, png,
raster, satellite, scales, sf, sp, stars, svglite, uuid,
viridisLite, webshot
Package: markdown
......@@ -350,11 +357,11 @@ Source: CRAN
Version: 1.16
Hash: 68486da5802565ac2e2ed038caf2b858
Package: ncdf4.helpers
Package: ncmeta
Source: CRAN
Version: 0.3-3
Hash: b9bf7e51da3d44efafef8468b5db3c1f
Requires: PCICt, abind, ncdf4
Version: 0.0.3
Hash: 66d48eb33712e0496b6a4f7b4a2c9b6e
Requires: RNetCDF, dplyr, rlang, tibble
Package: openssl
Source: CRAN
......@@ -384,11 +391,12 @@ Hash: 81a8008a5e7858552503935f1abe48aa
Package: plotly
Source: CRAN
Version: 4.7.1
Hash: 18d2de809bf346363cd4ee7a587d58bf
Version: 4.8.0
Hash: 539126bf8ab38483d27005b4533f3ebe
Requires: RColorBrewer, base64enc, crosstalk, data.table, digest,
dplyr, ggplot2, hexbin, htmltools, htmlwidgets, httr, jsonlite,
lazyeval, magrittr, purrr, scales, tibble, tidyr, viridisLite
lazyeval, magrittr, promises, purrr, rlang, scales, tibble, tidyr,
viridisLite
Package: plyr
Source: CRAN
......@@ -438,9 +446,9 @@ Requires: Rcpp, sp
Package: readr
Source: CRAN
Version: 1.1.1
Hash: 23cf0dbcf5e1ccd3ec5832aaaedc1fdc
Requires: BH, R6, Rcpp, hms, tibble
Version: 1.2.1
Hash: 61195cce9bc923473e4175a18c4a0e60
Requires: BH, R6, Rcpp, clipr, hms, tibble
Package: reshape2
Source: CRAN
......@@ -450,8 +458,8 @@ Requires: Rcpp, plyr, stringr
Package: rgdal
Source: CRAN
Version: 1.3-3
Hash: 2e424c11224d0ea29755fd0f3e45ce00
Version: 1.3-6
Hash: 06f259084b63aa1118ad251d9f651e02
Requires: sp
Package: rlang
......@@ -478,14 +486,14 @@ GithubSha1: d76791529be802ff84bf9a343347335d58e03b1f
Package: sf
Source: CRAN
Version: 0.6-3
Hash: 07417af99b65851b70b73d3273b01f9c
Version: 0.7-1
Hash: 61dddb39135414ea3939a7217ea60b70
Requires: DBI, Rcpp, classInt, magrittr, units
Package: shiny
Source: CRAN
Version: 1.1.0
Hash: ab653b45dba6dc82eef68fb1e18a3b1a
Version: 1.2.0
Hash: 6853126283ff0572d9f370a1520ac831
Requires: R6, crayon, digest, htmltools, httpuv, jsonlite, later, mime,
promises, rlang, sourcetools, xtable
......@@ -510,10 +518,16 @@ Source: CRAN
Version: 0.2.9.0
Hash: 1ff021793ae196b52070c676b02219a5
Package: stars
Source: CRAN
Version: 0.2-0
Hash: 865569ffef797a91b8c4e35bbe1693b2
Requires: abind, classInt, ncdf4, ncmeta, rlang, sf, units
Package: storr
Source: CRAN
Version: 1.2.0
Hash: 65045ba006c05a8e99c941d0218a118d
Version: 1.2.1
Hash: 76ffa40827f4102989daa97896baf696
Requires: R6, digest
Package: stringi
......
......@@ -10,9 +10,6 @@ library("httr")
library("leaflet")
library("lubridate")
library("mapview")
library("ncdf4")
library("ncdf4.helpers")
library("PCICt")
library("plotly")
library("purrr")
library("raster")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment