Commit 9a2532bc authored by Cynthia Crowley's avatar Cynthia Crowley

Merge branch 'read-nmme-climatology' into 'master'

Improve reading of netCDF files with degenerate dimensions

See merge request !31
parents 131e3f6a 8b262ce8
Pipeline #82484754 failed with stages
in 25 minutes and 6 seconds
......@@ -55,7 +55,6 @@ read_vars_from_cdf <- function(vardef, vars=as.character(c()), offset=NULL, coun
lats <- ncdf4::ncvar_get(cdf, latname)
lons <- ncdf4::ncvar_get(cdf, lonname)
dlat <- ifelse(length(lats) >=2, abs(lats[2] - lats[1]), 0)
dlon <- ifelse(length(lons) >=2, abs(lons[2] - lons[1]), 0)
......@@ -110,10 +109,9 @@ read_vars_from_cdf <- function(vardef, vars=as.character(c()), offset=NULL, coun
# data we don't care; it all just ends up in a data frame anyway.
if (is.null(offset) & is_spatial) {
# If any dims are constant (and not a single line of lat or single meridian of lon),
# then count those as extra dims
dims_lengths <- sapply(names(cdf$dim), function(x){
dims_lengths <- sapply(real_dims, function(x){
length(cdf$dim[[x]]$vals)
})
potential_degen <- which(dims_lengths == 1)
......@@ -144,16 +142,25 @@ read_vars_from_cdf <- function(vardef, vars=as.character(c()), offset=NULL, coun
if (is.null(offset)) {
d <- ncdf4::ncvar_get(cdf, var_name)
} else {
# Collapse 3D array to 2D array, but don't
# collapse a column (e.g., single meridian of longitude)
# to a vector
collapse <- !is.na(count[3]) && count[3] == 1
d <- ncdf4::ncvar_get(cdf,
var_name,
start=offset,
count=count,
collapse_degen=collapse)
collapse_degen=FALSE)
# Drop any degenerate non-spatial dimensions. Since ncvar_get can't
# distinguish between spatial and non-spatial dimensions, we figure
# it out ourselves and use adrop after the fact.
# (The assumption here is that a spatial dimension would end up with
# a count of -1; i.e., it wasn't passed in through extra_dims or through
# the 'count' argument)
# We can't rely on the position of the degenerate dimension because we
# could end up with T,Y,X just as well as Y,X,T.
dims_to_drop <- which(count == 1)
if (length(dims_to_drop) > 0) {
d <- abind::adrop(d, dims_to_drop)
}
}
if (!is.null(wrap_rows)) {
......
......@@ -194,3 +194,38 @@ test_that('we can use extra_dims arg to read from a N-D array', {
file.remove(fname)
})
test_that("read_vas_from_cdf tolerates degenerate dimensions that are not used by the variables being read", {
# create a file matching NMME climatologies supplied by NOAA. Those have a structure like this:
# dimensions(sizes): lon(360), lat(181), target(9), initial_time(1)
# variables(dimensions): float32 lon(lon),
# float32 lat(lat),
# float32 target(target),
# float32 initial_time(initial_time),
# float32 clim(target,lat,lon)
# When reading the `clim` variable, we need to make sure we ignore the degenerate dimension `initial_time`
fname <- tempfile(fileext='.nc')
dims <- list(
lon = ncdf4::ncdim_def('lon', 'degrees_east', seq(0, 360, 10)),
lat = ncdf4::ncdim_def('lat', 'degrees_north', seq(90, -90, -10)),
target = ncdf4::ncdim_def('target', 'months since 1960-01-01 00:00:00', 272:280),
initial_time = ncdf4::ncdim_def('initial_time', '', 1L, create_dimvar=FALSE)
)
vars <- list(
initial_time = ncdf4::ncvar_def('initial_time', 'months since 1960-01-01 00:00:00', dim=dims$initial_time),
clim = ncdf4::ncvar_def('clim', 'none', dim=dims[rev(c('target', 'lat', 'lon'))])
)
nc <- ncdf4::nc_create(fname, vars)
ncdf4::ncvar_put(nc, vars$initial_time, 272)
ncdf4::nc_close(nc)
vardef <- sprintf('%s::%s', fname, 'clim')
vals <- read_vars(vardef, extra_dims=list(target=274))
file.remove(fname)
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment