...
 
Commits (4)
......@@ -20,10 +20,10 @@ The CRxREST model is thoroughly documented within the
#### Dictionaries
Drug and product information within CRxREST is obtained from `Dictionary`s.
A dictionary generally corresponds to an external dataset of some form
(for example: the FDA's NDC database). As the data represented by various
datasets may vary, dictionaries may declare which portions of the CRxREST model
they support.
A dictionary generally corresponds to an external dataset of some form (for
example: the FDA's NDC database). As the data represented by various datasets
may vary, dictionaries may declare which portions of the CRxREST model they
support.
The underlying data for a `Dictionary` may come from any source that the
implementer wishes (e.g. a local file, a remote database.)
......@@ -45,7 +45,9 @@ NDC dataset and as an example of a dictionary + provider implementation.
## Configuration
All configuration properties are documented within `src/main/resources/application.properties`. Parameters may be set at run-time like so:
All configuration properties are documented within
`src/main/resources/application.yml`. Parameters may be set at
run-time like so:
`java -jar path/to/app.jar --crxrest.database.pool-size=2`
......@@ -61,10 +63,11 @@ OpenAPI documentation will be generated as part of the build process for
CRxREST. The resulting documentation will be placed within the `api-docs`
directory.
Included in the same directory is an HTML file providing the [ReDoc](https://github.com/Rebilly/ReDoc) viewer. Unfortunately due to flaw in the JSON Schema design, the viewer
will not be able to load the spec from the filesystem. Instead, to view the
documentation locally you will have to launch a web server which will serve
the `api-docs` directory. Yes, really.
Included in the same directory is an HTML file providing the
[ReDoc](https://github.com/Rebilly/ReDoc) viewer. Unfortunately due to flaw
in the JSON Schema design, the viewer will not be able to load the spec from
the filesystem. Instead, to view the documentation locally you will have to
launch a web server which will serve the `api-docs` directory. Yes, really.
## Barcode Search
......@@ -100,8 +103,9 @@ SPI mechanism.
Due to a flaw in `swagger-core`, there appears to be no way to properly
represent an empty body for those API operations that produce no content.
According to the generated API spec, such operations appear to produce an empty object. In reality they do not. Therefore, whenever an API operation is
documented as returning a 204 (No Content) status code, do not expect a
According to the generated API spec, such operations appear to produce an
empty object. In reality they do not. Therefore, whenever an API operation
is documented as returning a 204 (No Content) status code, do not expect a
response body to be present. None will.
......
......@@ -15,8 +15,8 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>1.5.7.RELEASE</version>
<relativePath /> <!-- lookup parent from repository -->
<version>2.0.1.RELEASE</version>
<relativePath />
</parent>
<properties>
......@@ -25,7 +25,6 @@
<!-- dependency version -->
<swagger.version>1.5.16</swagger.version>
<jackson.version>2.9.0</jackson.version>
</properties>
<build>
......@@ -204,7 +203,7 @@
<dependency>
<groupId>org.jooq</groupId>
<artifactId>jooq</artifactId>
<version>3.10.3</version><!--$NO-MVN-MAN-VER$ -->
<version>3.10.7</version><!--$NO-MVN-MAN-VER$ -->
</dependency>
<dependency>
......@@ -220,31 +219,26 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.5</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>${jackson.version}</version><!--$NO-MVN-MAN-VER$ -->
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>${jackson.version}</version><!--$NO-MVN-MAN-VER$ -->
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version><!--$NO-MVN-MAN-VER$ -->
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
<version>${jackson.version}</version><!--$NO-MVN-MAN-VER$ -->
</dependency>
<dependency>
......@@ -276,7 +270,7 @@
<dependency>
<groupId>io.github.lukehutch</groupId>
<artifactId>fast-classpath-scanner</artifactId>
<version>2.0.19</version>
<version>2.21</version>
</dependency>
<dependency>
......@@ -284,7 +278,6 @@
<artifactId>model</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>
......@@ -7,51 +7,92 @@ import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
import javax.validation.constraints.NotNull;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import edu.unc.cscc.crxrest.ConfigurableProvider;
import edu.unc.cscc.crxrest.DBDictionaryProvider;
import edu.unc.cscc.crxrest.DictionaryProvider;
import edu.unc.cscc.crxrest.SimpleDictionaryProvider;
import edu.unc.cscc.crxrest.config.loader.DictionaryLoader;
@Configuration
public class DictionaryConfig
@ConfigurationProperties("crxrest.providers")
public class ProviderConfig
implements InitializingBean
{
private static final Logger LOG =
private static final Logger LOG =
LoggerFactory.getLogger("dictionary-config");
private final List<String> additionalPaths;
@Autowired
public DictionaryConfig(@Value("${crxrest.dictionary.path}") String pathStr)
private final List<String> additionalPaths;
private final Map<String, Properties> providerProperties;
private final Map<String, String> providerConfigs;
private boolean failIfNoneConfigured;
public ProviderConfig()
{
this.additionalPaths = new ArrayList<>();
if (pathStr == null || pathStr.trim().isEmpty())
{
return;
}
String[] paths = pathStr.split("\\Q;\\E");
for (final String path : paths)
{
this.additionalPaths.add(path);
}
this.providerProperties = new HashMap<>();
this.providerConfigs = new HashMap<>();
}
public boolean
getFailIfNoneConfigured()
{
return failIfNoneConfigured;
}
public void
setFailIfNoneConfigured(boolean fail)
{
this.failIfNoneConfigured = fail;
}
public List<String>
getPaths()
{
return additionalPaths;
}
public void
setPaths(List<String> paths)
{
this.additionalPaths.clear();
this.additionalPaths.addAll(paths);
}
@Bean
public Map<String, String>
getConfig()
{
return this.providerConfigs;
}
public void
setConfig(Map<String, String> providerConfigs)
{
this.providerConfigs.clear();
this.providerConfigs.putAll(providerConfigs);
}
@Bean
public DictionaryProviderSource
providerSource(DBConfig dbConfig)
throws IOException
......@@ -87,8 +128,18 @@ public class DictionaryConfig
if (initialized.isEmpty())
{
LOG.error("Failed to find and initialize any dictionary providers. "
+ "This instance will be useless.");
final String baseMsg =
"Failed to find and initialize any dictionary providers.";
if (this.failIfNoneConfigured)
{
RuntimeException ex = new RuntimeException(baseMsg);
LOG.error("initialzation failure", ex);
throw ex;
}
else
{
LOG.error(baseMsg + " This instance will be useless.");
}
}
return new DictionaryProviderSource(initialized);
......@@ -103,9 +154,18 @@ public class DictionaryConfig
* access)
* @return provider, or <code>null</code> if initialization failed
*/
private static final DictionaryProvider
private final DictionaryProvider
initializeProvider(DictionaryProvider provider, DBConfig dbConfig)
{
if (provider instanceof ConfigurableProvider)
{
ConfigurableProvider cp = ((ConfigurableProvider) provider);
Properties config =
this.forProvider(cp.configurationIdentifier());
cp.setProperties(config);
}
if (provider instanceof DBDictionaryProvider)
{
try
......@@ -142,7 +202,7 @@ public class DictionaryConfig
}
private URLClassLoader
createLoader()
createLoader() throws IOException
{
final List<URL> urls = new ArrayList<>();
final List<File> searchPaths =
......@@ -168,7 +228,14 @@ public class DictionaryConfig
try
{
urls.add(f.toURI().toURL());
URL url = f.toURI().toURL();
if (f.isDirectory())
{
LOG.info("Will scan dir: {}", f.getCanonicalPath());
url = new URL(url.toString());
}
urls.add(url);
}
catch (MalformedURLException e)
{
......@@ -177,10 +244,61 @@ public class DictionaryConfig
}
}
if (urls.isEmpty())
{
return null;
}
return new URLClassLoader(urls.toArray(new URL[0]),
this.getClass().getClassLoader());
}
/**
* Get the configuration properties configured for a provider with the given
* identifier.
*
* @param identifier identifier, not <code>null</code>, not blank
* @return properties, not <code>null</code>
*/
@NotNull
public Properties
forProvider(String identifier)
{
if (! this.providerProperties.containsKey(identifier))
{
return new Properties();
}
return new Properties(this.providerProperties.get(identifier));
}
@Override
public void afterPropertiesSet()
throws Exception
{
if (this.providerConfigs == null)
{
return;
}
for (Entry<String, String> e : this.providerConfigs.entrySet())
{
String[] split = StringUtils.split(e.getKey(), '.');
if (split.length < 2)
{
continue;
}
String id = split[0];
String subKey = e.getKey().substring((id + ".").length());
Properties p = this.providerProperties.get(id);
if (p == null)
{
p = new Properties();
this.providerProperties.put(id, p);
}
p.put(subKey, e.getValue());
}
}
}
# ---------------------------------------------------------------------------
# CRxREST properties
# ---------------------------------------------------------------------------
# Path which will be scanned for dictionary provider implementations. May be
# a directory or a JAR. Multiple paths are supported, delimited by a
# ';' (semicolon)
crxrest.dictionary.path=/tmp
# Size of database pool. This value applies to *each* database-backed
# provider. Must be >= 1
crxrest.database.pool-size=4
# Path for database persistence. Leave empty to disable persistence.
crxrest.database.path=/var/tmp/
# ---------------------------------------------------------------------------
# Spring properties
# ---------------------------------------------------------------------------
# prevent Spring from initializing our datasource of its own free will
# (don't touch this)
spring.datasource.initialize=false
# prevent Spring from exposing things via JMX
# (don't touch this)
spring.datasource.jmx-enabled=false
spring.jmx.enabled=false
\ No newline at end of file
# ---------------------------------------------------------------------------
# CRxREST properties
# ---------------------------------------------------------------------------
# Paths which will be scanned for dictionary provider implementations. May be
# a directory or a JAR.
crxrest:
database:
# Size of database pool. This value applies to *each* database-backed
# provider. Must be >= 1
pool-size: 4
# Path for database persistence. Leave empty to disable persistence.
path: /var/tmp/
providers:
# whether instance startup will fail if no providers are found and
# configured. If false the instance will still start even though it
# cannot serve any requests.
fail-if-none-configured: true
config:
# provider specific configuration goes under this key space
# example:
# crx-ndc:
# product-path: /var/tmp/ndc/product.txt
paths:
- /opt/crxrest/providers/
# ---------------------------------------------------------------------------
# Spring properties
# ---------------------------------------------------------------------------
spring:
datasource:
# prevent Spring from initializing our datasource of its own free will
# (don't touch this)
initialization-mode: never
jmx-enabled: false
jmx.enabled: false
\ No newline at end of file
/*-
* ========================LICENSE_START=================================
* model
* %%
* Copyright (C) 2017 - 2018 CSCC - University of North Carolina
* %%
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the CSCC - University of North Carolina nor the names of its contributors
* may be used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
* OF THE POSSIBILITY OF SUCH DAMAGE.
* =========================LICENSE_END==================================
*/
package edu.unc.cscc.crxrest;
import java.util.Properties;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
public interface ConfigurableProvider
extends DictionaryProvider
{
/**
* A unique identifier of the provider, used for instance-managed
* configuration.
*
*
* @return identifier, not <code>null</code>, not blank
*/
@NotNull
@NotBlank
String configurationIdentifier();
/**
* <p>
* Set the properties configured for this provider. The key space of the
* given properties will be any detected properties <i>beneath</i> the
* {@link #configurationIdentifier() configuration identifier}, stripped of
* their prefix(es). So for example, a service with the identifier
* {@code foo-svc} when configured with a property
* {@code foo-svc.initialize=true} would see a property entry of
* {@code initialize} with a value of <code>true</code>.
* </p>
*
* <p>
* This method will only be invoked once, prior to initialization.
* </p>
*
* @param properties properties, not <code>null</code>
*/
void setProperties(Properties properties);
}
......@@ -269,7 +269,7 @@ implements DictionaryService
final Function<NDCProduct, Pair<NDCProduct, Double>> scoreMapper =
product -> Pair.of(product, scoreProduct(product, term));
final Comparator<Pair<?, Double>> pairSort =
(a, b) -> (int) Math.round((b.getRight() - a.getRight()) * 1000);
(a, b) -> Double.compare(b.getRight(), a.getRight());
/* Establish a fallback limit. Since trivially-short strings are likely
* to produce huge numbers of mostly-irrelevant results, we limit our
......@@ -304,27 +304,28 @@ implements DictionaryService
.map(PhoneticHash :: hash)
.map(t -> field(name("phonetic_hash")).containsIgnoreCase(t))
.reduce(DSL.trueCondition(), (a, b) -> a.and(b)));
return builder
.build()
.parallel()
.flatMap(c ->
this.loadProducts(
this.ctx.select(idField)
.from(D_TABLE)
.where(c)
.limit(fbLimit)
.fetch(idField))
.stream()
.map(scoreMapper)
.sorted(pairSort)
.limit(limit)
)
.unordered()
.distinct()
.sorted(pairSort)
.limit(limit)
.map(p -> p.getLeft())
.collect(Collectors.toList());
.build()
.parallel()
.flatMap(c ->
this.loadProducts(
this.ctx.select(idField)
.from(D_TABLE)
.where(c)
.limit(fbLimit)
.fetch(idField))
.stream()
.map(scoreMapper)
.sorted(pairSort)
.limit(limit)
)
.unordered()
.distinct()
.sorted(pairSort)
.map(Pair :: getLeft)
.limit(limit)
.collect(Collectors.toList());
}
@Override
......
......@@ -40,11 +40,14 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
import java.util.Properties;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import javax.sql.DataSource;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import org.jooq.DSLContext;
import org.jooq.SQLDialect;
......@@ -57,27 +60,32 @@ import org.jooq.impl.DefaultDSLContext;
import com.fasterxml.uuid.NoArgGenerator;
import com.fasterxml.uuid.impl.RandomBasedGenerator;
import edu.unc.cscc.crxrest.ConfigurableProvider;
import edu.unc.cscc.crxrest.DBDictionaryProvider;
import edu.unc.cscc.crxrest.DictionaryService;
import edu.unc.cscc.crxrest.Discoverable;
@Discoverable
public class NDCServiceProvider
implements DBDictionaryProvider
implements DBDictionaryProvider, ConfigurableProvider
{
public static final String IDENTIFIER = "FDA-NDC";
public static final String DEFAULT_PATH =
public static final String IDENTIFIER = "FDA-NDC";
public static final String DEFAULT_PATH =
"/usr/local/share/crxrest/ndc/product.txt";
static final Table<?> D_TABLE =
static final Table<?> D_TABLE =
DSL.table(DSL.name("entries"));
static final Table<?> N_TABLE =
static final Table<?> N_TABLE =
DSL.table(DSL.name("non_proprietary_names"));
static final Table<?> S_TABLE =
static final Table<?> S_TABLE =
DSL.table(DSL.name("substances"));
private DictionaryService ndcService;
private NoArgGenerator generator = new RandomBasedGenerator(null);
private final NoArgGenerator uuidGenerator =
new RandomBasedGenerator(null);
private String productFilePath;
private DictionaryService ndcService;
@Override
public String persistenceID()
......@@ -102,16 +110,13 @@ implements DBDictionaryProvider
}
/* see if we have a search path defined */
final String productFilePath =
System.getProperty("crxrest.providers.ndc_dictionary.product_path",
DEFAULT_PATH);
if (productFilePath == null)
if (this.productFilePath == null)
{
throw new IOException("No dictionary file path specified");
}
final File f = new File(productFilePath);
final File f = new File(this.productFilePath);
if (! f.canRead())
{
......@@ -127,7 +132,7 @@ implements DBDictionaryProvider
for (final ProductParser.Entry e : parser)
{
final UUID id = this.generator.generate();
final UUID id = this.uuidGenerator.generate();
DSL.using(config)
.insertInto(D_TABLE)
......@@ -225,5 +230,22 @@ implements DBDictionaryProvider
.getResourceAsStream("edu/unc/cscc/crxrest/providers/ndc/schema.sql");
}
@Override
public @NotNull @NotBlank String
configurationIdentifier()
{
return "crx-ndc";
}
@Override
public void
setProperties(Properties properties)
{
this.productFilePath = properties.getProperty("product-path");
if (this.productFilePath == null)
{
this.productFilePath = DEFAULT_PATH;
}
}
}
......@@ -135,7 +135,7 @@
<dependency>
<groupId>javax.validation</groupId>
<artifactId>validation-api</artifactId>
<version>1.1.0.Final</version>
<version>2.0.1.Final</version>
<scope>provided</scope>
</dependency>
......@@ -174,7 +174,7 @@
<dependency>
<groupId>org.jooq</groupId>
<artifactId>jooq</artifactId>
<version>3.10.3</version>
<version>3.10.7</version>
</dependency>
<dependency>
......