Commit ae7cc10b authored by Chris Coughlin's avatar Chris Coughlin

Implemented serialization in DatasetOperations; misc. bugfixes

parent 60c53bd5
......@@ -20,10 +20,15 @@ package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
import java.util.Map;
/**
* AbsoluteValueOperation - returns absolute value of a Dataset's elements.
*/
public class AbsoluteValueOperation implements DatasetOperation {
private static final long serialVersionUID = 1L; // try never to change - indicates backwards compatibility is broken
private static final int VERSION = 1; // current implementation version
/**
* Returns a new Dataset comprised of the absolute value of the input's elements.
* @param input input dataset.
......@@ -41,4 +46,17 @@ public class AbsoluteValueOperation implements DatasetOperation {
}
return null;
}
@Override
public long getSerializationVersion() {
return serialVersionUID;
}
@Override
public int getVersion() {
return VERSION;
}
@Override
public void initCurrentVersion(Map<String, Object> objectMap) {}
}
......@@ -20,15 +20,19 @@ package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
import java.util.Map;
/**
* BinarizeOperation - returns a new Dataset with binarized values: values less than the threshold are set to 0,
* values greater than the threshold are set to 1.
*/
public class BinarizeOperation implements DatasetOperation {
private static final long serialVersionUID = 1L; // try never to change - indicates backwards compatibility is broken
private static final int VERSION = 1; // current implementation version
/**
* Threshold for binarization
*/
double threshold;
private double threshold;
/**
* Creates a new BinarizeOperation with the specified threshold.
......@@ -54,16 +58,53 @@ public class BinarizeOperation implements DatasetOperation {
@Override
public Dataset run(Dataset input) {
if (input != null) {
double[] data = input.getData();
for (int i = 0; i < data.length; i++) {
if (data[i] < threshold) {
data[i] = 0;
double[] inputData = input.getData();
double[] binarizedData = new double[inputData.length];
for (int i = 0; i < inputData.length; i++) {
if (inputData[i] < threshold) {
binarizedData[i] = 0;
} else {
data[i] = 1;
binarizedData[i] = 1;
}
}
return new Dataset(data, input.getWidth(), input.getHeight());
return new Dataset(binarizedData, input.getWidth(), input.getHeight());
}
return null;
}
@Override
public long getSerializationVersion() {
return serialVersionUID;
}
@Override
public int getVersion() {
return VERSION;
}
@Override
public Map<String, Object> getObjectMap() {
Map<String, Object> map = DatasetOperation.super.getObjectMap();
map.put("threshold", threshold);
return map;
}
@Override
public void initCurrentVersion(Map<String, Object> objectMap) {
threshold = (double) objectMap.getOrDefault("threshold", threshold);
}
/**
* Threshold for binarization
*/
public double getThreshold() {
return threshold;
}
/**
* Threshold for binarization
*/
public void setThreshold(double threshold) {
this.threshold = threshold;
}
}
......@@ -18,6 +18,8 @@
package com.emphysic.myriad.core.data.ops;
import java.util.Map;
/**
* BlurOperation - blurs a Dataset.
*/
......@@ -50,4 +52,16 @@ public abstract class BlurOperation implements DatasetOperation {
public void setRadius(int radius) {
this.radius = radius;
}
@Override
public Map<String, Object> getObjectMap() {
Map<String, Object> map = DatasetOperation.super.getObjectMap();
map.put("radius", radius);
return map;
}
@Override
public void initCurrentVersion(Map<String, Object> objectMap) {
radius = (int) objectMap.getOrDefault("radius", radius);
}
}
......@@ -21,10 +21,14 @@ package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
import com.emphysic.myriad.core.data.util.DatasetUtils;
import java.util.Map;
/**
* BoxBlur - fast implementation of a box blur based on its separability.
*/
public class BoxBlur extends BlurOperation {
private static final long serialVersionUID = 1L; // try never to change - indicates backwards compatibility is broken
private static final int VERSION = 1; // current implementation version
/**
* Creates a new BoxBlur of the specified rank (aka radius).
......@@ -100,4 +104,14 @@ public class BoxBlur extends BlurOperation {
}
return null;
}
@Override
public long getSerializationVersion() {
return serialVersionUID;
}
@Override
public int getVersion() {
return VERSION;
}
}
......@@ -24,12 +24,18 @@ import com.emphysic.myriad.core.data.ops.math.Stats;
import com.emphysic.myriad.core.data.util.DatasetUtils;
import lombok.extern.slf4j.Slf4j;
import java.lang.reflect.InvocationTargetException;
import java.util.Map;
/**
* CannyOperation - Canny Edge detection
* Created by chris on 8/25/2016.
*/
@Slf4j
public class CannyOperation implements DatasetOperation {
private static final long serialVersionUID = 1L; // try never to change - indicates backwards compatibility is broken
private static final int VERSION = 1; // current implementation version
/**
* Conversion factor for converting radians to degrees
*/
......@@ -317,4 +323,60 @@ public class CannyOperation implements DatasetOperation {
public void setLowerThreshold(double lowerThreshold) {
this.lowerThreshold = lowerThreshold;
}
@Override
public long getSerializationVersion() {
return serialVersionUID;
}
@Override
public int getVersion() {
return VERSION;
}
@Override
public Map<String, Object> getObjectMap() {
Map<String, Object> map = DatasetOperation.super.getObjectMap();
map.put("upper_threshold", upperThreshold);
map.put("lower_threshold", lowerThreshold);
if (blurOperation != null) {
map.put("blurclz", blurOperation.getClass());
map.put("blurmap", blurOperation.getObjectMap());
}
if (gradientOperation != null) {
map.put("gradclz", gradientOperation.getClass());
map.put("gradmap", gradientOperation.getObjectMap());
}
return map;
}
@Override
public void initCurrentVersion(Map<String, Object> objectMap) {
upperThreshold = (double) objectMap.getOrDefault("upper_threshold", upperThreshold);
lowerThreshold = (double) objectMap.getOrDefault("lower_threshold", lowerThreshold);
if (objectMap.containsKey("blurclz")) {
Class<? extends BlurOperation> clz = (Class<? extends BlurOperation>) objectMap.get("blurclz");
try {
blurOperation = clz.getConstructor().newInstance();
if (objectMap.containsKey("blurmap")) {
blurOperation.init((Map<String, Object>) objectMap.get("blurmap"));
}
} catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e) {
log.error("Unable to instantiate blur operation: ", e, " defaulting to standard Gaussian");
blurOperation = new GaussianBlur();
}
}
if (objectMap.containsKey("gradclz")) {
Class<? extends SobelOperation> clz = (Class<? extends SobelOperation>) objectMap.get("gradclz");
try {
gradientOperation = clz.getConstructor().newInstance();
if (objectMap.containsKey("gradmap")) {
gradientOperation.init((Map<String, Object>) objectMap.get("gradmap"));
}
} catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e) {
log.error("Unable to instantiate gradient operation: ", e, " defaulting to standard Sobel");
gradientOperation = new SobelOperation();
}
}
}
}
......@@ -19,14 +19,21 @@
package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
import lombok.extern.slf4j.Slf4j;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* ChainedDatasetOperation - performs multiple DatasetOperations on an input.
*/
@Slf4j
public class ChainedDatasetOperation implements DatasetOperation {
private static final long serialVersionUID = 1L; // try never to change - indicates backwards compatibility is broken
private static final int VERSION = 1; // current implementation version
/**
* List of DatasetOperations to perform on incoming data
*/
......@@ -93,4 +100,46 @@ public class ChainedDatasetOperation implements DatasetOperation {
public List<DatasetOperation> getOps() {
return operations;
}
@Override
public long getSerializationVersion() {
return serialVersionUID;
}
@Override
public int getVersion() {
return VERSION;
}
@Override
public Map<String, Object> getObjectMap() {
Map<String, Object> map = DatasetOperation.super.getObjectMap();
List<Map<String, Object>> opGraphs = new ArrayList<>();
for (DatasetOperation op : operations) {
Map<String, Object> opMap = op.getObjectMap();
opMap.put("opclz", op.getClass());
opGraphs.add(opMap);
}
map.put("operations", opGraphs);
return map;
}
@Override
public void initCurrentVersion(Map<String, Object> objectMap) {
if (objectMap.containsKey("operations")) {
operations = new ArrayList<>();
List<Map<String, Object>> opGraphs = (List<Map<String, Object>>) objectMap.get("operations");
for (Map<String, Object> graph : opGraphs) {
DatasetOperation op;
Class<? extends DatasetOperation> opclz = (Class<? extends DatasetOperation>) graph.get("opclz");
try {
op = opclz.getConstructor().newInstance();
op.init(graph);
operations.add(op);
} catch (IllegalAccessException | InstantiationException | NoSuchMethodException | InvocationTargetException e) {
log.error("Error instantiating DatasetOperation ", opclz, e);
}
}
}
}
}
......@@ -21,10 +21,15 @@ package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
import com.emphysic.myriad.core.data.util.DatasetUtils;
import java.util.Map;
/**
* ConvolutionOperation - performs convolution on a Dataset.
*/
public class ConvolutionOperation implements DatasetOperation {
private static final long serialVersionUID = 1L; // try never to change - indicates backwards compatibility is broken
private static final int VERSION = 1; // current implementation version
/**
* The convolution kernel
*/
......@@ -124,4 +129,26 @@ public class ConvolutionOperation implements DatasetOperation {
}
return output;
}
@Override
public long getSerializationVersion() {
return serialVersionUID;
}
@Override
public int getVersion() {
return VERSION;
}
@Override
public Map<String, Object> getObjectMap() {
Map<String, Object> map = DatasetOperation.super.getObjectMap();
map.put("kernel", kernel);
return map;
}
@Override
public void initCurrentVersion(Map<String, Object> objectMap) {
kernel = (double[][]) objectMap.getOrDefault("kernel", kernel);
}
}
......@@ -19,11 +19,12 @@
package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
import com.emphysic.myriad.core.data.util.ObjectMap;
/**
* DatasetOperation - an operation on Datasets that returns a resultant Dataset.
*/
public interface DatasetOperation {
public interface DatasetOperation extends ObjectMap {
/**
* Runs the operation on an input.
* @param input Dataset on which to operate
......
......@@ -19,12 +19,20 @@
package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
import lombok.extern.slf4j.Slf4j;
import java.lang.reflect.InvocationTargetException;
import java.util.Map;
/**
* DifferenceOfGaussiansOperation - performs the Difference of Gaussians (DOG) operation on an input.
* Primarily used as a Region of Interest (ROI) detector or for feature enhancement.
*/
@Slf4j
public class DifferenceOfGaussiansOperation implements DatasetOperation {
private static final long serialVersionUID = 1L; // try never to change - indicates backwards compatibility is broken
private static final int VERSION = 1; // current implementation version
/**
* First blur operation
*/
......@@ -88,4 +96,46 @@ public class DifferenceOfGaussiansOperation implements DatasetOperation {
}
return null;
}
@Override
public long getSerializationVersion() {
return serialVersionUID;
}
@Override
public int getVersion() {
return VERSION;
}
@Override
public Map<String, Object> getObjectMap() {
Map<String, Object> map = DatasetOperation.super.getObjectMap();
map.put("firstblurclz", firstBlur.getClass());
map.put("secondblurclz", secondBlur.getClass());
map.put("firstblurgraph", firstBlur.getObjectMap());
map.put("secondblurgraph", secondBlur.getObjectMap());
return map;
}
@Override
public void initCurrentVersion(Map<String, Object> objectMap) {
if (objectMap.containsKey("firstblurclz")) {
Class<? extends BlurOperation> clz = (Class<? extends BlurOperation>) objectMap.get("firstblurclz");
try {
firstBlur = clz.getConstructor().newInstance();
firstBlur.init((Map<String, Object>) objectMap.get("firstblurgraph"));
} catch (IllegalAccessException | InstantiationException | NoSuchMethodException | InvocationTargetException e) {
log.error("Error instantiating blur operation ", clz, e);
}
}
if (objectMap.containsKey("secondblurclz")) {
Class<? extends BlurOperation> clz = (Class<? extends BlurOperation>) objectMap.get("secondblurclz");
try {
secondBlur = clz.getConstructor().newInstance();
secondBlur.init((Map<String, Object>) objectMap.get("secondblurgraph"));
} catch (IllegalAccessException | InstantiationException | NoSuchMethodException | InvocationTargetException e) {
log.error("Error instantiating blur operation ", clz, e);
}
}
}
}
......@@ -32,6 +32,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* FeatureScalingOperation - scales features for machine learning by subtracting the mean and dividing by the
......@@ -39,6 +40,9 @@ import java.util.List;
*/
@Slf4j
public class FeatureScalingOperation implements DatasetOperation {
private static final long serialVersionUID = 1L; // try never to change - indicates backwards compatibility is broken
private static final int VERSION = 1; // current implementation version
/**
* Means of the features
*/
......@@ -104,7 +108,7 @@ public class FeatureScalingOperation implements DatasetOperation {
public void fit(List<? extends Dataset> datasets) {
featureMeans = new ArrayList<>();
featureStds = new ArrayList<>();
onlineStats = null;
onlineStats = new OnlineStats();
for (int i = 0; i < datasets.get(0).getSize(); i++) {
// ith feature
double[] features = new double[datasets.size()];
......@@ -178,32 +182,45 @@ public class FeatureScalingOperation implements DatasetOperation {
this.featureStds = featureStds;
}
/**
* Serializes the current instance. Used to persist feature scaling for use on future data.
*
* @param outfile full path and filename for output file
* @throws IOException if an error occurs during I/O
*/
public void save(File outfile) throws IOException {
Kryo kryo = new Kryo();
Output output = new Output(new FileOutputStream(outfile));
kryo.writeObject(output, this);
output.close();
@Override
public long getSerializationVersion() {
return serialVersionUID;
}
/**
* Loads a persisted scaler from storage.
*
* @param infile full path and filename for the saved scaler
* @return new scaler instance
* @throws IOException if an I/O error occurs
*/
public static FeatureScalingOperation load(File infile) throws IOException {
Kryo kryo = new Kryo();
Input input = new Input(new FileInputStream(infile));
FeatureScalingOperation stop = kryo.readObject(input, FeatureScalingOperation.class);
input.close();
return stop;
@Override
public int getVersion() {
return VERSION;
}
@Override
public Map<String, Object> getObjectMap() {
Map<String, Object> map = DatasetOperation.super.getObjectMap();
map.put("feature_means", featureMeans);
map.put("feature_stds", featureStds);
map.put("stats_numpoints", onlineStats.getN());
map.put("stats_mean", onlineStats.getCurrentMean());
map.put("stats_ssmd", onlineStats.getSumSquaredMeanDiff());
return map;
}
@Override
public void initCurrentVersion(Map<String, Object> objectMap) {
if (objectMap.containsKey("feature_means")) {
featureMeans = (List<Double>) objectMap.get("feature_means");
}
if (objectMap.containsKey("feature_stds")) {
featureStds = (List<Double>) objectMap.get("feature_stds");
}
onlineStats = new OnlineStats();
if (objectMap.containsKey("stats_numpoints")) {
onlineStats.setN((Integer) objectMap.get("stats_numpoints"));
}
if (objectMap.containsKey("stats_mean")) {
onlineStats.setCurrentMean((Double) objectMap.get("stats_mean"));
}
if (objectMap.containsKey("stats_ssmd")) {
onlineStats.setSumSquaredMeanDiff((Double) objectMap.get("stats_ssmd"));
}
}
/**
......
......@@ -19,12 +19,19 @@
package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
import lombok.extern.slf4j.Slf4j;
import java.lang.reflect.InvocationTargetException;
import java.util.Map;
/**
* GaussianPyramidOperation - repeated smoothing and subsampling of an input. Primarily
* used to allow other operations to consider input data at multiple scales.
*/
@Slf4j
public class GaussianPyramidOperation implements DatasetOperation {
private static final long serialVersionUID = 1L; // try never to change - indicates backwards compatibility is broken
private static final int VERSION = 1; // current implementation version
private BlurOperation blur;
private int scaleFactor;
private int windowSize;
......@@ -221,4 +228,44 @@ public class GaussianPyramidOperation implements DatasetOperation {
public void setWindowSize(int windowSize) {
this.windowSize = windowSize;
}
@Override
public long getSerializationVersion() {
return serialVersionUID;
}
@Override
public int getVersion() {
return VERSION;
}
@Override
public Map<String, Object> getObjectMap() {
Map<String, Object> map = DatasetOperation.super.getObjectMap();
map.put("scale_factor", scaleFactor);
map.put("window_size", windowSize);
if (blur != null) {
map.put("blurclz", blur.getClass());
map.put("blurmap", blur.getObjectMap());
}
return map;
}
@Override
public void initCurrentVersion(Map<String, Object> objectMap) {
scaleFactor = (int) objectMap.getOrDefault("scale_factor", scaleFactor);
windowSize = (int) objectMap.getOrDefault("window_size", windowSize);
if (objectMap.containsKey("blurclz")) {
Class<? extends BlurOperation> clz = (Class<? extends BlurOperation>) objectMap.get("blurclz");
try {
blur = clz.getConstructor().newInstance();
if (objectMap.containsKey("blurmap")) {
blur.init((Map<String, Object>) objectMap.get("blurmap"));
}
} catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e) {
log.error("Unable to instantiate blur operation: ", e, " defaulting to standard Gaussian");
blur = new GaussianBlur();
}
}
}
}
......@@ -28,6 +28,7 @@ import java.io.File;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* HOGOperation - an implementation of the Histogram of Oriented Gradients (HOG) algorithm as outlined in
......@@ -41,6 +42,8 @@ import java.util.List;
*/
@Slf4j
public class HOGOperation implements DatasetOperation {
private static final long serialVersionUID = 1L; // try never to change - indicates backwards compatibility is broken
private static final int VERSION = 1; // current implementation version
/**
* Number of data points in a cell's radius.
*/
......@@ -272,6 +275,38 @@ public class HOGOperation implements DatasetOperation {
this.MINANGLE = MINANGLE;
}
@Override
public long getSerializationVersion() {
return serialVersionUID;
}
@Override
public int getVersion() {
return VERSION;
}
@Override
public Map<String, Object> getObjectMap() {
Map<String, Object> map = DatasetOperation.super.getObjectMap();
map.put("cell_radius", cellRadius);
map.put("cell_width", cellWidth);
map.put("block_size", blockSize);
map.put("nbins", nbins);
map.put("angle_max", MAXANGLE);
map.put("angle_min", MINANGLE);
return map;
}
@Override
public void initCurrentVersion(Map<String, Object> objectMap) {
cellRadius = (int) objectMap.getOrDefault("cell_radius", cellRadius);
cellWidth = (int) objectMap.getOrDefault("cell_width", cellWidth);
blockSize = (int) objectMap.getOrDefault("block_size", blockSize);
nbins = (int) objectMap.getOrDefault("nbins", nbins);
MAXANGLE = (double) objectMap.getOrDefault("angle_max", MAXANGLE);
MINANGLE = (double) objectMap.getOrDefault("angle_min", MAXANGLE);
}
/*
* Demo of scanning over a larger dataset
*/
......