Commit ae9e8edb authored by Chris Coughlin's avatar Chris Coughlin

Refactored into multiple modules

parent 380401a5
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright (c) 2016 Emphysic LLC. All rights reserved.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>myriad</artifactId>
<groupId>com.emphysic</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>core</artifactId>
<dependencies>
<dependency>
<groupId>com.github.jai-imageio</groupId>
<artifactId>jai-imageio-core</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>com.github.haifengl</groupId>
<artifactId>smile-core</artifactId>
<version>1.1.0</version>
</dependency>
<dependency>
<groupId>com.github</groupId>
<artifactId>sgdtk</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>com.pixelmed</groupId>
<artifactId>dicomtoolkit</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>com.twelvemonkeys.imageio</groupId>
<artifactId>imageio-tiff</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.mahout</groupId>
<artifactId>mahout-mr</artifactId>
<version>0.10.0</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
package com.emphysic.data.findflaw;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
package com.emphysic.myriad.core.data.findflaw;
import com.emphysic.data.io.Dataset;
import com.emphysic.myriad.core.data.io.Dataset;
import lombok.extern.slf4j.Slf4j;
import org.apache.mahout.classifier.sgd.AdaptiveLogisticRegression;
import org.apache.mahout.classifier.sgd.ElasticBandPrior;
......@@ -10,11 +14,6 @@ import org.apache.mahout.math.Vector;
import java.io.*;
/**
* Meta-learner that trains multiple SGD predictors in parallel and chooses the best model for predictions.
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 7/9/16.
*/
@Slf4j
public class AdaptiveSGDFlawFinder implements MLFlawFinder {
private int numCategories;
......
package com.emphysic.data.findflaw;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
package com.emphysic.myriad.core.data.findflaw;
import com.emphysic.data.io.Dataset;
import com.emphysic.data.util.ExternalProcess;
import com.emphysic.myriad.core.data.io.Dataset;
import com.emphysic.myriad.core.data.util.ExternalProcess;
import lombok.extern.slf4j.Slf4j;
import java.io.*;
import java.util.List;
import java.util.concurrent.TimeUnit;
/**
* ExternalFlawFinder - example of calling external applications for finding NDE flaw signals.
* Writes the current data to the external app's standard input; reads standard output for classification.
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 6/21/16.
*/
@Slf4j
public class ExternalFlawFinder implements FlawFinder {
private ExternalProcess processRunner;
......
package com.emphysic.data.findflaw;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
package com.emphysic.myriad.core.data.findflaw;
import com.emphysic.data.io.Dataset;
import com.emphysic.myriad.core.data.io.Dataset;
import java.util.ArrayList;
import java.util.List;
/**
* Representation of a detected flaw.
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 6/20/16.
*/
public class Flaw {
protected Dataset dataset;
protected List<Integer> origin;
......
package com.emphysic.data.findflaw;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
import com.emphysic.data.io.Dataset;
package com.emphysic.myriad.core.data.findflaw;
import com.emphysic.myriad.core.data.io.Dataset;
/**
* Defines the interface for a class to predict NDE flaw signals
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 6/6/16.
*/
public interface FlawFinder {
/**
* Examine an array of data and report whether it appears to contain a flaw signal
......
package com.emphysic.data.findflaw;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
import com.emphysic.data.io.Dataset;
package com.emphysic.myriad.core.data.findflaw;
import com.emphysic.myriad.core.data.io.Dataset;
/**
* Interface for a machine learning-based Flaw Finder
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 6/6/16.
*/
public interface MLFlawFinder extends FlawFinder {
/**
* Trains the flaw finder on new data
......
package com.emphysic.data.findflaw;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
package com.emphysic.myriad.core.data.findflaw;
import com.emphysic.data.io.Dataset;
import com.emphysic.myriad.core.data.io.Dataset;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
......@@ -12,13 +16,6 @@ import org.apache.mahout.math.Vector;
import java.io.*;
/**
* NDE flaw finding based on the Passive Aggressive algorithm outlined in
* Crammer, Koby, et al. "Online passive-aggressive algorithms."
* Journal of Machine Learning Research 7.Mar (2006): 551-585.
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 7/13/16.
*/
public class PassiveAggressiveFlawFinder implements MLFlawFinder {
private int numCategories = 2;
private int numFeatures;
......
package com.emphysic.data.findflaw;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
package com.emphysic.myriad.core.data.findflaw;
import com.emphysic.data.io.Dataset;
import com.emphysic.myriad.core.data.io.Dataset;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
......@@ -13,11 +17,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* SGDFlawFinder - uses Stochastic Gradient Descent to predict defect signals
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 6/6/16.
*/
public class SGDFlawFinder implements MLFlawFinder {
private Learner learner; // TODO: refactor to use LearnerCreator - need to persist model configs as well
private Model linearModel;
......
package com.emphysic.data.io;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
package com.emphysic.myriad.core.data.io;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.util.Arrays;
/**
* Dataset - representation of input data
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 2016-03-16.
*/
public class Dataset {
protected int width;
......
package com.emphysic.data.io;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
package com.emphysic.myriad.core.data.io;
import com.pixelmed.dicom.DicomException;
import com.pixelmed.display.SourceImage;
......@@ -8,11 +12,6 @@ import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
/**
* DicomDataset - reads data from DICOM / DICONDE files.
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 6/18/16.
*/
@Slf4j
public class DicomDataset extends ImageDataset {
private SourceImage reader;
......
package com.emphysic.data.io;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
package com.emphysic.myriad.core.data.io;
import java.io.File;
import java.io.IOException;
/**
* Abstract extension of a Dataset for reading input files
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 2016-05-24.
*/
public abstract class IODataset extends Dataset {
protected File source;
......
package com.emphysic.data.io;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
package com.emphysic.myriad.core.data.io;
import lombok.extern.slf4j.Slf4j;
......@@ -11,11 +15,6 @@ import java.io.File;
import java.io.IOException;
import java.util.Iterator;
/**
* ImageDataset - reads data from image files
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 2016-03-16.
*/
@Slf4j
public class ImageDataset extends IODataset {
private ImageReader reader;
......
package com.emphysic.data.io;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
package com.emphysic.myriad.core.data.io;
import lombok.extern.slf4j.Slf4j;
......@@ -6,11 +10,6 @@ import java.io.File;
import java.io.IOException;
import java.util.*;
/**
* TextDataset - reads ASCII Separated Variable files e.g. CSV
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 2016-03-20.
*/
@Slf4j
public class TextDataset extends IODataset {
private static final String defaultDelimiter = ",";
......
package com.emphysic.data.ops;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
import com.emphysic.data.io.Dataset;
package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
/**
* Returns the absolute value of data.
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 7/11/16.
*/
public class AbsoluteValueOperation implements DatasetOperation {
@Override
public Dataset run(Dataset input) {
......
package com.emphysic.data.ops;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
import com.emphysic.data.io.Dataset;
package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
/**
* Binarize operation - elements less than the threshold are set to 0, elements greater than or equal to are set to 1.
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 6/20/16.
*/
public class BinarizeOperation implements DatasetOperation {
double threshold;
......
package com.emphysic.data.ops;
/**
* Separable 2D blur operation
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 2016-05-30.
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
package com.emphysic.myriad.core.data.ops;
public abstract class BlurOperation implements DatasetOperation {
protected int radius;
......
package com.emphysic.data.ops;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
import com.emphysic.data.io.Dataset;
import com.emphysic.data.util.DatasetUtils;
package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
import com.emphysic.myriad.core.data.util.DatasetUtils;
/**
* Square blur function
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 2016-05-30.
*/
public class BoxBlur extends BlurOperation {
public BoxBlur(int rank) {
......
package com.emphysic.data.ops;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
package com.emphysic.myriad.core.data.ops;
import com.emphysic.data.io.Dataset;
import com.emphysic.myriad.core.data.io.Dataset;
import java.util.ArrayList;
import java.util.List;
/**
* Run a series of Dataset Operations and return the final result.
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 7/12/16.
*/
public class ChainedDatasetOperation implements DatasetOperation {
private List<DatasetOperation> operations;
......
package com.emphysic.data.ops;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
import com.emphysic.data.io.Dataset;
import com.emphysic.data.util.DatasetUtils;
package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
import com.emphysic.myriad.core.data.util.DatasetUtils;
/**
* Base Convolution operation - convolves with identity kernel
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 6/15/16.
*/
public class ConvolutionOperation implements DatasetOperation {
protected double[][] kernel;
......
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
public interface DatasetOperation {
Dataset run(Dataset input);
}
package com.emphysic.data.ops;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
import com.emphysic.data.io.Dataset;
package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
/**
* Difference of Gaussians (DOG) Regions of Interest detector, used to detect corners, blobs, and regions of interest.
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 6/15/16.
*/
public class DoGROIDetection extends ROIDetection {
BlurOperation firstBlur;
......
package com.emphysic.data.ops;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
package com.emphysic.myriad.core.data.ops;
import com.emphysic.data.io.Dataset;
import com.emphysic.data.ops.math.OnlineStats;
import com.emphysic.data.ops.math.Stats;
import com.emphysic.myriad.core.data.io.Dataset;
import com.emphysic.myriad.core.data.ops.math.OnlineStats;
import com.emphysic.myriad.core.data.ops.math.Stats;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
......@@ -15,11 +19,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* Feature scaling for machine learning - removes feature mean and divides by feature standard deviation.
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 7/7/16.
*/
@Slf4j
public class FeatureScalingOperation implements DatasetOperation {
......
package com.emphysic.data.ops;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
import com.emphysic.data.io.Dataset;
package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
/**
* Two dimensional (square) Gaussian blur approximation based on multiple box blur passes.
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 2016-06-01.
*/
public class GaussianBlur extends BoxBlur {
private int numPasses = 3; // Number of box blur passes (3 passes is ~ 97% of a "real" Gaussian blur)
......
package com.emphysic.data.ops;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
import com.emphysic.data.io.Dataset;
package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
/**
* Gaussian Pyramid operation - decimates and blurs a Dataset.
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 6/13/16.
*/
public class GaussianPyramidOperation implements DatasetOperation {
private BlurOperation blur; // Blur operation
private int scaleFactor; // Decimation factor
......
package com.emphysic.data.ops;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
import com.emphysic.data.io.Dataset;
package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
/**
* Normalizes a signal by dividing each element by its range i.e. Mij = Mij / (max - min).
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 6/20/16.
*/
public class NormalizeSignalOperation implements DatasetOperation {
@Override
public Dataset run(Dataset input) {
......
package com.emphysic.data.ops;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
import com.emphysic.data.io.Dataset;
package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
/**
* Pads Datasets with 0's to have power of two dimensions.
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 6/24/16.
*/
public class PowerOfTwoOperation implements DatasetOperation {
/**
* Returns the next power of two i.e. returns x such that 2^x >= a
......
package com.emphysic.data.ops;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
package com.emphysic.myriad.core.data.ops;
import com.emphysic.data.io.Dataset;
import com.emphysic.data.io.IODataset;
import com.emphysic.data.util.FileSniffer;
import com.emphysic.myriad.core.data.io.Dataset;
import com.emphysic.myriad.core.data.io.IODataset;
import com.emphysic.myriad.core.data.util.FileSniffer;
import java.io.File;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
/**
* Operations performed on Gaussian Pyramids
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 6/27/16.
*/
public class PyramidOperations {
private static final UpscaleOperation upo = new UpscaleOperation();
......
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
package com.emphysic.myriad.core.data.ops;
public abstract class ROIDetection implements DatasetOperation {
}
package com.emphysic.data.ops;
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
import com.emphysic.data.io.Dataset;
import com.emphysic.data.ops.math.Stats;
package com.emphysic.myriad.core.data.ops;
import com.emphysic.myriad.core.data.io.Dataset;
import com.emphysic.myriad.core.data.ops.math.Stats;
/**
* Standardizes a Dataset by subtracting its mean and dividing by its standard deviation.
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 7/7/16.
*/
public class ScalingOperation implements DatasetOperation {
private double mean;
private double std;
......
package com.emphysic.data.ops;
/**
* A variant of the Sobel operator, designed to provide better rotational symmetry. See
* Scharr, Hanno, 2000, Dissertation, Optimal Operators in Digital Image Processing for details.
* Copyright (C) 2016 Emphysic LLC. All rights reserved.
* Created by ccoughlin on 6/15/16.
/*
* Copyright (c) 2016 Emphysic LLC. All rights reserved.
*/
package com.emphysic.myriad.core.data.ops;