Commit 37e6b4ba authored by shubham's avatar shubham

new build scripts

parent c03ea67d
......@@ -9,6 +9,12 @@ This project contains the different software layers of the Gesall big data platf
### Automated Building
Note the path of the base directory where the gesall-core, gesall-htsjdk, gesall-picard and gesall-libs is present.
#### Update build.xml files
In the ant-build/ folder we need to update the path of the haddop home directory if we are building from scratch.
Update the value of the variable `hadoophome` in `build_clean.xml` and `build_md.xml`
#### Run bash script
After going into the gesall-core dir simply run 'build.sh' with this path:
```
......@@ -57,4 +63,4 @@ Note: Apache Hadoop JAR files which you have added, should be of the same versio
### License
Our code is released under MIT license.
\ No newline at end of file
Our code is released under MIT license.
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<project default="create_run_jar" name="Create Runnable Jar for Project gesall-core">
<!--this file was created by Eclipse Runnable JAR Export Wizard-->
<!--ANT 1.7 is required -->
<!--define folder properties-->
<property name="dir.buildfile" value="."/>
<property name="dir.jarfile" value="${dir.buildfile}"/>
<property name="homedir" value="HOMEDIR" />
<property name="hadoophome" value="/home/shubham/hadoop-2.5.2" />
<path id="classpath">
<pathelement path='${homedir}/gesall-core/src' />
<!-- <fileset dir="${homedir}/gesall-core/src"/> -->
<fileset dir="${hadoophome}/share/hadoop/common" includes="hadoop-common-2.5.2.jar" />
<fileset dir="${hadoophome}/share/hadoop/hdfs" includes="hadoop-hdfs-2.5.2.jar" />
<fileset dir="${hadoophome}/share/hadoop/hdfs/lib" includes="commons-logging-1.1.3.jar" />
<fileset dir="${hadoophome}/share/hadoop/mapreduce" includes="*.jar" />
<fileset dir="${homedir}/gesall-libs/" includes="testng/testng-5.5-jdk15.jar"/>
<fileset dir="${homedir}/gesall-libs/" includes="kryo/kryo-2.24.0.jar"/>
<fileset dir="${homedir}/gesall-libs/" includes="json/json-20140107.jar"/>
<fileset dir="${homedir}/gesall-libs/" includes="jopt/jopt-simple-4.8.jar"/>
<fileset dir="${homedir}/gesall-libs/" includes="guava-18.0/guava-18.0.jar"/>
<fileset dir="${homedir}/gesall-libs/" includes="apache/cmdline/commons-exec-1.2.jar"/>
<fileset dir="${homedir}/gesall-htsjdk/" includes="dist/htsjdk-1.120.jar"/>
<fileset dir="${homedir}/gesall-picard/" includes="dist/picard-1.117.jar"/>
</path>
<target name="create_run_jar">
<javac srcdir="${homedir}/gesall-core/src/program/clean/latest" destdir="${homedir}/gesall-core/bin/program/clean/latest" classpathref="classpath"/>
<jar destfile="${homedir}/gesall-core/dist/program_clean.jar" filesetmanifest="mergewithoutmain">
<manifest>
<attribute name="Main-Class" value="program.clean.latest.CleanMRMain"/>
......
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<project default="create_run_jar" name="Create Runnable Jar for Project gesall-core">
<!--this file was created by Eclipse Runnable JAR Export Wizard-->
<!--ANT 1.7 is required -->
<!--define folder properties-->
<property name="dir.buildfile" value="."/>
<property name="dir.jarfile" value="${dir.buildfile}"/>
<property name="homedir" value="HOMEDIR" />
<target name="create_run_jar">
<!-- <javac srcdir="${homedir}/gesall-core/src/program/clean/latest" destdir="${homedir}/gesall-core/bin/program/clean/latest" classpathref="classpath"/> -->
<jar destfile="${homedir}/gesall-core/dist/program_clean.jar" filesetmanifest="mergewithoutmain">
<manifest>
<attribute name="Main-Class" value="program.clean.latest.CleanMRMain"/>
<attribute name="Class-Path" value="."/>
</manifest>
<fileset dir="${homedir}/gesall-core/bin"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-libs/testng/testng-5.5-jdk15.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-libs/kryo/kryo-2.24.0.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-libs/json/json-20140107.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-libs/jopt/jopt-simple-4.8.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-libs/guava-18.0/guava-18.0.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-libs/apache/cmdline/commons-exec-1.2.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-htsjdk/dist/htsjdk-1.120.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-picard/dist/picard-1.117.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-picard/lib/ant/bcel-5.2.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-htsjdk/lib/commons-jexl-2.1.1.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-htsjdk/lib/commons-logging-1.1.1.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-htsjdk/lib/snappy-java-1.0.3-rc3.jar"/>
</jar>
</target>
</project>
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<project default="create_run_jar" name="Create Runnable Jar for Project gesall-core">
<!--this file was created by Eclipse Runnable JAR Export Wizard-->
<!--ANT 1.7 is required -->
<!--define folder properties-->
<property name="dir.buildfile" value="."/>
<property name="dir.jarfile" value="${dir.buildfile}"/>
<property name="homedir" value="HOMEDIR" />
<path id="classpath">
<!-- <pathelement path='${homedir}/gesall-core/src' /> -->
<fileset dir="${homedir}/gesall-core/src"/>
<fileset dir="${homedir}/gesall-libs/" includes="testng/testng-5.5-jdk15.jar"/>
<fileset dir="${homedir}/gesall-libs/" includes="kryo/kryo-2.24.0.jar"/>
<fileset dir="${homedir}/gesall-libs/" excludes="META-INF/*.SF" includes="json/json-20140107.jar"/>
<fileset dir="${homedir}/gesall-libs/" excludes="META-INF/*.SF" includes="jopt/jopt-simple-4.8.jar"/>
<fileset dir="${homedir}/gesall-libs/" excludes="META-INF/*.SF" includes="guava-18.0/guava-18.0.jar"/>
<fileset dir="${homedir}/gesall-libs/" excludes="META-INF/*.SF" includes="apache/cmdline/commons-exec-1.2.jar"/>
<fileset dir="${homedir}/gesall-htsjdk/" excludes="META-INF/*.SF" includes="dist/htsjdk-1.120.jar"/>
<fileset dir="${homedir}/gesall-picard/" excludes="META-INF/*.SF" includes="dist/picard-1.117.jar"/>
<!-- <fileset excludes="META-INF/*.SF" includes="${homedir}/gesall-picard/lib/ant/bcel-5.2.jar"/>
<fileset excludes="META-INF/*.SF" includes="${homedir}/gesall-htsjdk/lib/commons-jexl-2.1.1.jar"/>
<fileset excludes="META-INF/*.SF" includes="${homedir}/gesall-htsjdk/lib/commons-logging-1.1.1.jar"/>
<fileset excludes="META-INF/*.SF" includes="${homedir}/gesall-htsjdk/lib/snappy-java-1.0.3-rc3.jar"/> -->
</path>
<target name="create_run_jar">
<javac srcdir="${homedir}/gesall-core/src/program/clean/latest" destdir="${homedir}/gesall-core/bin/program/clean/latest" classpathref="classpath"/>
<jar destfile="${homedir}/gesall-core/dist/program_clean.jar" filesetmanifest="mergewithoutmain">
<manifest>
<attribute name="Main-Class" value="program.clean.latest.CleanMRMain"/>
<attribute name="Class-Path" value="."/>
</manifest>
<fileset dir="${homedir}/gesall-core/bin"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-libs/testng/testng-5.5-jdk15.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-libs/kryo/kryo-2.24.0.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-libs/json/json-20140107.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-libs/jopt/jopt-simple-4.8.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-libs/guava-18.0/guava-18.0.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-libs/apache/cmdline/commons-exec-1.2.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-htsjdk/dist/htsjdk-1.120.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-picard/dist/picard-1.117.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-picard/lib/ant/bcel-5.2.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-htsjdk/lib/commons-jexl-2.1.1.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-htsjdk/lib/commons-logging-1.1.1.jar"/>
<zipfileset excludes="META-INF/*.SF" src="${homedir}/gesall-htsjdk/lib/snappy-java-1.0.3-rc3.jar"/>
</jar>
</target>
</project>
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<project default="create_run_jar" name="Create Runnable Jar for Project gesall-core">
<!--this file was created by Eclipse Runnable JAR Export Wizard-->
<!--ANT 1.7 is required -->
<!--define folder properties-->
<property name="dir.buildfile" value="."/>
<property name="dir.jarfile" value="${dir.buildfile}"/>
<property name="homedir" value="HOMEDIR" />
<property name="hadoophome" value="/home/shubham/hadoop-2.5.2" />
<path id="classpath">
<pathelement path='${homedir}/gesall-core/src' />
<!-- <fileset dir="${homedir}/gesall-core/src"/> -->
<fileset dir="${hadoophome}/share/hadoop/common" includes="hadoop-common-2.5.2.jar" />
<fileset dir="${hadoophome}/share/hadoop/hdfs" includes="hadoop-hdfs-2.5.2.jar" />
<fileset dir="${hadoophome}/share/hadoop/hdfs/lib" includes="commons-logging-1.1.3.jar" />
<fileset dir="${hadoophome}/share/hadoop/mapreduce" includes="*.jar" />
<fileset dir="${homedir}/gesall-libs/" includes="testng/testng-5.5-jdk15.jar"/>
<fileset dir="${homedir}/gesall-libs/" includes="kryo/kryo-2.24.0.jar"/>
<fileset dir="${homedir}/gesall-libs/" includes="json/json-20140107.jar"/>
<fileset dir="${homedir}/gesall-libs/" includes="jopt/jopt-simple-4.8.jar"/>
<fileset dir="${homedir}/gesall-libs/" includes="guava-18.0/guava-18.0.jar"/>
<fileset dir="${homedir}/gesall-libs/" includes="apache/cmdline/commons-exec-1.2.jar"/>
<fileset dir="${homedir}/gesall-htsjdk/" includes="dist/htsjdk-1.120.jar"/>
<fileset dir="${homedir}/gesall-picard/" includes="dist/picard-1.117.jar"/>
</path>
<target name="create_run_jar">
<javac srcdir="${homedir}/gesall-core/src/program/clean/latest" destdir="${homedir}/gesall-core/bin/program/clean/latest" classpathref="classpath"/>
<jar destfile="${homedir}/gesall-core/dist/program_md.jar" filesetmanifest="mergewithoutmain">
<manifest>
<attribute name="Main-Class" value="program.md.latest.MarkDuplicatesMain"/>
......
......@@ -34,11 +34,10 @@ sed -i "s@HOMEDIR@$homedir@" $gesallcore/ant-build/build_md.xml
ant -f $gesallcore/ant-build/build_clean.xml
ant -f $gesallcore/ant-build/build_md.xml
cd $gesallcore/build/build_alignment/
jar cvf $gesallcore/dist/program_aligment.jar ./
# replace back with place holder - so that in case of a mistake,
# the same script can be run again
sed -i "s@$homedir@HOMEDIR@" $gesallcore/ant-build/build_clean.xml
sed -i "s@$homedir@HOMEDIR@" $gesallcore/ant-build/build_md.xml
\ No newline at end of file
sed -i "s@$homedir@HOMEDIR@" $gesallcore/ant-build/build_md.xml
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="src" path=""/>
<classpathentry kind="lib" path="/home/shubham/gesall/gesall-picard/dist/picard-1.117.jar"/>
<classpathentry kind="lib" path="/home/shubham/gesall/gesall-htsjdk/dist/htsjdk-1.120.jar"/>
<classpathentry kind="lib" path="/home/shubham/gesall/gesall-libs/testng/testng-5.5-jdk15.jar"/>
<classpathentry kind="lib" path="/home/shubham/gesall/gesall-libs/kryo/kryo-2.24.0.jar"/>
<classpathentry kind="lib" path="/home/shubham/gesall/gesall-libs/json/json-20140107.jar"/>
<classpathentry kind="lib" path="/home/shubham/gesall/gesall-libs/jopt/jopt-simple-4.8.jar"/>
<classpathentry kind="lib" path="/home/shubham/gesall/gesall-libs/guava-18.0/guava-18.0.jar"/>
<classpathentry kind="lib" path="/home/shubham/gesall/gesall-libs/apache/cmdline/commons-exec-1.2.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/hadoop-common-2.5.2.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/hadoop-common-2.5.2-tests.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/hadoop-nfs-2.5.2.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/activation-1.1.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/apacheds-i18n-2.0.0-M15.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/apacheds-kerberos-codec-2.0.0-M15.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/api-asn1-api-1.0.0-M20.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/api-util-1.0.0-M20.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/asm-3.2.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/avro-1.7.4.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/commons-beanutils-1.7.0.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/commons-beanutils-core-1.8.0.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/commons-cli-1.2.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/commons-codec-1.4.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/commons-collections-3.2.1.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/commons-compress-1.4.1.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/commons-configuration-1.6.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/commons-digester-1.8.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/commons-el-1.0.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/commons-httpclient-3.1.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/commons-io-2.4.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/commons-lang-2.6.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/commons-logging-1.1.3.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/commons-math3-3.1.1.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/commons-net-3.1.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/hadoop-annotations-2.5.2.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/hadoop-auth-2.5.2.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/hamcrest-core-1.3.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/httpclient-4.2.5.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/httpcore-4.2.5.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/jackson-core-asl-1.9.13.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/jackson-jaxrs-1.9.13.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/jackson-mapper-asl-1.9.13.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/jackson-xc-1.9.13.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/jasper-compiler-5.5.23.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/jasper-runtime-5.5.23.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/java-xmlbuilder-0.4.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/jaxb-api-2.2.2.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/jaxb-impl-2.2.3-1.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/jersey-core-1.9.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/jersey-json-1.9.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/jersey-server-1.9.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/jets3t-0.9.0.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/jettison-1.1.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/jetty-6.1.26.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/jetty-util-6.1.26.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/jsch-0.1.42.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/jsp-api-2.1.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/jsr305-1.3.9.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/junit-4.11.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/log4j-1.2.17.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/mockito-all-1.8.5.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/netty-3.6.2.Final.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/paranamer-2.3.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/protobuf-java-2.5.0.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/servlet-api-2.5.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/slf4j-api-1.7.5.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/slf4j-log4j12-1.7.5.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/snappy-java-1.0.4.1.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/stax-api-1.0-2.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/xmlenc-0.52.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/xz-1.0.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/common/lib/zookeeper-3.4.6.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/mapreduce/hadoop-mapreduce-client-app-2.5.2.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/mapreduce/hadoop-mapreduce-client-common-2.5.2.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/mapreduce/hadoop-mapreduce-client-core-2.5.2.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/mapreduce/hadoop-mapreduce-client-hs-2.5.2.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/mapreduce/hadoop-mapreduce-client-hs-plugins-2.5.2.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-2.5.2.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-2.5.2-tests.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/mapreduce/hadoop-mapreduce-client-shuffle-2.5.2.jar"/>
<classpathentry kind="lib" path="/home/shubham/hadoop-2.5.2/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.5.2.jar"/>
<classpathentry kind="output" path=""/>
</classpath>
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>src</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>
/BamConstants.class
/BamFileUtils.class
/BgzfUtils.class
/ByteArraySeekableStream.class
/ByteArrayUtils.class
/LogisticWeights.class
/BamDiffComparator.class
/BamDiffMain.class
/BamDiffMapper$MapperRecords.class
/BamDiffMapper.class
/BamDiffReducer$ReducerRecords.class
/BamDiffReducer.class
/BamDiffResources.class
/BamDiffValue.class
/HdfsDupCounter.class
/HdfsDupExtractor.class
/VcfErrorCounter.class
/AllBamMain.class
/AllBamMapper$MapperRecords.class
/AllBamMapper.class
/AllBamReducer$ReducerRecords.class
/AllBamReducer.class
/AllBamMain.class
/AllBamMapper$MapperRecords.class
/AllBamMapper.class
/AllBamReducer$ReducerRecords.class
/AllBamReducer.class
/MarkDuplicatesDisk.class
/MarkDuplicatesDiskMapper$MapperRecords.class
/MarkDuplicatesDiskMapper.class
/MarkDuplicatesDiskReducer$ReducerRecords.class
/MarkDuplicatesDiskReducer.class
/NewMDKey.class
/MdMapMain.class
/MdMapMapper$MapperRecords.class
/MdMapMapper.class
/BamBlockEnumeration.class
/BamBlockWrapper.class
/BamInputFormat.class
/BamRecordEnumeration.class
/BamRecordReader.class
/DummyRecordReader.class
/DupSamCoordComparator.class
/DupSamWritable.class
/FixDupSamWritable.class
/LogicalBlockBamInputFormat.class
/LogicalCompleteBamInputFormat.class
/MemoryBamManager.class
/SamInputStream.class
/SamRecordWrapper.class
/SamRecordWritable.class
/StreamingDiskBamManager.class
/BloomFilesLoader.class
/BloomPathFilter.class
/GenomicPositionKey.class
/CmdLineArgs.class
/PropertyReader.class
/LogicalMain.class
/LogicalMapper$MapperRecords.class
/LogicalMapper.class
/LogicalPlacementPolicy.class
/LogicalPlacementPolicyTest.class
/MapredLogicalTextInputFormat.class
/IPartitioner.class
/KeyRangePartitioner.class
/KeyValuePair.class
/ReadNameGroupByPartitioner.class
/SamHdfsInputFormat.class
/SamHdfsRecordReader.class
/BamPathFilter.class
/FSMethods.class
/LogPrinter.class
/MarkDuplicatesShuffleKey.class
/RangePartitionInputFormat.class
/RangePartitionKey.class
/RangePartitionMain.class
/RangePartitionMapper$MapperRecords.class
/RangePartitionMapper.class
/RangePartitionRecordReader.class
/RangePartitionReducer$ReducerRecords.class
/RangePartitionReducer.class
/SpanKey.class
/SpanMaintainer.class
/JustBytesIdentifierResolver.class
/JustBytesInputFormat$JustBytesRecordReader.class
/JustBytesInputFormat.class
/JustBytesInputWriter.class
/JustBytesOutputFormat$JustBytesRecordWriter.class
/JustBytesOutputFormat.class
/JustBytesOutputReader.class
/JustBytesWritable.class
/CleanMRMain.class
/CleanMapper$MapperRecords.class
/CleanMapper.class
/CleanReducer$ReducerRecords.class
/CleanReducer.class
......@@ -24,7 +24,7 @@ import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import com.google.common.base.Stopwatch;
//import com.google.common.base.Stopwatch;
import picard.sam.AddOrReplaceReadGroups;
import picard.sam.CleanSam;
......@@ -77,7 +77,7 @@ public class CleanMapper extends
public void cleanup(Context context) throws IOException,
InterruptedException {
Stopwatch timer = Stopwatch.createStarted();
//Stopwatch timer = Stopwatch.createStarted();
SAMFileReader addReplInput = samEnum.getSAMFileReader(bamBlocks);
/*
......@@ -110,21 +110,21 @@ public class CleanMapper extends
}
samFileReader.close();
timer.stop();
LogPrinter.printSecondsElaped(log, "CleanMapper",
timer.elapsed(TimeUnit.SECONDS));
//timer.stop();
//LogPrinter.printSecondsElaped(log, "CleanMapper",
// timer.elapsed(TimeUnit.SECONDS));
}
private byte[] runAddOrReplaceReadGroups(SAMFileReader input)
throws IOException {
ByteArrayOutputStream outputWriter = new ByteArrayOutputStream();
Stopwatch subTimer = Stopwatch.createStarted();
//Stopwatch subTimer = Stopwatch.createStarted();
new AddOrReplaceReadGroups().instanceMainWithoutExit(addRplArgs, input,
outputWriter);
subTimer.stop();
LogPrinter.printSecondsElaped(log, "AddOrReplaceReadGroups",
subTimer.elapsed(TimeUnit.SECONDS));
//subTimer.stop();
//LogPrinter.printSecondsElaped(log, "AddOrReplaceReadGroups",
// subTimer.elapsed(TimeUnit.SECONDS));
outputWriter.flush();
byte[] output = outputWriter.toByteArray();
......@@ -134,12 +134,12 @@ public class CleanMapper extends
private byte[] runCleanSam(SAMFileReader input) throws IOException {
ByteArrayOutputStream outputWriter = new ByteArrayOutputStream();
Stopwatch subTimer = Stopwatch.createStarted();
//Stopwatch subTimer = Stopwatch.createStarted();
new CleanSam().instanceMainWithoutExit(cleanSamArgs, input,
outputWriter);
subTimer.stop();
LogPrinter.printSecondsElaped(log, "CleanSam",
subTimer.elapsed(TimeUnit.SECONDS));
//subTimer.stop();
//LogPrinter.printSecondsElaped(log, "CleanSam",
// subTimer.elapsed(TimeUnit.SECONDS));
outputWriter.flush();
byte[] output = outputWriter.toByteArray();
......
......@@ -6,6 +6,8 @@ import hdfs.config.CmdLineArgs;
import hdfs.config.PropertyReader;
import hdfs.utils.LogPrinter;
import htsjdk.samtools.SAMFileReader;
import htsjdk.samtools.SAMFileHeader;
import htsjdk.samtools.SAMFileHeader.SortOrder;
import htsjdk.samtools.SAMFileWriter;
import htsjdk.samtools.SAMFileWriterFactory;
import htsjdk.samtools.SAMReadGroupRecord;
......@@ -31,7 +33,7 @@ import org.apache.hadoop.mapreduce.Reducer;
import picard.sam.FixMateInformation;
import com.google.common.base.Stopwatch;
//import com.google.common.base.Stopwatch;
import com.google.common.hash.BloomFilter;
@SuppressWarnings("deprecation")
......@@ -73,15 +75,15 @@ public class CleanReducer extends
String[] fixMateInfoArgs;
// time
Stopwatch reducerTimer;
Stopwatch fixMateTimer;
//Stopwatch reducerTimer;
//Stopwatch fixMateTimer;
BloomFilter<GenomicPositionKey> bloomFilter;
@Override
public void setup(Context context) throws IOException {
reducerTimer = Stopwatch.createUnstarted();
fixMateTimer = Stopwatch.createUnstarted();
//reducerTimer = Stopwatch.createUnstarted();
//fixMateTimer = Stopwatch.createUnstarted();
Configuration conf = context.getConfiguration();
MAX_RECORDS_FILE = conf.getInt("numRecordsInOutputFile", -1);
......@@ -181,16 +183,16 @@ public class CleanReducer extends
log.info("[CPRO] [ReduceTask] "
+ context.getTaskAttemptID().getTaskID().getId() + "-"
+ context.getTaskAttemptID().getId());
LogPrinter.printSecondsElaped(log, "FixMateInformation",
fixMateTimer.elapsed(TimeUnit.SECONDS));
LogPrinter.printSecondsElaped(log, "CleanQuickReducer",
reducerTimer.elapsed(TimeUnit.SECONDS));
//LogPrinter.printSecondsElaped(log, "FixMateInformation",
// fixMateTimer.elapsed(TimeUnit.SECONDS));
//LogPrinter.printSecondsElaped(log, "CleanQuickReducer",
// reducerTimer.elapsed(TimeUnit.SECONDS));
}
private void doWork(Context context) throws IOException,
InterruptedException {
reducerTimer.start();
//reducerTimer.start();
SAMFileReader fixMateInput = inMemoryBam.getSamFileReader();
byte[] fixMateOutput = runFixMateInfo(fixMateInput);
......@@ -213,9 +215,12 @@ public class CleanReducer extends
Path path = new Path(outputBamName);
reducerOutput = fs.create(path);
SAMFileHeader header = fixMateOutputReader.getFileHeader();
header.setSortOrder(SortOrder.unsorted);
reducerWriter = new SAMFileWriterFactory().makeBAMWriter(
fixMateOutputReader.getFileHeader(), true, reducerOutput);
header, true, reducerOutput);
recordsInFile = 0;
++fileCount;
......@@ -235,7 +240,7 @@ public class CleanReducer extends
recordsInFile += recordsInMemory;
cleanUp();
reducerTimer.stop();
//reducerTimer.stop();
}
private void writeToBloomFilter(SAMRecord record, Context context)
......@@ -261,10 +266,10 @@ public class CleanReducer extends
private byte[] runFixMateInfo(SAMFileReader input) throws IOException {
ByteArrayOutputStream outputWriter = new ByteArrayOutputStream();
fixMateTimer.start();
//fixMateTimer.start();
new FixMateInformation().instanceMainWithoutExit(fixMateInfoArgs,
input, outputWriter);
fixMateTimer.stop();
//fixMateTimer.stop();
outputWriter.flush();
byte[] output = outputWriter.toByteArray();
return output;
......
/CleanMainWithRangePartitioning.class
/CleanMapper$MapperRecords.class
/CleanMapper.class
/CleanReducer$ReducerRecords.class
/CleanReducer.class
/MarkDuplicatesMain.class
/MarkDuplicatesMapper$MapperRecords.class
/MarkDuplicatesMapper.class
/MarkDuplicatesReducer$ReducerRecords.class
/MarkDuplicatesReducer.class
/NewMDKey.class
......@@ -23,7 +23,7 @@ import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import com.google.common.base.Stopwatch;
//import com.google.common.base.Stopwatch;
import com.google.common.collect.Iterators;
import com.google.common.collect.PeekingIterator;
import com.google.common.hash.BloomFilter;
......@@ -48,16 +48,17 @@ public class MarkDuplicatesMapper extends Mapper<Text, Text, MarkDuplicatesShuff
@Override
public void setup(Context context) throws IOException {
Stopwatch bloomTimer = Stopwatch.createStarted();
//Stopwatch bloomTimer = Stopwatch.createStarted();
Configuration conf = context.getConfiguration();
String inputPath = conf.get("InputDir");
BloomFilesLoader bloomFilesLoader = new BloomFilesLoader(conf,
inputPath);
bloomFilter = bloomFilesLoader.getCombinedBloomFilter();
// bloomFilters = bloomFilesLoader.getBloomFilterList();
bloomTimer.stop();
LogPrinter.printSecondsElaped(log, "LoadBloomFilter",
bloomTimer.elapsed(TimeUnit.SECONDS));
//bloomTimer.stop();
//LogPrinter.printSecondsElaped(log, "LoadBloomFilter",
// bloomTimer.elapsed(TimeUnit.SECONDS));
/*
* String bloomFilePath = conf.get("BloomFilePath"); FileSystem fs =
......@@ -82,7 +83,7 @@ public class MarkDuplicatesMapper extends Mapper<Text, Text, MarkDuplicatesShuff
public void doWork(Context context) throws IOException,
InterruptedException {
Stopwatch timer = Stopwatch.createStarted();
//Stopwatch timer = Stopwatch.createStarted();
Configuration conf = context.getConfiguration();
FileSystem fs = FileSystem.get(conf);
Path path = new Path(bamFilePath);
......@@ -165,9 +166,10 @@ public class MarkDuplicatesMapper extends Mapper<Text, Text, MarkDuplicatesShuff
}
reader.close();
timer.stop();
LogPrinter.printSecondsElaped(log, "MarkDupMapper",
timer.elapsed(TimeUnit.SECONDS));
//timer.stop();
//LogPrinter.printSecondsElaped(log, "MarkDupMapper",
// timer.elapsed(TimeUnit.SECONDS));
}
private void writeFragmentRecord(Context context, SAMRecord record)
......
......@@ -33,7 +33,7 @@ import org.apache.hadoop.mapreduce.Reducer;
import picard.sam.MarkDuplicates;
import picard.sam.SortSam;
import com.google.common.base.Stopwatch;
//import com.google.common.base.Stopwatch;
@SuppressWarnings("deprecation")