Commit bfb5d873 authored by Jim Hargrave's avatar Jim Hargrave

Add integration tests for markdown and tex filters

parent 702812ed
......@@ -363,6 +363,18 @@
<version>${okapi.version}</version>
</dependency>
<dependency>
<groupId>net.sf.okapi.filters</groupId>
<artifactId>okapi-filter-markdown</artifactId>
<version>${okapi.version}</version>
</dependency>
<dependency>
<groupId>net.sf.okapi.filters</groupId>
<artifactId>okapi-filter-tex</artifactId>
<version>${okapi.version}</version>
</dependency>
<dependency>
<groupId>net.sf.okapi.lib</groupId>
<artifactId>okapi-lib-extra</artifactId>
......
package net.sf.okapi.roundtrip.integration;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileNotFoundException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import net.sf.okapi.common.ClassUtil;
import net.sf.okapi.common.FileCompare;
import net.sf.okapi.common.LocaleId;
import net.sf.okapi.common.Util;
import net.sf.okapi.common.integration.IntegrationtestUtils;
import net.sf.okapi.common.integration.RoundTripUtils;
import net.sf.okapi.filters.markdown.MarkdownFilter;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class RoundTripMarkdownIT
{
private final Logger LOGGER = LoggerFactory.getLogger(getClass());
private MarkdownFilter markdownFilter;
@Rule
public ErrorCollector errCol = new ErrorCollector();
@Before
public void setUp() throws Exception {
markdownFilter = new MarkdownFilter();
}
@After
public void tearDown() throws Exception {
markdownFilter.close();
}
@Test
public void markdownFiles() throws FileNotFoundException, URISyntaxException {
// run top level files (without config)
for (File file : IntegrationtestUtils.getTestFiles("/markdown/DirectShape.md", Arrays.asList(".md"))) {
runTest(false, file, "okf_markdown", null);
runTest(true, file, "okf_markdown", null);
}
// run each subdirectory where we assume there is a custom config)
for(File d : IntegrationtestUtils.getSubDirs(ClassUtil.getResourceParent(IntegrationtestUtils.class, "/markdown/DirectShape.md")))
{
for(File c : IntegrationtestUtils.getConfigFile(d.getPath()))
{
for(File file : IntegrationtestUtils.getTestFiles(d.getPath(), Arrays.asList(".md"), true))
{
String configName = Util.getFilename(c.getAbsolutePath(), false);
String customConfigPath = c.getParent();
runTest(false, file, configName, customConfigPath);
runTest(true, file, configName, customConfigPath);
}
}
}
}
private void runTest(boolean segment, File file, String configName, String customConfigPath)
throws FileNotFoundException, URISyntaxException {
String f = file.getName();
LOGGER.info(f);
String root = file.getParent() + File.separator;
String xliff = root + f + ".xliff";
String original = root + f;
String tkitMerged = root + f + ".tkitMerged";
String mergedRoundTrip = root + f + ".mergedRoundTrip";
RoundTripUtils.extract(LocaleId.ENGLISH, LocaleId.FRENCH, original, xliff, configName, customConfigPath, segment);
RoundTripUtils.merge(LocaleId.ENGLISH, LocaleId.FRENCH, false, original, xliff, tkitMerged, configName, customConfigPath);
RoundTripUtils.extract(LocaleId.ENGLISH, LocaleId.FRENCH, tkitMerged, xliff, configName, customConfigPath, segment);
RoundTripUtils.merge(LocaleId.ENGLISH, LocaleId.FRENCH, false, tkitMerged, xliff, mergedRoundTrip, configName, customConfigPath);
FileCompare compare = new FileCompare();
try {
assertTrue("Compare Lines: " + f, compare.compareFilesPerLines(mergedRoundTrip, tkitMerged, StandardCharsets.UTF_8.name()));
} catch(Throwable e) {
}
}
}
package net.sf.okapi.roundtrip.integration;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileNotFoundException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import net.sf.okapi.common.ClassUtil;
import net.sf.okapi.common.FileCompare;
import net.sf.okapi.common.LocaleId;
import net.sf.okapi.common.Util;
import net.sf.okapi.common.integration.IntegrationtestUtils;
import net.sf.okapi.common.integration.RoundTripUtils;
import net.sf.okapi.filters.tex.TEXFilter;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class RoundTripTexIT
{
private final Logger LOGGER = LoggerFactory.getLogger(getClass());
private TEXFilter texFilter;
@Rule
public ErrorCollector errCol = new ErrorCollector();
@Before
public void setUp() throws Exception {
texFilter = new TEXFilter();
}
@After
public void tearDown() throws Exception {
texFilter.close();
}
@Test
public void texFiles() throws FileNotFoundException, URISyntaxException {
// run top level files (without config)
for (File file : IntegrationtestUtils.getTestFiles("/tex/Test01.tex", Arrays.asList(".tex"))) {
runTest(false, file, "okf_tex", null);
runTest(true, file, "okf_tex", null);
}
// run each subdirectory where we assume there is a custom config)
for(File d : IntegrationtestUtils.getSubDirs(ClassUtil.getResourceParent(IntegrationtestUtils.class, "/tex/Test01.tex")))
{
for(File c : IntegrationtestUtils.getConfigFile(d.getPath()))
{
for(File file : IntegrationtestUtils.getTestFiles(d.getPath(), Arrays.asList(".tex"), true))
{
String configName = Util.getFilename(c.getAbsolutePath(), false);
String customConfigPath = c.getParent();
runTest(false, file, configName, customConfigPath);
runTest(true, file, configName, customConfigPath);
}
}
}
}
private void runTest(boolean segment, File file, String configName, String customConfigPath)
throws FileNotFoundException, URISyntaxException {
String f = file.getName();
LOGGER.info(f);
String root = file.getParent() + File.separator;
String xliff = root + f + ".xliff";
String original = root + f;
String tkitMerged = root + f + ".tkitMerged";
String mergedRoundTrip = root + f + ".mergedRoundTrip";
RoundTripUtils.extract(LocaleId.ENGLISH, LocaleId.FRENCH, original, xliff, configName, customConfigPath, segment);
RoundTripUtils.merge(LocaleId.ENGLISH, LocaleId.FRENCH, false, original, xliff, tkitMerged, configName, customConfigPath);
RoundTripUtils.extract(LocaleId.ENGLISH, LocaleId.FRENCH, tkitMerged, xliff, configName, customConfigPath, segment);
RoundTripUtils.merge(LocaleId.ENGLISH, LocaleId.FRENCH, false, tkitMerged, xliff, mergedRoundTrip, configName, customConfigPath);
FileCompare compare = new FileCompare();
try {
assertTrue("Compare Lines: " + f, compare.compareFilesPerLines(mergedRoundTrip, tkitMerged, StandardCharsets.UTF_8.name()));
} catch(Throwable e) {
}
}
}
package net.sf.okapi.tkit.integration;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileNotFoundException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import net.sf.okapi.common.ClassUtil;
import net.sf.okapi.common.FileCompare;
import net.sf.okapi.common.LocaleId;
import net.sf.okapi.common.Util;
import net.sf.okapi.common.integration.IntegrationtestUtils;
import net.sf.okapi.common.integration.RoundTripUtils;
import net.sf.okapi.filters.markdown.MarkdownFilter;
public class RoundTripMarkdownTkitsIT
{
private final Logger LOGGER = LoggerFactory.getLogger(getClass());
private MarkdownFilter markdownFilter;
@Rule
public ErrorCollector errCol = new ErrorCollector();
@Before
public void setUp() throws Exception {
markdownFilter = new MarkdownFilter();
}
@After
public void tearDown() throws Exception {
markdownFilter.close();
}
@Test
public void markdownFiles() throws FileNotFoundException, URISyntaxException {
// run top level files (without config)
for (File file : IntegrationtestUtils.getTestFiles("/markdown/example1.md", Arrays.asList(".md"))) {
runTest(false, file, "okf_markdown", null);
runTest(true, file, "okf_markdown", null);
}
// run each subdirectory where we assume there is a custom config)
for(File d : IntegrationtestUtils.getSubDirs(ClassUtil.getResourceParent(IntegrationtestUtils.class, "/markdown/example1.md")))
{
for(File c : IntegrationtestUtils.getConfigFile(d.getPath()))
{
for(File file : IntegrationtestUtils.getTestFiles(d.getPath(), Arrays.asList(".md"), true))
{
String configName = Util.getFilename(c.getAbsolutePath(), false);
String customConfigPath = c.getParent();
runTest(false, file, configName, customConfigPath);
runTest(true, file, configName, customConfigPath);
}
}
}
}
private void runTest(boolean segment, File file, String configName, String customConfigPath)
throws FileNotFoundException, URISyntaxException {
String f = file.getName();
LOGGER.info(f);
String root = file.getParent() + File.separator;
String xliff = root + f + ".xliff";
String original = root + f;
String tkitMerged = root + f + ".tkitMerged";
RoundTripUtils.extract(LocaleId.ENGLISH, LocaleId.FRENCH, original, xliff, configName, customConfigPath, segment);
RoundTripUtils.merge(LocaleId.ENGLISH, LocaleId.FRENCH, false, original, xliff, tkitMerged, configName, customConfigPath);
FileCompare compare = new FileCompare();
try {
assertTrue("Compare Lines: " + f, compare.compareFilesPerLines(tkitMerged, tkitMerged, StandardCharsets.UTF_8.name()));
} catch(Throwable e) {
errCol.addError(e);
}
}
}
package net.sf.okapi.tkit.integration;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileNotFoundException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import net.sf.okapi.common.ClassUtil;
import net.sf.okapi.common.FileCompare;
import net.sf.okapi.common.LocaleId;
import net.sf.okapi.common.Util;
import net.sf.okapi.common.integration.IntegrationtestUtils;
import net.sf.okapi.common.integration.RoundTripUtils;
import net.sf.okapi.filters.tex.TEXFilter;
public class RoundTripTexTkitsIT
{
private final Logger LOGGER = LoggerFactory.getLogger(getClass());
private TEXFilter texFilter;
@Rule
public ErrorCollector errCol = new ErrorCollector();
@Before
public void setUp() throws Exception {
texFilter = new TEXFilter();
}
@After
public void tearDown() throws Exception {
texFilter.close();
}
@Test
public void texFiles() throws FileNotFoundException, URISyntaxException {
// run top level files (without config)
for (File file : IntegrationtestUtils.getTestFiles("/tex/Test01.tex", Arrays.asList(".tex"))) {
runTest(false, file, "okf_tex", null);
runTest(true, file, "okf_tex", null);
}
// run each subdirectory where we assume there is a custom config)
for(File d : IntegrationtestUtils.getSubDirs(ClassUtil.getResourceParent(IntegrationtestUtils.class, "/tex/Test01.tex")))
{
for(File c : IntegrationtestUtils.getConfigFile(d.getPath()))
{
for(File file : IntegrationtestUtils.getTestFiles(d.getPath(), Arrays.asList(".tex"), true))
{
String configName = Util.getFilename(c.getAbsolutePath(), false);
String customConfigPath = c.getParent();
runTest(false, file, configName, customConfigPath);
runTest(true, file, configName, customConfigPath);
}
}
}
}
private void runTest(boolean segment, File file, String configName, String customConfigPath)
throws FileNotFoundException, URISyntaxException {
String f = file.getName();
LOGGER.info(f);
String root = file.getParent() + File.separator;
String xliff = root + f + ".xliff";
String original = root + f;
String tkitMerged = root + f + ".tkitMerged";
RoundTripUtils.extract(LocaleId.ENGLISH, LocaleId.FRENCH, original, xliff, configName, customConfigPath, segment);
RoundTripUtils.merge(LocaleId.ENGLISH, LocaleId.FRENCH, false, original, xliff, tkitMerged, configName, customConfigPath);
FileCompare compare = new FileCompare();
try {
assertTrue("Compare Lines: " + f, compare.compareFilesPerLines(tkitMerged, tkitMerged, StandardCharsets.UTF_8.name()));
} catch(Throwable e) {
errCol.addError(e);
}
}
}
package net.sf.okapi.xliffcompare.integration;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileNotFoundException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import net.sf.okapi.common.ClassUtil;
import net.sf.okapi.common.Event;
import net.sf.okapi.common.LocaleId;
import net.sf.okapi.common.Util;
import net.sf.okapi.common.filters.FilterTestDriver;
import net.sf.okapi.common.integration.IntegrationtestUtils;
import net.sf.okapi.common.integration.RoundTripUtils;
import net.sf.okapi.common.resource.RawDocument;
import net.sf.okapi.filters.markdown.MarkdownFilter;
import net.sf.okapi.filters.xliff.XLIFFFilter;
public class MarkdownXliffCompareIT
{
private final Logger LOGGER = LoggerFactory.getLogger(getClass());
private MarkdownFilter markdownFilter;
@Rule
public ErrorCollector errCol = new ErrorCollector();
@Before
public void setUp() throws Exception {
markdownFilter = new MarkdownFilter();
}
@After
public void tearDown() throws Exception {
markdownFilter.close();
}
@Test
public void markdownXliffCompareFiles() throws FileNotFoundException, URISyntaxException {
// run top level files (without config)
for (File file : IntegrationtestUtils.getTestFilesNoRecurse("/markdown/example1.md", Arrays.asList(".md"))) {
runTest(true, file, "okf_markdown", null, null);
}
// run each subdirectory where we assume there is a custom config)
for(File d : IntegrationtestUtils.getSubDirs(ClassUtil.getResourceParent(IntegrationtestUtils.class, "/markdown/example1.md")))
{
for(File c : IntegrationtestUtils.getConfigFile(d.getPath()))
{
for(File file : IntegrationtestUtils.getTestFiles(d.getPath(), Arrays.asList(".md"), true))
{
String configName = Util.getFilename(c.getAbsolutePath(), false);
String customConfigPath = c.getParent();
runTest(true, file, configName, customConfigPath, d);
}
}
}
}
private void runTest(boolean segment, File file, String configName, String customConfigPath, File subDir)
throws FileNotFoundException, URISyntaxException {
String f = file.getName();
LOGGER.info(f);
String root = file.getParent() + File.separator;
String xliff = root + f + ".xliff";
String original = root + f;
String sd = ((subDir == null) ? "" : subDir.getName() + "/");
String xliffPrevious = XliffCompareUtils.CURRENT_XLIFF_ROOT + "markdown/" + sd + f + ".xliff";
RoundTripUtils.extract(LocaleId.ENGLISH, LocaleId.FRENCH, original, xliff, configName, customConfigPath, segment);
try (XLIFFFilter xf = new XLIFFFilter();
RawDocument ox = new RawDocument(Util.toURI(xliff), StandardCharsets.UTF_8.name(), LocaleId.ENGLISH, LocaleId.FRENCH);
RawDocument px = new RawDocument(Util.toURI(xliffPrevious), StandardCharsets.UTF_8.name(), LocaleId.ENGLISH, LocaleId.FRENCH)) {
List<Event> oe = IntegrationtestUtils.getTextUnitEvents(xf, ox);
List<Event> pe = IntegrationtestUtils.getTextUnitEvents(xf, px);
assertTrue("Compare Lines: " + f, FilterTestDriver.compareEvents(oe, pe));
} catch(Throwable e) {
errCol.addError(e);
}
}
}
package net.sf.okapi.xliffcompare.integration;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileNotFoundException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import net.sf.okapi.common.ClassUtil;
import net.sf.okapi.common.Event;
import net.sf.okapi.common.LocaleId;
import net.sf.okapi.common.Util;
import net.sf.okapi.common.filters.FilterTestDriver;
import net.sf.okapi.common.integration.IntegrationtestUtils;
import net.sf.okapi.common.integration.RoundTripUtils;
import net.sf.okapi.common.resource.RawDocument;
import net.sf.okapi.filters.tex.TEXFilter;
import net.sf.okapi.filters.xliff.XLIFFFilter;
public class TexXliffCompareIT
{
private final Logger LOGGER = LoggerFactory.getLogger(getClass());
private TEXFilter texFilter;
@Rule
public ErrorCollector errCol = new ErrorCollector();
@Before
public void setUp() throws Exception {
texFilter = new TEXFilter();
}
@After
public void tearDown() throws Exception {
texFilter.close();
}
@Test
public void texXliffCompareFiles() throws FileNotFoundException, URISyntaxException {
// run top level files (without config)
for (File file : IntegrationtestUtils.getTestFilesNoRecurse("/tex/sample1.tex", Arrays.asList(".tex"))) {
runTest(true, file, "okf_tex", null, null);
}
// run each subdirectory where we assume there is a custom config)
for(File d : IntegrationtestUtils.getSubDirs(ClassUtil.getResourceParent(IntegrationtestUtils.class, "/tex/sample1.tex")))
{
for(File c : IntegrationtestUtils.getConfigFile(d.getPath()))
{
for(File file : IntegrationtestUtils.getTestFiles(d.getPath(), Arrays.asList(".tex"), true))
{
String configName = Util.getFilename(c.getAbsolutePath(), false);
String customConfigPath = c.getParent();
runTest(true, file, configName, customConfigPath, d);
}
}
}
}
private void runTest(boolean segment, File file, String configName, String customConfigPath, File subDir)
throws FileNotFoundException, URISyntaxException {
String f = file.getName();
LOGGER.info(f);
String root = file.getParent() + File.separator;
String xliff = root + f + ".xliff";
String original = root + f;
String sd = ((subDir == null) ? "" : subDir.getName() + "/");
String xliffPrevious = XliffCompareUtils.CURRENT_XLIFF_ROOT + "tex/" + sd + f + ".xliff";
RoundTripUtils.extract(LocaleId.ENGLISH, LocaleId.FRENCH, original, xliff, configName, customConfigPath, segment);
try (XLIFFFilter xf = new XLIFFFilter();
RawDocument ox = new RawDocument(Util.toURI(xliff), StandardCharsets.UTF_8.name(), LocaleId.ENGLISH, LocaleId.FRENCH);
RawDocument px = new RawDocument(Util.toURI(xliffPrevious), StandardCharsets.UTF_8.name(), LocaleId.ENGLISH, LocaleId.FRENCH)) {
List<Event> oe = IntegrationtestUtils.getTextUnitEvents(xf, ox);
List<Event> pe = IntegrationtestUtils.getTextUnitEvents(xf, px);
assertTrue("Compare Lines: " + f, FilterTestDriver.compareEvents(oe, pe));
} catch(Throwable e) {
errCol.addError(e);
}
}
}
<?xml version="1.0" encoding="UTF-8"?>
<xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:okp="okapi-framework:xliff-extensions" xmlns:its="http://www.w3.org/2005/11/its" xmlns:itsxlf="http://www.w3.org/ns/its-xliff/" its:version="2.0">
<file original="DirectShape.md" source-language="en" target-language="fr" datatype="x-text/x-markdown" okp:inputEncoding="UTF-8">
<header><tool tool-id="okapi" tool-name="okapi-tests" tool-version="M29" tool-company="okapi"></tool></header>
<body>
<group id="sd1_ssf1" resname="sub-filter:sd1">
</group>
<group id="sd1_ssf2" resname="sub-filter:sd1">
</group>
<group id="sd1_ssf3" resname="sub-filter:sd1">
</group>
<group id="sd1_ssf4" resname="sub-filter:sd1">
</group>
<group id="sd1_ssf5" resname="sub-filter:sd1">
<trans-unit id="sd1_sf5_tu1" resname="sd1_1" restype="x-td" xml:space="preserve">
<source xml:lang="en"></source>
<target xml:lang="fr"></target>
</trans-unit>
</group>
<trans-unit id="tu1" xml:space="preserve">
<source xml:lang="en"><ph id="1" ctype="x-STRONG_EMPHASIS">**</ph>Code Region: Create a DirectShape<ph id="2" ctype="x-STRONG_EMPHASIS">**</ph></source>
<seg-source><mrk mid="0" mtype="seg"><ph id="1" ctype="x-STRONG_EMPHASIS">**</ph>Code Region: Create a DirectShape<ph id="2" ctype="x-STRONG_EMPHASIS">**</ph></mrk></seg-source>
<target xml:lang="fr"><mrk mid="0" mtype="seg"><ph id="1" ctype="x-STRONG_EMPHASIS">**</ph>Code Region: Create a DirectShape<ph id="2" ctype="x-STRONG_EMPHASIS">**</ph></mrk></target>
</trans-unit>
<group id="sd1_ssf6" resname="sub-filter:sd1">
</group>
<group id="sd1_ssf7" resname="sub-filter:sd1">
</group>
<group id="sd1_ssf8" resname="sub-filter:sd1">
</group>
<group id="sd1_ssf9" resname="sub-filter:sd1">