Commit a780b928 authored by jhargrave's avatar jhargrave

add xliff2 integration tests

parent 2505be56
package net.sf.okapi.roundtrip.integration;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileNotFoundException;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.List;
import net.sf.okapi.common.ClassUtil;
import net.sf.okapi.common.FileUtil;
import net.sf.okapi.common.LocaleId;
import net.sf.okapi.common.Util;
import net.sf.okapi.common.XMLFileCompare;
import net.sf.okapi.common.integration.IntegrationtestUtils;
import net.sf.okapi.common.integration.RoundTripUtils;
import net.sf.okapi.filters.xliff2.XLIFF2Filter;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class RoundTripXliff2IT
{
private final Logger LOGGER = LoggerFactory.getLogger(getClass());
private XLIFF2Filter xliffFilter;
@Rule
public ErrorCollector errCol = new ErrorCollector();
@Before
public void setUp() throws Exception {
xliffFilter = new XLIFF2Filter();
}
@After
public void tearDown() throws Exception {
xliffFilter.close();
}
@Test
public void xliffSingleFileXliff() throws FileNotFoundException, URISyntaxException {
for (File file : IntegrationtestUtils.getTestFiles("/xliff2/test02.xlf", Arrays.asList(".xlf_debug"))) {
runTest(false, file, "okf_xliff2", null);
runTest(true, file, "okf_xliff2", null);
}
}
@Test
public void xliff2Files() throws FileNotFoundException, URISyntaxException {
// run top level files (without config)
for (File file : IntegrationtestUtils.getTestFiles("/xliff2/test02.xlf", Arrays.asList(".xliff", ".xlf"))) {
runTest(false, file, "okf_xliff2", null);
runTest(true, file, "okf_xliff2", null);
}
// run each subdirectory where we assume there is a custom config)
for(File d : IntegrationtestUtils.getSubDirs(ClassUtil.getResourceParent(IntegrationtestUtils.class, "/xliff2/test02.xlf")))
{
for(File c : IntegrationtestUtils.getConfigFile(d.getPath()))
{
for(File file : IntegrationtestUtils.getTestFiles(d.getPath(), Arrays.asList(".xliff", ".xlf"), true))
{
String configName = Util.getFilename(c.getAbsolutePath(), false);
String customConfigPath = c.getParent();
runTest(false, file, configName, customConfigPath);
runTest(true, file, configName, customConfigPath);
}
}
}
}
private void runTest(boolean segment, File file, String configName, String customConfigPath)
throws FileNotFoundException, URISyntaxException {
String f = file.getName();
LOGGER.info(f);
String root = file.getParent() + File.separator;
String xliff = root + f + ".xliff_extracted";
String original = root + f;
String tkitMerged = root + f + ".tkitMerged";
String mergedRoundTrip = root + f + ".mergedRoundTrip";
List<String> locales = FileUtil.guessLanguages(file.getAbsolutePath());
LocaleId source = LocaleId.fromString(locales.get(0));
LocaleId target = LocaleId.FRENCH;
if (locales.size() >= 2) {
target = LocaleId.fromString(locales.get(1));
}
RoundTripUtils.extract(source, target, original, xliff, configName, customConfigPath, segment);
RoundTripUtils.merge(source, target, false, original, xliff, tkitMerged, configName, customConfigPath);
RoundTripUtils.extract(source, target, tkitMerged, xliff, configName, customConfigPath, segment);
RoundTripUtils.merge(source, target, false, tkitMerged, xliff, mergedRoundTrip, configName, customConfigPath);
XMLFileCompare compare = new XMLFileCompare();
try {
assertTrue("Compare XLIFF2: " + f, compare.compareFilesPerLines(mergedRoundTrip, tkitMerged));
} catch(Throwable e) {
errCol.addError(e);
}
}
}
package net.sf.okapi.simplifier.integration;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileNotFoundException;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.List;
import net.sf.okapi.common.ClassUtil;
import net.sf.okapi.common.FileUtil;
import net.sf.okapi.common.LocaleId;
import net.sf.okapi.common.Util;
import net.sf.okapi.common.XMLFileCompare;
import net.sf.okapi.common.integration.IntegrationtestUtils;
import net.sf.okapi.common.integration.RoundTripUtils;
import net.sf.okapi.filters.xliff2.XLIFF2Filter;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class RoundTripSimplifyXliff2TkitsIT
{
private final Logger LOGGER = LoggerFactory.getLogger(getClass());
private XLIFF2Filter xliffFilter;
@Rule
public ErrorCollector errCol = new ErrorCollector();
@Before
public void setUp() throws Exception {
xliffFilter = new XLIFF2Filter();
}
@After
public void tearDown() throws Exception {
xliffFilter.close();
}
@Test
public void xliff2Files() throws FileNotFoundException, URISyntaxException {
// run top level files (without config)
for (File file : IntegrationtestUtils.getTestFiles("/xliff2/test02.xlf", Arrays.asList(".xliff", ".xlf"))) {
runTest(true, file, "okf_xliff2", null);
}
// run each subdirectory where we assume there is a custom config)
for(File d : IntegrationtestUtils.getSubDirs(ClassUtil.getResourceParent(IntegrationtestUtils.class, "/xliff2/test02.xlf")))
{
for(File c : IntegrationtestUtils.getConfigFile(d.getPath()))
{
for(File file : IntegrationtestUtils.getTestFiles(d.getPath(), Arrays.asList(".xliff", ".xlf"), true))
{
String configName = Util.getFilename(c.getAbsolutePath(), false);
String customConfigPath = c.getParent();
runTest(true, file, configName, customConfigPath);
}
}
}
}
private void runTest(boolean segment, File file, String configName, String customConfigPath)
throws FileNotFoundException, URISyntaxException {
String f = file.getName();
LOGGER.info(f);
String root = file.getParent() + File.separator;
String xliff = root + f + ".simplify_xliff_extracted";
String original = root + f;
String tkitMerged = root + f + ".tkitMerged";
String merged = root + f + ".merged";
List<String> locales = FileUtil.guessLanguages(file.getAbsolutePath());
LocaleId source = LocaleId.fromString(locales.get(0));
LocaleId target = LocaleId.FRENCH;
if (locales.size() >= 2) {
target = LocaleId.fromString(locales.get(1));
}
RoundTripUtils.extract(source, target, original, xliff, configName, customConfigPath, segment, false);
RoundTripUtils.merge(source, target, false, original, xliff, merged, configName, customConfigPath);
RoundTripUtils.extract(source, target, original, xliff, configName, customConfigPath, segment, true);
RoundTripUtils.merge(source, target, false, original, xliff, tkitMerged, configName, customConfigPath);
XMLFileCompare compare = new XMLFileCompare();
try {
assertTrue("Compare XML: " + f, compare.compareFilesPerLines(merged, tkitMerged));
} catch(Throwable e) {
errCol.addError(e);
}
}
}
package net.sf.okapi.tkit.integration;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileNotFoundException;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.List;
import net.sf.okapi.common.ClassUtil;
import net.sf.okapi.common.FileUtil;
import net.sf.okapi.common.LocaleId;
import net.sf.okapi.common.Util;
import net.sf.okapi.common.XMLFileCompare;
import net.sf.okapi.common.integration.IntegrationtestUtils;
import net.sf.okapi.common.integration.RoundTripUtils;
import net.sf.okapi.filters.xliff.XLIFFFilter;
import net.sf.okapi.filters.xliff2.XLIFF2Filter;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class RoundTripXliff2TkitsIT
{
private final Logger LOGGER = LoggerFactory.getLogger(getClass());
private XLIFF2Filter xliffFilter;
@Rule
public ErrorCollector errCol = new ErrorCollector();
@Before
public void setUp() throws Exception {
xliffFilter = new XLIFF2Filter();
}
@After
public void tearDown() throws Exception {
xliffFilter.close();
}
@Test
public void xliff2Files() throws FileNotFoundException, URISyntaxException {
// run top level files (without config)
for (File file : IntegrationtestUtils.getTestFiles("/xliff2/test02.xlf", Arrays.asList(".xliff", ".xlf"))) {
runTest(false, file, "okf_xliff2", null);
runTest(true, file, "okf_xliff2", null);
}
// run each subdirectory where we assume there is a custom config)
for(File d : IntegrationtestUtils.getSubDirs(ClassUtil.getResourceParent(IntegrationtestUtils.class, "/xliff2/test02.xlf")))
{
for(File c : IntegrationtestUtils.getConfigFile(d.getPath()))
{
for(File file : IntegrationtestUtils.getTestFiles(d.getPath(), Arrays.asList(".xliff", ".xlf"), true))
{
String configName = Util.getFilename(c.getAbsolutePath(), false);
String customConfigPath = c.getParent();
runTest(false, file, configName, customConfigPath);
runTest(true, file, configName, customConfigPath);
}
}
}
}
private void runTest(boolean segment, File file, String configName, String customConfigPath)
throws FileNotFoundException, URISyntaxException {
String f = file.getName();
LOGGER.info(f);
String root = file.getParent() + File.separator;
String xliff = root + f + ".xliff_extracted";
String original = root + f;
String tkitMerged = root + f + ".tkitMerged";
String merged = root + f + ".merged";
List<String> locales = FileUtil.guessLanguages(file.getAbsolutePath());
LocaleId source = LocaleId.fromString(locales.get(0));
LocaleId target = LocaleId.FRENCH;
if (locales.size() >= 2) {
target = LocaleId.fromString(locales.get(1));
}
RoundTripUtils.extract(source, target, original, xliff, configName, customConfigPath, segment);
RoundTripUtils.merge(source, target, false, original, xliff, tkitMerged, configName, customConfigPath);
XMLFileCompare compare = new XMLFileCompare();
// many tests fail, but the legacy merger may not work well for bilingual merge
// and some of the differences are its annotations
try {
assertTrue("Compare XML: " + f, compare.compareFilesPerLines(tkitMerged, tkitMerged));
} catch(Throwable e) {
errCol.addError(e);
}
}
}
package net.sf.okapi.xliffcompare.integration;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileNotFoundException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import net.sf.okapi.common.ClassUtil;
import net.sf.okapi.common.Event;
import net.sf.okapi.common.FileUtil;
import net.sf.okapi.common.LocaleId;
import net.sf.okapi.common.Util;
import net.sf.okapi.common.filters.FilterTestDriver;
import net.sf.okapi.common.integration.IntegrationtestUtils;
import net.sf.okapi.common.integration.RoundTripUtils;
import net.sf.okapi.common.resource.RawDocument;
import net.sf.okapi.filters.xliff.XLIFFFilter;
import net.sf.okapi.filters.xliff2.XLIFF2Filter;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class Xliff2XliffCompareIT
{
private final Logger LOGGER = LoggerFactory.getLogger(getClass());
private XLIFF2Filter xliffFilter;
@Rule
public ErrorCollector errCol = new ErrorCollector();
@Before
public void setUp() throws Exception {
xliffFilter = new XLIFF2Filter();
}
@After
public void tearDown() throws Exception {
xliffFilter.close();
}
@Test
public void xliffXliffCompareFiles() throws FileNotFoundException, URISyntaxException {
// run top level files (without config)
for (File file : IntegrationtestUtils.getTestFilesNoRecurse("/xliff2/test01.xlf", Arrays.asList(".xliff", ".xlf"))) {
runTest(true, file, "okf_xliff2", null, null);
}
// run each subdirectory where we assume there is a custom config)
for(File d : IntegrationtestUtils.getSubDirs(ClassUtil.getResourceParent(IntegrationtestUtils.class, "/xliff2/test01.xlf")))
{
for(File c : IntegrationtestUtils.getConfigFile(d.getPath()))
{
for(File file : IntegrationtestUtils.getTestFiles(d.getPath(), Arrays.asList(".xliff", ".xlf"), true))
{
String configName = Util.getFilename(c.getAbsolutePath(), false);
String customConfigPath = c.getParent();
runTest(true, file, configName, customConfigPath, d);
}
}
}
}
private void runTest(boolean segment, File file, String configName, String customConfigPath, File subDir)
throws FileNotFoundException, URISyntaxException {
String f = file.getName();
LOGGER.info(f);
String root = file.getParent() + File.separator;
String xliff = root + f + ".xliff_extracted";
String original = root + f;
String sd = ((subDir == null) ? "" : subDir.getName() + "/");
String xliffPrevious = XliffCompareUtils.CURRENT_XLIFF_ROOT + "xliff/" + sd + f + ".xliff_extracted";
List<String> locales = FileUtil.guessLanguages(file.getAbsolutePath());
LocaleId source = LocaleId.fromString(locales.get(0));
LocaleId target = LocaleId.EMPTY;
if (locales.size() >= 2) {
target = LocaleId.fromString(locales.get(1));
}
RoundTripUtils.extract(source, target, original, xliff, configName, customConfigPath, segment);
XLIFFFilter xf = new XLIFFFilter();
RawDocument ox = new RawDocument(Util.toURI(xliff), StandardCharsets.UTF_8.name(), source, target);
List<Event> oe = IntegrationtestUtils.getTextUnitEvents(xf, ox);
RawDocument px = new RawDocument(Util.toURI(xliffPrevious), StandardCharsets.UTF_8.name(), source, target);
List<Event> pe = IntegrationtestUtils.getTextUnitEvents(xf, px);
try {
assertTrue("Compare Lines: " + f, FilterTestDriver.compareEvents(oe, pe));
} catch(Throwable e) {
errCol.addError(e);
} finally {
ox.close();
px.close();
}
}
}
<?xml version="1.0" encoding="UTF-8"?>
<xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:okp="okapi-framework:xliff-extensions" xmlns:its="http://www.w3.org/2005/11/its" xmlns:itsxlf="http://www.w3.org/ns/its-xliff/" its:version="2.0">
<file original="Graphic Example.psd" source-language="en-us" target-language="fr" datatype="x-undefined" okp:inputEncoding="UTF-8">
<header><tool tool-id="okapi" tool-name="okapi-tests" tool-version="M29" tool-company="okapi"></tool></header>
<body>
<trans-unit id="1">
<source xml:lang="en-us">Quetzal</source>
<seg-source><mrk mid="1" mtype="seg">Quetzal</mrk></seg-source>
<target xml:lang="fr"><mrk mid="1" mtype="seg">Quetzal</mrk></target>
</trans-unit>
<trans-unit id="2">
<source xml:lang="en-us">An application to manipulate and process XLIFF documents</source>
<seg-source><mrk mid="1" mtype="seg">An application to manipulate and process XLIFF documents</mrk></seg-source>
<target xml:lang="fr"><mrk mid="1" mtype="seg">Une demande de manipuler et de traiter des documents XLIFF</mrk></target>
</trans-unit>
<trans-unit id="3">
<source xml:lang="en-us">XLIFF Data Manager</source>
<seg-source><mrk mid="1" mtype="seg">XLIFF Data Manager</mrk></seg-source>
<target xml:lang="fr"><mrk mid="1" mtype="seg">Gestionnaire de données XLIFF</mrk></target>
</trans-unit>
</body>
</file>
</xliff>
<?xml version="1.0"?>
<xliff xmlns="urn:oasis:names:tc:xliff:document:2.0" version="2.0" srcLang="en" trgLang="fr" xmlns:my="myNamespace" xmlns:fs="urn:oasis:names:tc:xliff:fs:2.0" my:attr="value1">
<file id="f1" canResegment="no" translate="no" srcDir="rtl" trgDir="rtl" original="myfile" my:attr="value2" xml:space="preserve">
<skeleton>
Some non-standard data
</skeleton>
<mda:metadata xmlns:mda="urn:oasis:names:tc:xliff:metadata:2.0">
<mda:metaGroup category="someCat">
<mda:meta type="mtype1">info-mtype1</mda:meta>
<mda:metaGroup category="subCat">
<mda:meta type="mtype2">info-mtype2</mda:meta>
</mda:metaGroup>
</mda:metaGroup>
</mda:metadata>
<notes>
<note id="note1" appliesTo="source" priority="2" category="mycat" my:attr="value3">Text of note1</note>
</notes>
<unit id="tu1" canResegment="yes" translate="yes" srcDir="ltr" trgDir="ltr" name="unit1" type="z:typeu" my:attr="value5">
<segment id="1" canResegment="no" state="translated" subState="my:state">
<source>Sample segment.</source>
<target order="3">Exemple de segment.</target>
</segment>
<ignorable>
<source> </source>
</ignorable>
<segment id="2">
<source xml:space="default">Segment's content.</source>
<target xml:space="default" order="1">Contenu du segment.</target>
</segment>
</unit>
<group id="g1" canResegment="yes" translate="yes" srcDir="ltr" trgDir="ltr" name="group1" type="z:typeg" my:attr="value4">
<notes>
<note id="note-g1" appliesTo="source" priority="2" category="mycat" my:attr="value3">Text of note-g1</note>
</notes>
<unit id="tu3">
<segment id="1">
<source>Bolded text</source>
</segment>
</unit>
<unit id="tu3end">
<segment id="1">
<source>Extra stuff</source>
</segment>
</unit>
<unit id="tu2">
<originalData>
<data id="d1">[b [#$tu3] ]</data>
<data id="d2">[/b [#t3end] ]</data>
<data id="d3" dir="rtl">[rtl-data ]</data>
</originalData>
<segment id="s1">
<source><pc id="1" subFlowsStart="tu3" subFlowsEnd="tu3end" dataRefEnd="d2" dataRefStart="d1">special text</pc> and more
<ph id="2" canDelete="no" canReorder="firstNo" canCopy="no" dataRef="d3"/>.</source>
</segment>
</unit>
</group>
</file>
</xliff>
<xliff xmlns="urn:oasis:names:tc:xliff:document:2.0" version="2.0"
srcLang="en-US" trgLang="fr">
<file id="f1" original="Graphic Example.psd">
<skeleton href="Graphic Example.psd.skl"/>
<unit id="1">
<segment>
<source>Quetzal</source>
<target>Quetzal</target>
</segment>
</unit>
<unit id="2">
<segment>
<source>An application to manipulate and process XLIFF documents</source>
<target>Une demande de manipuler et de traiter des documents XLIFF</target>
</segment>
</unit>
<unit id="3">
<segment>
<source>XLIFF Data Manager</source>
<target>Gestionnaire de données XLIFF</target>
</segment>
</unit>
</file>
</xliff>
<xliff xmlns="urn:oasis:names:tc:xliff:document:2.0" version="2.0"
srcLang="en-US" trgLang="fr">
<file id="f1" original="Graphic Example.psd">
<skeleton href="Graphic Example.psd.skl"/>
<unit id="1">
<segment>
<source>Quetzal</source>
<target>Quetzal</target>
</segment>
</unit>
<unit id="2">
<segment>
<source>An application to manipulate and process XLIFF documents</source>
<target>Une demande de manipuler et de traiter des documents XLIFF</target>
</segment>
</unit>
<unit id="3">
<segment>
<source>XLIFF Data Manager</source>
<target>Gestionnaire de données XLIFF</target>
</segment>
</unit>
</file>
</xliff>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment