Commit 99eb0071 authored by jhargrave's avatar jhargrave

update for YAML tests

parent cfaa21ac
......@@ -6,10 +6,13 @@ import java.nio.charset.StandardCharsets;
import java.util.LinkedList;
import java.util.List;
import net.sf.okapi.common.Event;
import net.sf.okapi.common.LocaleId;
import net.sf.okapi.common.Util;
import net.sf.okapi.common.filters.DefaultFilters;
import net.sf.okapi.common.filters.FilterConfigurationMapper;
import net.sf.okapi.common.filters.FilterTestDriver;
import net.sf.okapi.common.filters.IFilter;
import net.sf.okapi.common.filterwriter.XLIFFWriter;
import net.sf.okapi.common.filterwriter.XLIFFWriterParameters;
import net.sf.okapi.common.pipelinedriver.BatchItemContext;
......@@ -125,4 +128,28 @@ public final class RoundTripUtils {
driver.processBatch();
driver.destroy();
}
/**
* Assumes merge has been called to set source and target locales
*/
public static boolean compareOriginalWithTarget(IFilter filter, String originalfile, String targetFile) {
return compareOriginalWithTarget(filter, originalfile, targetFile, true, false, false, false);
}
/**
* Assumes merge has been called to set source and target locales
*/
public static boolean compareOriginalWithTarget(IFilter filter, String originalfile, String targetFile,
boolean includeSkeleton, boolean ignoreSkelWhitespace, boolean ignoreSegmentation, boolean ignoreFragmentWhitespace) {
RawDocument ord = new RawDocument(Util.toURI(originalfile), "UTF-8", LocaleId.ENGLISH, LocaleId.FRENCH);
RawDocument trd = new RawDocument(Util.toURI(targetFile), "UTF-8", LocaleId.ENGLISH, LocaleId.FRENCH);
List<Event> o = IntegrationtestUtils.getEvents(filter, ord);
List<Event> t = IntegrationtestUtils.getEvents(filter, trd);
boolean r = FilterTestDriver.compareEvents(o, t, includeSkeleton, ignoreSkelWhitespace, ignoreSegmentation, ignoreFragmentWhitespace);
ord.close();
trd.close();
o.clear();
t.clear();
return r;
}
}
#v1
extractIsolatedStrings.b=false
extractAllPairs.b=true
exceptions=
useKeyAsName.b=true
useFullKeyPath.b=true
useCodeFinder.b=true
codeFinderRules.count.i=1
codeFinderRules.rule0=</?([A-Z0-9a-z]*)\b[^>]*>
codeFinderRules.sample=&name; <tag></at><tag/> <tag attr='val'> </tag="val">
codeFinderRules.useAllRulesWhenTesting.b=true
subfilter=
#v1
extractIsolatedStrings.b=false
extractAllPairs.b=true
exceptions=
useKeyAsName.b=true
useFullKeyPath.b=true
useCodeFinder.b=true
codeFinderRules.count.i=1
codeFinderRules.rule0=</?([A-Z0-9a-z]*)\b[^>]*>
codeFinderRules.sample=&name; <tag></at><tag/> <tag attr='val'> </tag="val">
codeFinderRules.useAllRulesWhenTesting.b=true
subfilter=
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:util="http://www.springframework.org/schema/util" xmlns:p="http://www.springframework.org/schema/p"
xsi:schemaLocation="
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.0.xsd">
<!-- the most powerful way -->
<bean id="yamlConstructor" class="examples.CustomConstructor" scope="prototype" />
<bean id="yamlRepresenter" class="org.yaml.snakeyaml.representer.Representer" scope="prototype" />
<bean id="yamlOptions" class="org.yaml.snakeyaml.DumperOptions" scope="prototype">
<property name="indent" value="2" />
</bean>
<bean id="snakeYaml" class="org.yaml.snakeyaml.Yaml" scope="prototype">
<constructor-arg ref="yamlConstructor" />
<constructor-arg ref="yamlRepresenter" />
<constructor-arg ref="yamlOptions" />
</bean>
<!-- for a single JavaBean -->
<bean id="beanConstructor" class="org.yaml.snakeyaml.constructor.Constructor" scope="prototype">
<constructor-arg value="org.yaml.snakeyaml.Invoice" />
</bean>
<bean id="javabeanYaml" class="org.yaml.snakeyaml.Yaml" scope="prototype">
<constructor-arg ref="beanConstructor" />
</bean>
<!-- the simplest way -->
<bean id="standardYaml" class="org.yaml.snakeyaml.Yaml" scope="prototype" />
</beans>
\ No newline at end of file
folded_style: >
This entire block of text will be the value of 'folded_style', but this
time, all newlines will be replaced with a single space.
Blank lines, like above, are converted to a newline character.
'More-indented' lines keep their newlines, too -
this text will appear over two lines.
\ No newline at end of file
#v1
extractIsolatedStrings.b=false
extractAllPairs.b=true
exceptions=
useKeyAsName.b=true
useFullKeyPath.b=true
useCodeFinder.b=true
codeFinderRules.count.i=1
codeFinderRules.rule0=</?([A-Z0-9a-z]*)\b[^>]*>
codeFinderRules.sample=&name; <tag></at><tag/> <tag attr='val'> </tag="val">
codeFinderRules.useAllRulesWhenTesting.b=true
subfilter=
#v1
extractIsolatedStrings.b=false
extractAllPairs.b=true
exceptions=
useKeyAsName.b=true
useFullKeyPath.b=true
useCodeFinder.b=true
codeFinderRules.count.i=1
codeFinderRules.rule0=</?([A-Z0-9a-z]*)\b[^>]*>
codeFinderRules.sample=&name; <tag></at><tag/> <tag attr='val'> </tag="val">
codeFinderRules.useAllRulesWhenTesting.b=true
subfilter=
#v1
extractIsolatedStrings.b=false
extractAllPairs.b=true
exceptions=
useKeyAsName.b=true
useFullKeyPath.b=true
useCodeFinder.b=true
codeFinderRules.count.i=1
codeFinderRules.rule0=</?([A-Z0-9a-z]*)\b[^>]*>
codeFinderRules.sample=&name; <tag></at><tag/> <tag attr='val'> </tag="val">
codeFinderRules.useAllRulesWhenTesting.b=true
subfilter=
#v1
extractIsolatedStrings.b=false
extractAllPairs.b=true
exceptions=
useKeyAsName.b=true
useFullKeyPath.b=true
useCodeFinder.b=true
codeFinderRules.count.i=1
codeFinderRules.rule0=</?([A-Z0-9a-z]*)\b[^>]*>
codeFinderRules.sample=&name; <tag></at><tag/> <tag attr='val'> </tag="val">
codeFinderRules.useAllRulesWhenTesting.b=true
subfilter=
......@@ -5,7 +5,6 @@ import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileNotFoundException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import org.junit.After;
......@@ -16,7 +15,6 @@ import org.junit.rules.ErrorCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import net.sf.okapi.common.FileCompare;
import net.sf.okapi.common.LocaleId;
import net.sf.okapi.common.Util;
import net.sf.okapi.common.integration.IntegrationtestUtils;
......@@ -42,6 +40,23 @@ public class RoundTripYamlIT
yamlFilter.close();
}
@Test
public void yamlWithHtmlSubfilter() throws FileNotFoundException, URISyntaxException {
// run each subdirectory where we assume there is a custom config)
for(File d : IntegrationtestUtils.getSubDirs((new File(IntegrationtestUtils.class.getResource("/yaml/subfilter/dummy.txt").toURI())).getParent()))
{
for(File c : IntegrationtestUtils.getConfigFile(d.getPath()))
{
for(File file : IntegrationtestUtils.getTestFiles(d.getPath(), Arrays.asList(".yml", ".yaml"), true))
{
String configName = Util.getFilename(c.getAbsolutePath(), false);
String customConfigPath = c.getParent();
runTest(true, file, configName, customConfigPath);
}
}
}
}
@Test
public void yamlFiles() throws FileNotFoundException, URISyntaxException {
// run top level files (without config)
......@@ -75,17 +90,19 @@ public class RoundTripYamlIT
String original = root + f;
String tkitMerged = root + f + ".tkitMerged";
String mergedRoundTrip = root + f + ".mergedRoundTrip";
RoundTripUtils.extract(LocaleId.ENGLISH, LocaleId.FRENCH, original, xliff, configName, customConfigPath, segment);
RoundTripUtils.merge(LocaleId.ENGLISH, LocaleId.FRENCH, false, original, xliff, tkitMerged, configName, customConfigPath);
RoundTripUtils.extract(LocaleId.ENGLISH, LocaleId.FRENCH, tkitMerged, xliff, configName, customConfigPath, segment);
RoundTripUtils.merge(LocaleId.ENGLISH, LocaleId.FRENCH, false, tkitMerged, xliff, mergedRoundTrip, configName, customConfigPath);
FileCompare compare = new FileCompare();
try {
assertTrue("Compare Lines: " + f, compare.compareFilesPerLines(mergedRoundTrip, tkitMerged, StandardCharsets.UTF_8.name()));
RoundTripUtils.extract(LocaleId.ENGLISH, LocaleId.FRENCH, original, xliff, configName, customConfigPath, segment);
RoundTripUtils.merge(LocaleId.ENGLISH, LocaleId.FRENCH, false, original, xliff, tkitMerged, configName, customConfigPath);
RoundTripUtils.extract(LocaleId.ENGLISH, LocaleId.FRENCH, tkitMerged, xliff, configName, customConfigPath, segment);
RoundTripUtils.merge(LocaleId.ENGLISH, LocaleId.FRENCH, false, tkitMerged, xliff, mergedRoundTrip, configName, customConfigPath);
// can't compare files as lines have been wrapped
assertTrue("Compare original and merged: " + f,
RoundTripUtils.compareOriginalWithTarget(yamlFilter, original, mergedRoundTrip, false, true, false, true));
} catch(Throwable e) {
System.err.println("ERROR+++: " + f);
errCol.addError(e);
}
}
......
......@@ -75,12 +75,12 @@ public class RoundTripYamlTkitsIT
String original = root + f;
String tkitMerged = root + f + ".tkitMerged";
String merged = root + f + ".merged";
RoundTripUtils.extract(LocaleId.ENGLISH, LocaleId.FRENCH, original, xliff, configName, customConfigPath, segment);
RoundTripUtils.merge(LocaleId.ENGLISH, LocaleId.FRENCH, false, original, xliff, tkitMerged, configName, customConfigPath);
RoundTripUtils.merge(LocaleId.ENGLISH, LocaleId.FRENCH, true, original, xliff, merged, configName, customConfigPath);
FileCompare compare = new FileCompare();
try {
FileCompare compare = new FileCompare();
try {
assertTrue("Compare Lines: " + f, compare.compareFilesPerLines(tkitMerged, merged, StandardCharsets.UTF_8.name()));
} catch(Throwable e) {
errCol.addError(e);
......
<?xml version="1.0" encoding="UTF-8"?>
<xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:okp="okapi-framework:xliff-extensions" xmlns:its="http://www.w3.org/2005/11/its" xmlns:itsxlf="http://www.w3.org/ns/its-xliff/" its:version="2.0">
<file original="Test01.yml" source-language="en" target-language="fr" datatype="x-text/x-yaml" okp:inputEncoding="UTF-8">
<header><tool tool-id="okapi" tool-name="okapi-tests" tool-version="M28" tool-company="okapi"></tool></header>
<body>
<trans-unit id="1" resname="config.title" xml:space="preserve">
<source xml:lang="en">My Rails Website</source>
<seg-source><mrk mid="0" mtype="seg">My Rails Website</mrk></seg-source>
<target xml:lang="fr"><mrk mid="0" mtype="seg">My Rails Website</mrk></target>
</trans-unit>
<trans-unit id="2" resname="config.author" xml:space="preserve">
<source xml:lang="en">Santa Claus with é</source>
<seg-source><mrk mid="0" mtype="seg">Santa Claus with é</mrk></seg-source>
<target xml:lang="fr"><mrk mid="0" mtype="seg">Santa Claus with é</mrk></target>
</trans-unit>
<trans-unit id="3" resname="config.email" xml:space="preserve">
<source xml:lang="en">email@company.com</source>
<seg-source><mrk mid="0" mtype="seg">email@company.com</mrk></seg-source>
<target xml:lang="fr"><mrk mid="0" mtype="seg">email@company.com</mrk></target>
</trans-unit>
<trans-unit id="4" resname="config.css_file" xml:space="preserve">
<source xml:lang="en">default.css</source>
<seg-source><mrk mid="0" mtype="seg">default.css</mrk></seg-source>
<target xml:lang="fr"><mrk mid="0" mtype="seg">default.css</mrk></target>
</trans-unit>
<trans-unit id="5" resname="config.test" xml:space="preserve">
<source xml:lang="en">"\</source>
<seg-source><mrk mid="0" mtype="seg">"\</mrk></seg-source>
<target xml:lang="fr"><mrk mid="0" mtype="seg">"\</mrk></target>
</trans-unit>
</body>
</file>
</xliff>
This source diff could not be displayed because it is too large. You can view the blob instead.
<?xml version="1.0" encoding="UTF-8"?>
<xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:okp="okapi-framework:xliff-extensions" xmlns:its="http://www.w3.org/2005/11/its" xmlns:itsxlf="http://www.w3.org/ns/its-xliff/" its:version="2.0">
<file original="en (5).yml" source-language="en" target-language="fr" datatype="x-text/x-yaml" okp:inputEncoding="UTF-8">
<header><tool tool-id="okapi" tool-name="okapi-tests" tool-version="M28" tool-company="okapi"></tool></header>
<body>
</body>
</file>
</xliff>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment