Commit 37e31cd9 authored by Simon Legner's avatar Simon Legner

JOSM/wikipedia: fix ISE: Duplicate key - #josm13991

git-svn-id: https://svn.openstreetmap.org/applications/editors/josm/plugins/wikipedia@33066 b9d5c4c9-76e1-0310-9c85-f3177eceb1e4
parent 25f73ac5
...@@ -17,6 +17,7 @@ import java.util.Locale; ...@@ -17,6 +17,7 @@ import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.TreeSet;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
...@@ -193,7 +194,8 @@ public final class WikipediaApp { ...@@ -193,7 +194,8 @@ public final class WikipediaApp {
*/ */
public static Map<String, String> getWikidataForArticles(String wikipediaLang, List<String> articles) { public static Map<String, String> getWikidataForArticles(String wikipediaLang, List<String> articles) {
if (articles.size() > 50) { if (articles.size() > 50) {
return partitionList(articles, 50).stream() final List<String> withoutDuplicates = new ArrayList<>(new TreeSet<>(articles));
return partitionList(withoutDuplicates, 50).stream()
.flatMap(chunk -> getWikidataForArticles(wikipediaLang, chunk).entrySet().stream()) .flatMap(chunk -> getWikidataForArticles(wikipediaLang, chunk).entrySet().stream())
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
} else if (articles.isEmpty()) { } else if (articles.isEmpty()) {
......
...@@ -7,11 +7,16 @@ import org.openstreetmap.josm.testutils.JOSMTestRules; ...@@ -7,11 +7,16 @@ import org.openstreetmap.josm.testutils.JOSMTestRules;
import org.wikipedia.data.WikidataEntry; import org.wikipedia.data.WikidataEntry;
import org.wikipedia.data.WikipediaEntry; import org.wikipedia.data.WikipediaEntry;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import static org.hamcrest.CoreMatchers.hasItem; import static org.hamcrest.CoreMatchers.hasItem;
import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.is;
...@@ -113,6 +118,13 @@ public class WikipediaAppTest { ...@@ -113,6 +118,13 @@ public class WikipediaAppTest {
assertThat(map.size(), is(4)); assertThat(map.size(), is(4));
} }
@Test
public void testTicket13991() throws Exception {
final Map<String, String> map = WikipediaApp.getWikidataForArticles("en",
Stream.iterate("London", x -> x).limit(100).collect(Collectors.toList()));
assertThat(map, is(Collections.singletonMap("London", "Q84")));
}
@Test @Test
public void testGetLabelForWikidata() throws Exception { public void testGetLabelForWikidata() throws Exception {
assertThat(WikipediaApp.getLabelForWikidata("Q1741", Locale.GERMAN), is("Wien")); assertThat(WikipediaApp.getLabelForWikidata("Q1741", Locale.GERMAN), is("Wien"));
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment