Commit ab54c73c authored by Florian Schäfer's avatar Florian Schäfer

Simplify WikipediaApp.getLabelForWikidata() by using ListUtil.processInBatches()

parent 05e8b25a
......@@ -18,12 +18,12 @@ import java.util.Map;
import java.util.Objects;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collector;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
......@@ -37,8 +37,13 @@ import org.openstreetmap.josm.tools.Pair;
import org.openstreetmap.josm.tools.Utils;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.wikipedia.api.wikidata_action.ApiQueryClient;
import org.wikipedia.api.wikidata_action.WikidataActionApiUrl;
import org.wikipedia.api.wikidata_action.json.CheckEntityExistsResult;
import org.wikipedia.api.wikidata_action.json.SerializationSchema;
import org.wikipedia.data.WikidataEntry;
import org.wikipedia.data.WikipediaEntry;
import org.wikipedia.tools.ListUtil;
import org.wikipedia.tools.RegexUtil;
import org.wikipedia.tools.XPath;
......@@ -394,47 +399,43 @@ public final class WikipediaApp {
}
static List<WikidataEntry> getLabelForWikidata(List<? extends WikipediaEntry> entries, Locale locale, String ... preferredLanguage) {
if (entries.size() > 50) {
return partitionList(entries, 50).stream()
.flatMap(chunk -> getLabelForWikidata(chunk, locale, preferredLanguage).stream())
.collect(Collectors.toList());
} else if (entries.isEmpty()) {
return Collections.emptyList();
final Collection<String> languages = new ArrayList<>();
if (locale != null) {
languages.add(getMediawikiLocale(locale));
languages.add(getMediawikiLocale(new Locale(locale.getLanguage())));
}
try {
final String url = "https://www.wikidata.org/w/api.php" +
languages.addAll(Arrays.asList(preferredLanguage));
languages.add("en");
languages.add(null);
final List<WikidataEntry> result = new ArrayList<>(entries.size());
ListUtil.processInBatches(entries, 50, batch -> {
try {
final String url = "https://www.wikidata.org/w/api.php" +
"?action=wbgetentities" +
"&props=labels|descriptions" +
"&ids=" + entries.stream().map(x -> x.article).collect(Collectors.joining("|")) +
"&format=xml";
final Collection<String> languages = new ArrayList<>();
if (locale != null) {
languages.add(getMediawikiLocale(locale));
languages.add(getMediawikiLocale(new Locale(locale.getLanguage())));
}
languages.addAll(Arrays.asList(preferredLanguage));
languages.add("en");
languages.add(null);
final List<WikidataEntry> r = new ArrayList<>(entries.size());
try (final InputStream in = connect(url).getContent()) {
final Document xml = newDocumentBuilder().parse(in);
for (final WikipediaEntry entry : entries) {
final Node entity = X_PATH.evaluateNode("//entity[@id='" + entry.article + "']", xml);
if (entity == null) {
continue;
}
r.add(new WikidataEntry(
try (InputStream in = connect(url).getContent()) {
final Document xml = newDocumentBuilder().parse(in);
for (final WikipediaEntry entry : entries) {
final Node entity = X_PATH.evaluateNode("//entity[@id='" + entry.article + "']", xml);
if (entity == null) {
continue;
}
result.add(new WikidataEntry(
entry.article,
getFirstField(languages, "label", entity),
entry.coordinate,
getFirstField(languages, "description", entity)
));
));
}
}
} catch (Exception ex) {
throw new RuntimeException(ex);
}
return r;
} catch (Exception ex) {
throw new RuntimeException(ex);
}
});
return result;
}
private static String getFirstField(Collection<String> languages, String field, Node entity) {
......
......@@ -14,7 +14,6 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import javax.swing.AbstractAction;
import javax.swing.DefaultListCellRenderer;
import javax.swing.DefaultListModel;
......@@ -51,6 +50,7 @@ import org.openstreetmap.josm.tools.OpenBrowser;
import org.wikipedia.WikipediaApp;
import org.wikipedia.actions.FetchWikidataAction;
import org.wikipedia.data.WikipediaEntry;
import org.wikipedia.tools.ListUtil;
public class WikipediaToggleDialog extends ToggleDialog implements ActiveLayerChangeListener, DataSetListenerAdapter.Listener {
......@@ -187,8 +187,8 @@ public class WikipediaToggleDialog extends ToggleDialog implements ActiveLayerCh
final List<WikipediaEntry> entries = getEntries();
entries.sort(null);
publish(entries.toArray(new WikipediaEntry[entries.size()]));
WikipediaApp.partitionList(entries, 20).forEach(chunk -> {
WikipediaApp.forLanguage(chunk.get(0).lang).updateWIWOSMStatus(chunk);
ListUtil.processInBatches(entries, 20, batch -> {
WikipediaApp.forLanguage(batch.get(0).lang).updateWIWOSMStatus(batch);
list.repaint();
});
return null;
......
......@@ -10,6 +10,8 @@ public final class ListUtil {
// Private constructor to avoid instantiation
}
private static final BiConsumer<Integer, Integer> EMPTY_BICONSUMER = (a, b) -> {};
/**
* Splits the given list {@code fullList} into batches of a size of {@code maxBatchSize} or less and each batch is
* then consumed by the given {@link Consumer} {@code processBatch}.
......@@ -27,5 +29,9 @@ public final class ListUtil {
}
}
public static <T> void processInBatches(final List<T> fullList, int maxBatchSize, final Consumer<List<T>> processBatch) {
processInBatches(fullList, maxBatchSize, processBatch, EMPTY_BICONSUMER);
}
}
\ No newline at end of file
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment