Commit f40d4277 authored by Florian Schäfer's avatar Florian Schäfer

Add API query for the names of all languages used in Wikidata, refine existing queries

parent adeaa86a
Pipeline #27433762 passed with stages
in 15 minutes and 13 seconds
......@@ -5,17 +5,23 @@ import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import org.openstreetmap.josm.tools.HttpClient;
import org.openstreetmap.josm.tools.Utils;
import org.wikipedia.api.ApiQuery;
import org.wikipedia.api.ApiUrl;
import org.wikipedia.api.SerializationSchema;
import org.wikipedia.api.wikidata_action.json.QueryResult;
import org.wikipedia.api.wikidata_action.json.SitematrixResult;
import org.wikipedia.api.wikidata_action.json.WbgetclaimsResult;
import org.wikipedia.api.wikidata_action.json.WbgetentitiesResult;
import org.wikipedia.tools.RegexUtil;
/**
* Utility class for getting queries against the Wikidata Action API
* @param <T> the type which is returned as a result of the query
*/
public final class WikidataActionApiQuery<T> extends ApiQuery<T> {
static URL defaultUrl = ApiUrl.url("https://www.wikidata.org/w/api.php");
private static final String FORMAT_PARAMS = "format=json&utf8=1&formatversion=1";
......@@ -23,29 +29,60 @@ public final class WikidataActionApiQuery<T> extends ApiQuery<T> {
private final String queryString;
private WikidataActionApiQuery(final String query, final SerializationSchema<T> schema, final long cacheExpiryTime) {
super(defaultUrl, schema, cacheExpiryTime);
this.queryString = query;
/**
* Convenience constructor
* @see #WikidataActionApiQuery(String, SerializationSchema, long, Function)
*/
private WikidataActionApiQuery(final String queryString, final SerializationSchema<T> schema) {
this(queryString, schema, -1, it -> it);
}
private WikidataActionApiQuery(final String query, final SerializationSchema<T> schema) {
this(query, schema, -1);
/**
* Convenience constructor
* @see #WikidataActionApiQuery(String, SerializationSchema, long, Function)
*/
private <S> WikidataActionApiQuery(final String queryString, final SerializationSchema<S> schema, final Function<S, T> converter) {
this(queryString, schema, -1, converter);
}
private <S> WikidataActionApiQuery(final String queryString, final SerializationSchema<S> schema, final Function<S, T> converter) {
super(defaultUrl, schema, -1, converter);
/**
* A query against the Wikidata Action API
* @param queryString the query string containing the arguments of this query
* @param schema the {@link SerializationSchema} that defines how deserialization of the response is handled
* @param cacheExpiryTime the number of milliseconds for which the response will stay in the cache
* @param converter a function that maps the
* @param <S> the type to which the {@link SerializationSchema} deserializes
*/
private <S> WikidataActionApiQuery(final String queryString, final SerializationSchema<S> schema, final long cacheExpiryTime, final Function<S, T> converter) {
super(defaultUrl, schema, cacheExpiryTime, converter);
this.queryString = queryString;
}
public String getQueryString() {
String getQueryString() {
return queryString;
}
public static WikidataActionApiQuery<SitematrixResult> sitematrix() {
/**
* @return a query for all wikimedia sites
*/
public static WikidataActionApiQuery<SitematrixResult.Sitematrix> sitematrix() {
return new WikidataActionApiQuery<>(
FORMAT_PARAMS + "&action=sitematrix",
SitematrixResult.SCHEMA,
2_592_000_000L // = 1000*60*60*24*30 = number of ms in 30 days
TimeUnit.DAYS.toMillis(30),
SitematrixResult::getSitematrix
);
}
/**
* @return a query for all languages that are used in Wikidata (e.g. for labels)
*/
public static WikidataActionApiQuery<Map<String, String>> queryLanguages() {
return new WikidataActionApiQuery<>(
FORMAT_PARAMS + "&action=query&meta=siteinfo&siprop=languages",
QueryResult.SCHEMA,
TimeUnit.DAYS.toMillis(30),
QueryResult::getLangMap
);
}
......@@ -77,13 +114,14 @@ public final class WikidataActionApiQuery<T> extends ApiQuery<T> {
);
}
public static WikidataActionApiQuery<Map<String, String>> wbgetentitiesLabels(final String qId) {
public static WikidataActionApiQuery<Map<String, WbgetentitiesResult.Entity.Label>> wbgetentitiesLabels(final String qId) {
if (!RegexUtil.isValidQId(qId)) {
throw new IllegalArgumentException("Invalid Q-ID: " + qId);
}
return new WikidataActionApiQuery<>(
FORMAT_PARAMS + "&action=wbgetentities&props=labels&ids=" + qId,
WbgetentitiesResult.SCHEMA,
TimeUnit.MINUTES.toMillis(5),
result -> result.getEntities().values().stream().findFirst().map(WbgetentitiesResult.Entity::getLabels).orElse(new HashMap<>())
);
}
......
package org.wikipedia.api.wikidata_action.json;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.DeserializationFeature;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.wikipedia.api.SerializationSchema;
public final class QueryResult {
public static final SerializationSchema<QueryResult> SCHEMA = new SerializationSchema<>(
QueryResult.class,
mapper -> mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
);
private final Query query;
@JsonCreator
public QueryResult(@JsonProperty("query") final Query query) {
this.query = query;
}
private static class Query {
private final List<Language> languages;
@JsonCreator
Query(@JsonProperty("languages") final Collection<Language> languages) {
this.languages = new ArrayList<>(languages);
}
private static class Language implements Comparable<Language> {
private final String code;
private final String name;
@JsonCreator
Language(@JsonProperty("code") final String code, @JsonProperty("*") final String name) {
this.code = code;
this.name = name;
}
@Override
public int compareTo(Language other) {
return this.code.compareTo(other.code);
}
}
}
public Map<String, String> getLangMap() {
return query.languages.stream().collect(Collectors.toMap(it -> it.code, it -> it.name));
}
}
......@@ -36,7 +36,7 @@ public final class SitematrixResult {
@JsonCreator
public SitematrixResult(@JsonProperty("sitematrix") final Sitematrix sitematrix) {
this.sitematrix = sitematrix;
this.sitematrix = Objects.requireNonNull(sitematrix);
}
public Sitematrix getSitematrix() {
......
......@@ -122,9 +122,9 @@ public final class WbgetclaimsResult {
return property;
}
static interface DataValue {
public interface DataValue {
static class Deserializer extends CustomDeserializer<DataValue> {
class Deserializer extends CustomDeserializer<DataValue> {
Deserializer(ObjectMapper mapper) {
super(mapper);
}
......@@ -139,6 +139,7 @@ public final class WbgetclaimsResult {
case "quantity": return mapper.treeToValue(nodeValue, QuantityValue.class);
case "globecoordinate": return mapper.treeToValue(nodeValue, GlobecoordinateValue.class);
case "time": return mapper.treeToValue(nodeValue, TimeValue.class);
case "monolingualtext": return mapper.treeToValue(nodeValue, MonolingualTextValue.class);
default:
Logging.warn("Unknown type: " + type.textValue());
}
......@@ -150,7 +151,7 @@ public final class WbgetclaimsResult {
static class GlobecoordinateValue implements DataValue {
private final double latitude;
private final double longitude;
private final double altitude;
private final Double altitude;
private final double precision;
private final String globe;
......@@ -158,7 +159,7 @@ public final class WbgetclaimsResult {
GlobecoordinateValue(
@JsonProperty("latitude") final double latitude,
@JsonProperty("longitude") final double longitude,
@JsonProperty("altitude") final double altitude,
@JsonProperty("altitude") final Double altitude,
@JsonProperty("precision") final double precision,
@JsonProperty("globe") final String globe
) {
......@@ -169,9 +170,14 @@ public final class WbgetclaimsResult {
this.globe = globe;
}
public LatLon toLatLon() {
private LatLon toLatLon() {
return new LatLon(latitude, longitude);
}
@Override
public String toString() {
return toLatLon().toDisplayString();
}
}
static class TimeValue implements DataValue {
......@@ -199,6 +205,11 @@ public final class WbgetclaimsResult {
this.precision = precision;
this.calendarModel = calendarModel;
}
@Override
public String toString() {
return time;
}
}
static class QuantityValue implements DataValue {
......@@ -219,6 +230,11 @@ public final class WbgetclaimsResult {
this.lowerBound = lowerBound;
this.upperBound = upperBound;
}
@Override
public String toString() {
return amount + " " + unit;
}
}
static class ItemValue implements DataValue {
......@@ -230,13 +246,39 @@ public final class WbgetclaimsResult {
this.entityType = entityType;
this.id = id;
}
@Override
public String toString() {
return id + " (" + entityType + ")";
}
}
static class StringValue implements DataValue {
private final String value;
StringValue(String value) {
StringValue(final String value) {
this.value = value;
}
@Override
public String toString() {
return value;
}
}
static class MonolingualTextValue implements DataValue {
private final String langCode;
private final String text;
@JsonCreator
MonolingualTextValue(@JsonProperty("language") final String langCode, @JsonProperty("text") final String text) {
this.langCode = langCode;
this.text = text;
}
@Override
public String toString() {
return text + " (" + langCode + ")";
}
}
}
......
......@@ -136,7 +136,7 @@ public final class WbgetentitiesResult {
private final String id;
private final String type;
private final Map<String, Sitelink> sitelinks = new HashMap<>();
private final Map<String, String> labels = new HashMap<>();
private final Map<String, Label> labels = new HashMap<>();
@JsonCreator
public Entity(
......@@ -151,7 +151,7 @@ public final class WbgetentitiesResult {
this.sitelinks.putAll(sitelinks);
}
if (labels != null) {
labels.values().forEach(label -> this.labels.put(label.getLanguage(), label.getValue()));
labels.values().forEach(label -> this.labels.put(label.getLangCode(), label));
}
}
......@@ -163,7 +163,7 @@ public final class WbgetentitiesResult {
return type;
}
public Map<String, String> getLabels() {
public Map<String, Label> getLabels() {
return Collections.unmodifiableMap(labels);
}
......@@ -186,7 +186,7 @@ public final class WbgetentitiesResult {
this.value = value;
}
public String getLanguage() {
public String getLangCode() {
return language;
}
......
......@@ -24,8 +24,8 @@ public class WikipediaSite implements IWikipediaSite {
*/
public WikipediaSite(final String langCode) throws IOException, IllegalArgumentException {
Objects.requireNonNull(langCode);
final SitematrixResult sitematrix = ApiQueryClient.query(WikidataActionApiQuery.sitematrix());
language = sitematrix.getSitematrix().getLanguages().stream()
final SitematrixResult.Sitematrix sitematrix = ApiQueryClient.query(WikidataActionApiQuery.sitematrix());
language = sitematrix.getLanguages().stream()
.filter(it -> langCode.equals(it.getCode()))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException(I18n.tr("''{0}'' is an illegal language code!", langCode)));
......
......@@ -3,6 +3,7 @@ package org.wikipedia.gui;
import java.awt.BorderLayout;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
......@@ -16,6 +17,7 @@ import org.openstreetmap.josm.tools.Pair;
import org.wikipedia.WikipediaPlugin;
import org.wikipedia.api.ApiQueryClient;
import org.wikipedia.api.wikidata_action.WikidataActionApiQuery;
import org.wikipedia.api.wikidata_action.json.WbgetentitiesResult;
/**
* Panel displaying the labels for a Wikidata item
......@@ -59,7 +61,7 @@ class WikidataInfoLabelPanel extends JPanel {
private static class LabelTableModel extends AbstractTableModel {
private final WikidataInfoLabelPanel parent;
private String qIdBeingDownloaded;
private final List<Pair<String, String>> valueMap = new ArrayList<>();
private final List<Pair<WbgetentitiesResult.Entity.Label, String>> valueMap = new ArrayList<>();
LabelTableModel(final WikidataInfoLabelPanel parent) {
this.parent = parent;
......@@ -77,14 +79,14 @@ class WikidataInfoLabelPanel extends JPanel {
new Thread(() -> {
try {
final Map<String, String> newValues = ApiQueryClient.query(WikidataActionApiQuery.wbgetentitiesLabels(qId));
final Map<String, WbgetentitiesResult.Entity.Label> newValues = ApiQueryClient.query(WikidataActionApiQuery.wbgetentitiesLabels(qId));
synchronized (valueMap) {
if (qIdBeingDownloaded != null && qIdBeingDownloaded.equals(qId)) {
valueMap.clear();
valueMap.addAll(
newValues.entrySet().stream()
.map(it -> Pair.create(it.getKey(), it.getValue()))
.sorted((x, y) -> x.a.compareTo(y.a))
.map(it -> Pair.create(it.getValue(), it.getKey()))
.sorted(Comparator.comparing(it -> it.a.getLangCode()))
.collect(Collectors.toList())
);
}
......
......@@ -121,13 +121,13 @@ public class WikidataActionApiQueryTest extends WikidataActionApiTestAbstract {
public void testWikidataItemLabelQuery() throws IOException, URISyntaxException {
simpleJsonStub(ResourceFileLoader.getResourceBytes(WikidataActionApiQueryTest.class, "response/wbgetentities/labels_Q42.json"));
final Map<String, String> result = ApiQueryClient.query(WikidataActionApiQuery.wbgetentitiesLabels("Q42"));
final Map<String, WbgetentitiesResult.Entity.Label> result = ApiQueryClient.query(WikidataActionApiQuery.wbgetentitiesLabels("Q42"));
assertEquals(138, result.size());
assertEquals("Douglas Adams", result.get("en"));
assertEquals("Дуглас Адамс", result.get("ru"));
assertEquals("더글러스 애덤스", result.get("ko"));
assertEquals("ಡಾಗ್ಲಸ್ ಆಡಮ್ಸ್", result.get("tcy"));
assertEquals("Douglas Adams", result.get("en").getValue());
assertEquals("Дуглас Адамс", result.get("ru").getValue());
assertEquals("더글러스 애덤스", result.get("ko").getValue());
assertEquals("ಡಾಗ್ಲಸ್ ಆಡಮ್ಸ್", result.get("tcy").getValue());
simpleRequestVerify("format=json&utf8=1&formatversion=1&action=wbgetentities&props=labels&ids=Q42");
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment