Commit a100deee authored by Florian Schäfer's avatar Florian Schäfer

Merge branch 'api_query'

parents 4c58d3bb 71ba1571
......@@ -15,7 +15,7 @@ plugins {
id "com.github.ben-manes.versions" version "0.17.0"
id "com.github.spotbugs" version "1.6.1"
id "net.ltgt.errorprone" version "0.0.14"
id "org.openstreetmap.josm" version "0.4.4"
id "org.openstreetmap.josm" version "0.4.6"
}
def versions = [
errorprone: "2.3.1",
......@@ -36,6 +36,7 @@ dependencies {
testImplementation "org.junit.vintage:junit-vintage-engine:${versions.junit}"
testImplementation "com.github.spotbugs:spotbugs-annotations:${versions.spotbugs}"
testImplementation("org.openstreetmap.josm:josm-unittest:"){changing=true}
testImplementation "com.github.tomakehurst:wiremock:2.18.0"
}
task copyToLib(type: Sync) {
from configurations.packIntoJar
......
......@@ -16,8 +16,6 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collector;
......@@ -35,6 +33,9 @@ import org.openstreetmap.josm.tools.Pair;
import org.openstreetmap.josm.tools.Utils;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.wikipedia.api.wikidata_action.ApiQueryClient;
import org.wikipedia.api.wikidata_action.WikidataActionApiQuery;
import org.wikipedia.api.wikidata_action.json.CheckEntityExistsResult;
import org.wikipedia.data.WikidataEntry;
import org.wikipedia.data.WikipediaEntry;
import org.wikipedia.tools.ListUtil;
......@@ -223,29 +224,12 @@ public final class WikipediaApp {
* @return article / wikidata id map
*/
public Map<String, String> getWikidataForArticles(Collection<String> articles) {
return articles.stream()
.distinct()
.collect(Collectors.groupingBy(new Function<String, Integer>() {
final AtomicInteger group = new AtomicInteger();
final AtomicInteger count = new AtomicInteger();
final AtomicInteger length = new AtomicInteger();
@Override
public Integer apply(String o) {
// max. 50 titles, max. 2048 of URL encoded title chars (to avoid HTTP 414)
if (count.incrementAndGet() > 50 || length.addAndGet(Utils.encodeUrl(o).length()) > 2048) {
count.set(0);
length.set(0);
return group.incrementAndGet();
} else {
return group.get();
}
}
}))
.values()
.stream()
.flatMap(chunk -> resolveWikidataItems(chunk).entrySet().stream())
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
final Map<String, String> result = new HashMap<>();
// maximum of 50 titles
ListUtil.processInBatches(new ArrayList<String>(articles), 50, batch -> {
result.putAll(resolveWikidataItems(batch));
});
return result;
}
/**
......@@ -297,26 +281,12 @@ public final class WikipediaApp {
return Collections.emptyMap();
}
try {
final String url = "https://www.wikidata.org/w/api.php" +
"?action=wbgetentities" +
"&props=sitelinks" +
"&sites=" + siteId +
"&sitefilter=" + siteId +
"&format=xml" +
"&titles=" + articles.stream().map(Utils::encodeUrl).collect(Collectors.joining("|"));
final Map<String, String> r = new TreeMap<>();
try (InputStream in = connect(url).getContent()) {
final Document xml = newDocumentBuilder().parse(in);
X_PATH.evaluateNodes("//entity", xml).forEach(node -> {
final String wikidata = X_PATH.evaluateString("./@id", node);
final String wikipedia = X_PATH.evaluateString("./sitelinks/sitelink/@title", node);
if (RegexUtil.isValidQId(wikidata)) { // non existing entries result in negative integers
r.put(wikipedia, wikidata);
}
});
}
return r;
} catch (Exception ex) {
return ApiQueryClient.query(WikidataActionApiQuery.wbgetentities(siteId, articles))
.getEntities().values()
.stream()
.filter(it -> RegexUtil.isValidQId(it.getId()) && it.getSitelinks().size() >= 1)
.collect(Collectors.toMap(it -> it.getSitelinks().iterator().next().getTitle(), CheckEntityExistsResult.Entity::getId));
} catch (IOException ex) {
throw new RuntimeException(ex);
}
}
......
package org.wikipedia.api;
import java.net.URL;
import org.wikipedia.api.wikidata_action.json.SerializationSchema;
public abstract class ApiQuery<T> {
private final SerializationSchema<T> schema;
private final URL url;
protected ApiQuery(final URL url, final SerializationSchema<T> schema) {
this.schema = schema;
this.url = url;
}
public SerializationSchema<T> getSchema() {
return schema;
}
public URL getUrl() {
return url;
}
public String getCacheKey() {
return url.toString();
}
}
......@@ -4,8 +4,8 @@ package org.wikipedia.api.wikidata_action;
import java.awt.GraphicsEnvironment;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.openstreetmap.josm.gui.bugreport.BugReportDialog;
......@@ -18,31 +18,50 @@ import org.wikipedia.api.InvalidApiQueryException;
import org.wikipedia.api.wikidata_action.json.SerializationSchema;
public final class ApiQueryClient {
private static final ObjectMapper JSON_OBJECT_MAPPER = new ObjectMapper();
static {
JSON_OBJECT_MAPPER.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
}
private ApiQueryClient() {
// Private constructor to avoid instantiation
}
private static String getUserAgent() {
return String.format("JOSM-wikipedia (%s).", WikipediaPlugin.getVersionInfo()) +
"Report issues at https://josm.openstreetmap.de/newticket?component=Plugin%20wikipedia&priority=major&keywords=api%20wikidata%20ActionAPI";
}
public static <T> T query(final WikidataActionApiQuery<T> query) throws IOException {
return query(
HttpClient.create(query.getUrl(), "POST")
.setAccept("application/json")
.setHeader("Content-Type", "text/plain; charset=utf-8")
.setHeader("User-Agent", getUserAgent())
.setRequestBody(query.getQuery().getBytes(StandardCharsets.UTF_8)),
query.getUrl(),
query.getSchema().getSchemaClass(),
query.getSchema().getMapper()
);
}
/**
* Queries the given URL and converts the received JSON to the given class using the Jackson library
* @param url the {@link URL} to query
* @param klass the class object of the desired type
* @param schema the {@link SerializationSchema} determining how to map from JSON to the desired type
* @param <T> the type to which the JSON is deserialized
* @return the deserialized object
* @throws IOException if any error occurs while executing the query, with a translated message that can be shown to the user.
*/
public static <T> T query(final URL url, final SerializationSchema<T> schema) throws IOException {
return query(
HttpClient.create(url).setAccept("application/json").setHeader("User-Agent", getUserAgent()),
url,
schema.getSchemaClass(),
schema.getMapper()
);
}
private static <T> T query(final HttpClient client, final URL url, final Class<T> resultClass, final ObjectMapper mapper) throws IOException {
final HttpClient.Response response;
try {
response = HttpClient.create(url)
.setAccept("application/json")
.setHeader("User-Agent", String.format("JOSM-wikipedia (%s). Report issues at https://josm.openstreetmap.de/newticket?component=Plugin%%20wikipedia&priority=major&keywords=api%%20wikidata%%20ActionAPI", WikipediaPlugin.getVersionInfo()))
.connect();
response = client.connect();
} catch (IOException e) {
// i18n: {0} is the name of the exception, {1} is the message of the exception. Typical values would be: {0}="UnknownHostException" {1}="www.wikidata.org"
throw new IOException(I18n.tr("Could not connect to the Wikidata Action API, probably a network issue or the website is currently offline ({0}: {1})", e.getClass().getSimpleName(), e.getLocalizedMessage()), e);
......@@ -63,10 +82,12 @@ public final class ApiQueryClient {
throw new IOException(I18n.tr("The Wikidata Action API reported that the query was invalid! Please report as bug to the Wikipedia plugin!"));
}
try {
return schema.getMapper().readValue(response.getContent(), schema.getSchemaClass());
return mapper.readValue(response.getContent(), resultClass);
} catch (JsonMappingException | JsonParseException e) {
Logging.warn(e);
throw new IOException(I18n.tr("The JSON response from the Wikidata Action API can't be read!"), e);
} catch (IOException e) {
Logging.warn(e);
throw new IOException(I18n.tr("When reading the JSON response from the Wikidata Action API, an error occured! ({0}: {1})", e.getClass().getSimpleName(), e.getLocalizedMessage()), e);
}
}
......
// License: GPL. For details, see LICENSE file.
package org.wikipedia.api.wikidata_action;
import java.net.URL;
import java.util.Collection;
import org.openstreetmap.josm.tools.Utils;
import org.wikipedia.api.ApiQuery;
import org.wikipedia.api.ApiUrl;
import org.wikipedia.api.wikidata_action.json.CheckEntityExistsResult;
import org.wikipedia.api.wikidata_action.json.SerializationSchema;
import org.wikipedia.api.wikidata_action.json.SitematrixResult;
import org.wikipedia.tools.RegexUtil;
public final class WikidataActionApiUrl {
private static final String BASE_URL = "https://www.wikidata.org/w/api.php?";
public class WikidataActionApiQuery<T> extends ApiQuery<T> {
static URL defaultUrl = ApiUrl.url("https://www.wikidata.org/w/api.php");
private static final String FORMAT_PARAMS = "format=json&utf8=1&formatversion=1";
private WikidataActionApiUrl() {
// Private constructor to avoid instantiation
private final String query;
private WikidataActionApiQuery(final String query, final SerializationSchema<T> schema) {
super(defaultUrl, schema);
this.query = query;
}
public static URL checkEntityExistsUrl(final Collection<String> qIds) {
public String getQuery() {
return query;
}
public static WikidataActionApiQuery<SitematrixResult> sitematrix() {
return new WikidataActionApiQuery<>(
FORMAT_PARAMS + "&action=sitematrix",
SerializationSchema.SITEMATRIX
);
}
public static WikidataActionApiQuery<CheckEntityExistsResult> wbgetentities(final Collection<String> qIds) {
if (qIds.size() < 1) {
throw new IllegalArgumentException("You must supply at least one Q-ID to construct a checkEntityExists URL.");
}
if (!qIds.stream().allMatch(RegexUtil::isValidQId)) {
throw new IllegalArgumentException("You must supply only Q-IDs as argument to construct a checkEntityExists URL.");
}
return ApiUrl.url(
BASE_URL,
FORMAT_PARAMS,
"&action=wbgetentities&sites=&props=&ids=",
Utils.encodeUrl(String.join("|", qIds))
return new WikidataActionApiQuery<>(
FORMAT_PARAMS + "&action=wbgetentities&sites=&props=&ids=" + Utils.encodeUrl(String.join("|", qIds)),
SerializationSchema.WBGETENTITIES
);
}
public static URL getEntityForSitelink(final String siteId, final Collection<String> titles) {
public static WikidataActionApiQuery<CheckEntityExistsResult> wbgetentities(final String siteId, final Collection<String> titles) {
if (siteId == null || titles == null || titles.size() <= 0) {
throw new IllegalArgumentException("The site ID and titles must be present!");
}
if (!RegexUtil.isValidSiteId(siteId)) {
throw new IllegalArgumentException("The site ID is not given in the expected format!");
}
return ApiUrl.url(
BASE_URL,
FORMAT_PARAMS,
"&action=wbgetentities&props=sitelinks",
"&sites=", siteId, // defines the language of the titles
"&sitefilter=", siteId, // defines for which languages sitelinks should be returned
"&titles=", Utils.encodeUrl(String.join("|", titles))
return new WikidataActionApiQuery<>(
FORMAT_PARAMS + "&action=wbgetentities&props=sitelinks&sites=" + siteId + // defines the language of the titles
"&sitefilter=" + siteId + // defines for which languages sitelinks should be returned
"&titles=" + Utils.encodeUrl(String.join("|", titles)),
SerializationSchema.WBGETENTITIES
);
}
@Override
public String getCacheKey() {
return getUrl().toString() + '?' + getQuery();
}
}
// License: GPL. For details, see LICENSE file.
package org.wikipedia.api.wikidata_action.json;
import java.util.Objects;
import java.util.function.Consumer;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
......@@ -21,14 +22,24 @@ public class SerializationSchema<T> {
));
}
);
public static final SerializationSchema<SitematrixResult> SITEMATRIX = new SerializationSchema<>(
SitematrixResult.class,
mapper -> {
mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
mapper.registerModule(new SimpleModule().addDeserializer(
SitematrixResult.Sitematrix.class,
new SitematrixResult.Sitematrix.Deserializer(mapper)
));
}
);
private final ObjectMapper mapper;
private final ObjectMapper mapper = new ObjectMapper();
private final Class<T> schemaClass;
private SerializationSchema(final Class<T> schemaClass, final Consumer<ObjectMapper> mapperConfig) {
Objects.requireNonNull(schemaClass);
this.schemaClass = schemaClass;
mapper = new ObjectMapper();
mapperConfig.accept(mapper);
mapperConfig.accept(mapper); // configure the object mapper
}
public ObjectMapper getMapper() {
......
package org.wikipedia.api.wikidata_action.json;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
import org.wikipedia.tools.RegexUtil;
public final class SitematrixResult {
private final Sitematrix sitematrix;
@JsonCreator
public SitematrixResult(@JsonProperty("sitematrix") final Sitematrix sitematrix) {
this.sitematrix = sitematrix;
}
public Sitematrix getSitematrix() {
return sitematrix;
}
public static final class Sitematrix {
private final Collection<Language> languages = new ArrayList<>();
private final Collection<Site> specialSites = new ArrayList<>();
private Sitematrix() {
// Instantiates empty object
}
public Collection<Language> getLanguages() {
return Collections.unmodifiableCollection(languages);
}
public Collection<Site> getSpecialSites() {
return Collections.unmodifiableCollection(specialSites);
}
public static class Deserializer extends StdDeserializer<Sitematrix> {
private final ObjectMapper mapper;
public Deserializer(final ObjectMapper mapper) {
super((Class<?>) null);
this.mapper = mapper;
}
@Override
public Sitematrix deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
final Sitematrix result = new Sitematrix();
final JsonNode node = p.getCodec().readTree(p);
final Iterator<Map.Entry<String, JsonNode>> fields = node.fields();
while (fields.hasNext()) {
final Map.Entry<String, JsonNode> field = fields.next();
if (RegexUtil.INTEGER_PATTERN.matcher(field.getKey()).matches()) {
result.languages.add(mapper.treeToValue(field.getValue(), Language.class));
}
if ("specials".equals(field.getKey())) {
final Iterator<JsonNode> elements = field.getValue().elements();
while (elements.hasNext()) {
result.specialSites.add(mapper.treeToValue(elements.next(), Site.class));
}
}
}
return result;
}
}
public static final class Language {
private final String code;
private final String name;
private final Collection<Site> sites = new ArrayList<>();
@JsonCreator
public Language(
@JsonProperty("code") final String code,
@JsonProperty("name") final String name,
@JsonProperty("site") final Collection<Site> sites
) {
this.code = code;
this.name = name;
if (sites != null) {
this.sites.addAll(sites);
}
}
public String getCode() {
return code;
}
public String getName() {
return name;
}
public Collection<Site> getSites() {
return Collections.unmodifiableCollection(sites);
}
}
public static final class Site {
private final boolean closed;
private final String code;
private final String dbName;
private final String url;
@JsonCreator
public Site(
@JsonProperty("url") final String url,
@JsonProperty("dbname") final String dbName,
@JsonProperty("code") final String code,
@JsonProperty("closed") final String closed
) {
this.closed = closed != null;
this.code = code;
this.dbName = dbName;
this.url = url;
}
public String getCode() {
return code;
}
public String getDbName() {
return dbName;
}
public String getUrl() {
return url;
}
public boolean isClosed() {
return closed;
}
}
}
}
......@@ -8,6 +8,8 @@ public class RegexUtil {
private static final Pattern SITE_ID_PATTERN = Pattern.compile("^[a-z][a-z][a-z]?wiki$");
public static final Pattern WIKIPEDIA_TAG_VALUE_PATTERN = Pattern.compile("([a-z][a-z][a-z]?):(.+)");
public static final Pattern INTEGER_PATTERN = Pattern.compile("^[0-9]+$");
private RegexUtil() {
// Private constructor to avoid instantiation
}
......
......@@ -5,7 +5,6 @@ import static org.wikipedia.validator.AllValidationTests.SEE_OTHER_CATEGORY_VALI
import static org.wikipedia.validator.AllValidationTests.VALIDATOR_MESSAGE_MARKER;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
......@@ -16,9 +15,8 @@ import org.openstreetmap.josm.gui.Notification;
import org.openstreetmap.josm.tools.I18n;
import org.wikipedia.WikipediaPlugin;
import org.wikipedia.api.wikidata_action.ApiQueryClient;
import org.wikipedia.api.wikidata_action.WikidataActionApiUrl;
import org.wikipedia.api.wikidata_action.WikidataActionApiQuery;
import org.wikipedia.api.wikidata_action.json.CheckEntityExistsResult;
import org.wikipedia.api.wikidata_action.json.SerializationSchema;
import org.wikipedia.tools.ListUtil;
import org.wikipedia.tools.RegexUtil;
......@@ -37,15 +35,19 @@ public class WikidataItemExists extends BatchProcessedTagTest<WikidataItemExists
public WikidataItemExists() {
super(I18n.tr("wikidata=* item exists"), I18n.tr("Make sure the Wikidata item for the Q-ID in the wikidata=* tag actually exists"));
super(
I18n.tr("wikidata=* item exists"),
I18n.tr("Make sure the Wikidata item for the Q-ID in the wikidata=* tag actually exists")
);
}
@Override
protected void check(List<TestCompanion> allPrimitives) {
ListUtil.processInBatches(allPrimitives, 50, primitiveBatch -> {
try {
final URL url = WikidataActionApiUrl.checkEntityExistsUrl(primitiveBatch.stream().map(tc -> tc.wikidataId).collect(Collectors.toList()));
final CheckEntityExistsResult entityQueryResult = ApiQueryClient.query(url, SerializationSchema.WBGETENTITIES);
final CheckEntityExistsResult entityQueryResult = ApiQueryClient.query(
WikidataActionApiQuery.wbgetentities(primitiveBatch.stream().map(tc -> tc.wikidataId).collect(Collectors.toList()))
);
if (entityQueryResult.getSuccess() != 1) {
errors.add(AllValidationTests.API_REQUEST_FAILED.getBuilder(this)
.primitives(primitiveBatch.stream().map(BatchProcessedTagTest.TestCompanion::getPrimitive).collect(Collectors.toList()))
......@@ -59,7 +61,11 @@ public class WikidataItemExists extends BatchProcessedTagTest<WikidataItemExists
}
} catch (IOException e) {
finalNotification = NETWORK_FAILED_NOTIFICATION;
errors.add(AllValidationTests.API_REQUEST_FAILED.getBuilder(this).primitives(primitiveBatch.stream().map(BatchProcessedTagTest.TestCompanion::getPrimitive).collect(Collectors.toList())).message(VALIDATOR_MESSAGE_MARKER + e.getMessage()).build());
errors.add(AllValidationTests.API_REQUEST_FAILED.getBuilder(this)
.primitives(primitiveBatch.stream().map(BatchProcessedTagTest.TestCompanion::getPrimitive).collect(Collectors.toList()))
.message(VALIDATOR_MESSAGE_MARKER + e.getMessage())
.build()
);
}
}, this::updateBatchProgress);
}
......@@ -75,20 +81,33 @@ public class WikidataItemExists extends BatchProcessedTagTest<WikidataItemExists
errors.add(
AllValidationTests.WIKIDATA_ITEM_DOES_NOT_EXIST.getBuilder(this)
.primitives(tc.getPrimitive())
.message(VALIDATOR_MESSAGE_MARKER + I18n.tr("The Wikidata item does not exist! Replace the wikidata=* tag with an existing Wikidata item or remove the Wikidata tag."), I18n.marktr("Item {0} does not exist!"), tc.wikidataId)
.message(
VALIDATOR_MESSAGE_MARKER + I18n.tr("The Wikidata item does not exist! Replace the wikidata=* tag with an existing Wikidata item or remove the Wikidata tag."),
I18n.marktr("Item {0} does not exist!"),
tc.wikidataId
)
.build()
);
} else if (entity == null) {
errors.add(
AllValidationTests.API_REQUEST_FAILED.getBuilder(this)
.primitives(tc.getPrimitive())
.message(VALIDATOR_MESSAGE_MARKER + I18n.tr("The Wikidata Action API did not respond with all requested entities!"), I18n.marktr("Item {0} is missing"), tc.wikidataId)
.message(
VALIDATOR_MESSAGE_MARKER + I18n.tr("The Wikidata Action API did not respond with all requested entities!"),
I18n.marktr("Item {0} is missing"),
tc.wikidataId
)
.build());
} else if (!tc.wikidataId.equals(entity.getId())) {
errors.add(
AllValidationTests.WIKIDATA_ITEM_IS_REDIRECT.getBuilder(this)
.primitives(tc.getPrimitive())
.message(VALIDATOR_MESSAGE_MARKER + I18n.tr("The Wikidata item is a redirect", tc.wikidataId, entity.getId()), I18n.marktr("Item {0} redirects to {1}"), tc.wikidataId, entity.getId())
.message(
VALIDATOR_MESSAGE_MARKER + I18n.tr("The Wikidata item is a redirect", tc.wikidataId, entity.getId()),
I18n.marktr("Item {0} redirects to {1}"),
tc.wikidataId,
entity.getId()
)
.fix(() -> new ChangePropertyCommand(tc.getPrimitive(), "wikidata", entity.getId()))
.build()
);
......@@ -105,7 +124,10 @@ public class WikidataItemExists extends BatchProcessedTagTest<WikidataItemExists
errors.add(
AllValidationTests.INVALID_QID.getBuilder(this)
.primitives(primitive)
.message(VALIDATOR_MESSAGE_MARKER + I18n.tr("Invalid Q-ID! Cannot exist on Wikidata."), I18n.marktr("{0} is not a valid Wikidata-ID"))
.message(
VALIDATOR_MESSAGE_MARKER + I18n.tr("Invalid Q-ID! Cannot exist on Wikidata."),
I18n.marktr("{0} is not a valid Wikidata-ID")
)
.build()
);
}
......
......@@ -16,9 +16,8 @@ import org.openstreetmap.josm.tools.I18n;
import org.openstreetmap.josm.tools.Pair;
import org.wikipedia.WikipediaPlugin;
import org.wikipedia.api.wikidata_action.ApiQueryClient;
import org.wikipedia.api.wikidata_action.WikidataActionApiUrl;
import org.wikipedia.api.wikidata_action.WikidataActionApiQuery;
import org.wikipedia.api.wikidata_action.json.CheckEntityExistsResult;
import org.wikipedia.api.wikidata_action.json.SerializationSchema;
import org.wikipedia.tools.ListUtil;
import org.wikipedia.tools.RegexUtil;
......@@ -30,7 +29,10 @@ public class WikipediaAgainstWikidata extends BatchProcessedTagTest<WikipediaAga
).setIcon(WikipediaPlugin.LOGO);
public WikipediaAgainstWikidata() {
super("Check wikipedia=* is interwiki link of wikidata=*", "make sure that the wikipedia=* article is connected to the wikidata=* item");
super(
I18n.tr("Check wikipedia=* is interwiki link of wikidata=*"),
I18n.tr("Makes sure that the wikipedia=* article is connected to the wikidata=* item")
);
}
@Override
......@@ -64,11 +66,12 @@ public class WikipediaAgainstWikidata extends BatchProcessedTagTest<WikipediaAga
private void checkBatch(final String language, final List<TestCompanion> primitiveBatch) {
try {
final Map<String, CheckEntityExistsResult.Entity.Sitelink> sitelinks =
ApiQueryClient.query(
WikidataActionApiUrl.getEntityForSitelink(language + "wiki", primitiveBatch.stream().map(it -> it.title).collect(Collectors.toList())),
SerializationSchema.WBGETENTITIES
).getEntities().values().stream()
final Map<String, CheckEntityExistsResult.Entity.Sitelink> sitelinks = ApiQueryClient
.query(WikidataActionApiQuery.wbgetentities(
language + "wiki",
primitiveBatch.stream().map(it -> it.title).collect(Collectors.toList())
))
.getEntities().values().stream()
.flatMap(entity -> entity.getSitelinks().stream().map(it -> Pair.create(entity.getId(), it)))
.collect(Collectors.toMap(it -> it.a, it -> it.b));
primitiveBatch.forEach(tc -> {
......
{"entities":{"Q64":{"type":"item","id":"Q64","sitelinks":{"dewiki":{"site":"dewiki","title":"Berlin","badges":["Q17437798"]}}}},"success":1}
\ No newline at end of file
{"entities":{"-1":{"site":"enwiki","title":"missing-article","missing":""},"-2":{"site":"enwiki","title":"Another missing article","missing":""},"Q30":{"type":"item","id":"Q30","sitelinks":{"enwiki":{"site":"enwiki","title":"United States","badges":["Q17437798"]}}},"Q23666":{"type":"item","id":"Q23666","sitelinks":{"enwiki":{"site":"enwiki","title":"Great Britain","badges":[]}}}},"success":1}
\ No newline at end of file
// License: GPL. For details, see LICENSE file.
package org.wikipedia.api.wikidata_action;
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
import static com.github.tomakehurst.wiremock.client.WireMock.equalTo;
import static com.github.tomakehurst.wiremock.client.WireMock.post;
import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor;
import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
import static com.github.tomakehurst.wiremock.client.WireMock.verify;
import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import com.github.tomakehurst.wiremock.junit.WireMockRule;
import com.github.tomakehurst.wiremock.matching.EqualToPattern;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.openstreetmap.josm.testutils.JOSMTestRules;