Commit 71ba1571 authored by Florian Schäfer's avatar Florian Schäfer

Move all remaining API requests from `WikidataActionApiUrl` to `WikidataActionApiQuery`

parent eaedcd06
......@@ -33,6 +33,19 @@ public class WikidataActionApiQuery<T> extends ApiQuery<T> {
);
}
public static WikidataActionApiQuery<CheckEntityExistsResult> wbgetentities(final Collection<String> qIds) {
if (qIds.size() < 1) {
throw new IllegalArgumentException("You must supply at least one Q-ID to construct a checkEntityExists URL.");
}
if (!qIds.stream().allMatch(RegexUtil::isValidQId)) {
throw new IllegalArgumentException("You must supply only Q-IDs as argument to construct a checkEntityExists URL.");
}
return new WikidataActionApiQuery<>(
FORMAT_PARAMS + "&action=wbgetentities&sites=&props=&ids=" + Utils.encodeUrl(String.join("|", qIds)),
SerializationSchema.WBGETENTITIES
);
}
public static WikidataActionApiQuery<CheckEntityExistsResult> wbgetentities(final String siteId, final Collection<String> titles) {
if (siteId == null || titles == null || titles.size() <= 0) {
throw new IllegalArgumentException("The site ID and titles must be present!");
......
// License: GPL. For details, see LICENSE file.
package org.wikipedia.api.wikidata_action;
import java.net.URL;
import java.util.Collection;
import org.openstreetmap.josm.tools.Utils;
import org.wikipedia.api.ApiUrl;
import org.wikipedia.tools.RegexUtil;
public class WikidataActionApiUrl {
private static final String BASE_URL = "https://www.wikidata.org/w/api.php?";
private static final String FORMAT_PARAMS = "format=json&utf8=1&formatversion=1";
private WikidataActionApiUrl() {
// Private constructor to avoid instantiation
}
public static URL checkEntityExistsUrl(final Collection<String> qIds) {
if (qIds.size() < 1) {
throw new IllegalArgumentException("You must supply at least one Q-ID to construct a checkEntityExists URL.");
}
if (!qIds.stream().allMatch(RegexUtil::isValidQId)) {
throw new IllegalArgumentException("You must supply only Q-IDs as argument to construct a checkEntityExists URL.");
}
return ApiUrl.url(
BASE_URL,
FORMAT_PARAMS,
"&action=wbgetentities&sites=&props=&ids=",
Utils.encodeUrl(String.join("|", qIds))
);
}
public static URL getEntityForSitelink(final String siteId, final Collection<String> titles) {
if (siteId == null || titles == null || titles.size() <= 0) {
throw new IllegalArgumentException("The site ID and titles must be present!");
}
if (!RegexUtil.isValidSiteId(siteId)) {
throw new IllegalArgumentException("The site ID is not given in the expected format!");
}
return ApiUrl.url(
BASE_URL,
FORMAT_PARAMS,
"&action=wbgetentities&props=sitelinks",
"&sites=", siteId, // defines the language of the titles
"&sitefilter=", siteId, // defines for which languages sitelinks should be returned
"&titles=", Utils.encodeUrl(String.join("|", titles))
);
}
}
......@@ -5,7 +5,6 @@ import static org.wikipedia.validator.AllValidationTests.SEE_OTHER_CATEGORY_VALI
import static org.wikipedia.validator.AllValidationTests.VALIDATOR_MESSAGE_MARKER;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
......@@ -17,9 +16,8 @@ import org.openstreetmap.josm.gui.Notification;
import org.openstreetmap.josm.tools.I18n;
import org.wikipedia.WikipediaPlugin;
import org.wikipedia.api.wikidata_action.ApiQueryClient;
import org.wikipedia.api.wikidata_action.WikidataActionApiUrl;
import org.wikipedia.api.wikidata_action.WikidataActionApiQuery;
import org.wikipedia.api.wikidata_action.json.CheckEntityExistsResult;
import org.wikipedia.api.wikidata_action.json.SerializationSchema;
import org.wikipedia.tools.ListUtil;
import org.wikipedia.tools.RegexUtil;
......@@ -38,15 +36,19 @@ public class WikidataItemExists extends BatchProcessedTagTest<WikidataItemExists
public WikidataItemExists() {
super(I18n.tr("wikidata=* item exists"), I18n.tr("Make sure the Wikidata item for the Q-ID in the wikidata=* tag actually exists"));
super(
I18n.tr("wikidata=* item exists"),
I18n.tr("Make sure the Wikidata item for the Q-ID in the wikidata=* tag actually exists")
);
}
@Override
protected void check(List<TestCompanion> allPrimitives) {
ListUtil.processInBatches(allPrimitives, 50, primitiveBatch -> {
try {
final URL url = WikidataActionApiUrl.checkEntityExistsUrl(primitiveBatch.stream().map(tc -> tc.wikidataId).collect(Collectors.toList()));
final CheckEntityExistsResult entityQueryResult = ApiQueryClient.query(url, SerializationSchema.WBGETENTITIES);
final CheckEntityExistsResult entityQueryResult = ApiQueryClient.query(
WikidataActionApiQuery.wbgetentities(primitiveBatch.stream().map(tc -> tc.wikidataId).collect(Collectors.toList()))
);
if (entityQueryResult.getSuccess() != 1) {
errors.add(AllValidationTests.API_REQUEST_FAILED.getBuilder(this)
.primitives(primitiveBatch.stream().map(BatchProcessedTagTest.TestCompanion::getPrimitive).collect(Collectors.toList()))
......@@ -60,7 +62,11 @@ public class WikidataItemExists extends BatchProcessedTagTest<WikidataItemExists
}
} catch (IOException e) {
finalNotification = NETWORK_FAILED_NOTIFICATION;
errors.add(AllValidationTests.API_REQUEST_FAILED.getBuilder(this).primitives(primitiveBatch.stream().map(BatchProcessedTagTest.TestCompanion::getPrimitive).collect(Collectors.toList())).message(VALIDATOR_MESSAGE_MARKER + e.getMessage()).build());
errors.add(AllValidationTests.API_REQUEST_FAILED.getBuilder(this)
.primitives(primitiveBatch.stream().map(BatchProcessedTagTest.TestCompanion::getPrimitive).collect(Collectors.toList()))
.message(VALIDATOR_MESSAGE_MARKER + e.getMessage())
.build()
);
}
}, this::updateBatchProgress);
}
......@@ -76,20 +82,33 @@ public class WikidataItemExists extends BatchProcessedTagTest<WikidataItemExists
errors.add(
AllValidationTests.WIKIDATA_ITEM_DOES_NOT_EXIST.getBuilder(this)
.primitives(tc.getPrimitive())
.message(VALIDATOR_MESSAGE_MARKER + I18n.tr("The Wikidata item does not exist! Replace the wikidata=* tag with an existing Wikidata item or remove the Wikidata tag."), I18n.marktr("Item {0} does not exist!"), tc.wikidataId)
.message(
VALIDATOR_MESSAGE_MARKER + I18n.tr("The Wikidata item does not exist! Replace the wikidata=* tag with an existing Wikidata item or remove the Wikidata tag."),
I18n.marktr("Item {0} does not exist!"),
tc.wikidataId
)
.build()
);
} else if (entity == null) {
errors.add(
AllValidationTests.API_REQUEST_FAILED.getBuilder(this)
.primitives(tc.getPrimitive())
.message(VALIDATOR_MESSAGE_MARKER + I18n.tr("The Wikidata Action API did not respond with all requested entities!"), I18n.marktr("Item {0} is missing"), tc.wikidataId)
.message(
VALIDATOR_MESSAGE_MARKER + I18n.tr("The Wikidata Action API did not respond with all requested entities!"),
I18n.marktr("Item {0} is missing"),
tc.wikidataId
)
.build());
} else if (!tc.wikidataId.equals(entity.getId())) {
errors.add(
AllValidationTests.WIKIDATA_ITEM_IS_REDIRECT.getBuilder(this)
.primitives(tc.getPrimitive())
.message(VALIDATOR_MESSAGE_MARKER + I18n.tr("The Wikidata item is a redirect", tc.wikidataId, entity.getId()), I18n.marktr("Item {0} redirects to {1}"), tc.wikidataId, entity.getId())
.message(
VALIDATOR_MESSAGE_MARKER + I18n.tr("The Wikidata item is a redirect", tc.wikidataId, entity.getId()),
I18n.marktr("Item {0} redirects to {1}"),
tc.wikidataId,
entity.getId()
)
.fix(() -> new ChangePropertyCommand(tc.getPrimitive(), "wikidata", entity.getId()))
.build()
);
......@@ -106,7 +125,10 @@ public class WikidataItemExists extends BatchProcessedTagTest<WikidataItemExists
errors.add(
AllValidationTests.INVALID_QID.getBuilder(this)
.primitives(primitive)
.message(VALIDATOR_MESSAGE_MARKER + I18n.tr("Invalid Q-ID! Cannot exist on Wikidata."), I18n.marktr("{0} is not a valid Wikidata-ID"))
.message(
VALIDATOR_MESSAGE_MARKER + I18n.tr("Invalid Q-ID! Cannot exist on Wikidata."),
I18n.marktr("{0} is not a valid Wikidata-ID")
)
.build()
);
}
......
......@@ -17,9 +17,8 @@ import org.openstreetmap.josm.tools.I18n;
import org.openstreetmap.josm.tools.Pair;
import org.wikipedia.WikipediaPlugin;
import org.wikipedia.api.wikidata_action.ApiQueryClient;
import org.wikipedia.api.wikidata_action.WikidataActionApiUrl;
import org.wikipedia.api.wikidata_action.WikidataActionApiQuery;
import org.wikipedia.api.wikidata_action.json.CheckEntityExistsResult;
import org.wikipedia.api.wikidata_action.json.SerializationSchema;
import org.wikipedia.tools.ListUtil;
import org.wikipedia.tools.RegexUtil;
......@@ -31,7 +30,10 @@ public class WikipediaAgainstWikidata extends BatchProcessedTagTest<WikipediaAga
).setIcon(WikipediaPlugin.LOGO);
public WikipediaAgainstWikidata() {
super("Check wikipedia=* is interwiki link of wikidata=*", "make sure that the wikipedia=* article is connected to the wikidata=* item");
super(
I18n.tr("Check wikipedia=* is interwiki link of wikidata=*"),
I18n.tr("Makes sure that the wikipedia=* article is connected to the wikidata=* item")
);
}
@Override
......@@ -65,11 +67,12 @@ public class WikipediaAgainstWikidata extends BatchProcessedTagTest<WikipediaAga
private void checkBatch(final String language, final List<TestCompanion> primitiveBatch) {
try {
final Map<String, CheckEntityExistsResult.Entity.Sitelink> sitelinks =
ApiQueryClient.query(
WikidataActionApiUrl.getEntityForSitelink(language + "wiki", primitiveBatch.stream().map(it -> it.title).collect(Collectors.toList())),
SerializationSchema.WBGETENTITIES
).getEntities().values().stream()
final Map<String, CheckEntityExistsResult.Entity.Sitelink> sitelinks = ApiQueryClient
.query(WikidataActionApiQuery.wbgetentities(
language + "wiki",
primitiveBatch.stream().map(it -> it.title).collect(Collectors.toList())
))
.getEntities().values().stream()
.flatMap(entity -> entity.getSitelinks().stream().map(it -> Pair.create(entity.getId(), it)))
.collect(Collectors.toMap(it -> it.a, it -> it.b));
primitiveBatch.forEach(tc -> {
......
......@@ -53,6 +53,37 @@ public class WikidataActionApiQueryTest {
WikidataActionApiQuery.defaultUrl = oldDefaultUrl;
}
@Test(expected = IllegalArgumentException.class)
public void testWbgetentities_nonQId() {
WikidataActionApiQuery.wbgetentities(Collections.singletonList("X1"));
}
@Test(expected = IllegalArgumentException.class)
public void testCheckEntityExists_nonQId2() {
WikidataActionApiQuery.wbgetentities(Arrays.asList("Q1", "Q2", "X1"));
}
@Test(expected = IllegalArgumentException.class)
public void testWbgetentities_nullId() {
WikidataActionApiQuery.wbgetentities(Arrays.asList("Q1", null, "Q3"));
}
@Test(expected = IllegalArgumentException.class)
public void testWbgetentities_emptyIdList() {
WikidataActionApiQuery.wbgetentities(Collections.emptyList());
}
@Test
public void testWbgetentitiesQuery() {
assertEquals(
"format=json&utf8=1&formatversion=1&action=wbgetentities&sites=&props=&ids=Q1",
WikidataActionApiQuery.wbgetentities(Collections.singletonList("Q1")).getQuery()
);
assertEquals(
"format=json&utf8=1&formatversion=1&action=wbgetentities&sites=&props=&ids=Q1%7CQ13%7CQ24%7CQ20150617%7CQ42%7CQ12345",
WikidataActionApiQuery.wbgetentities(Arrays.asList("Q1", "Q13", "Q24", "Q20150617", "Q42", "Q12345")).getQuery()
);
}
@Test
public void testWikidataForArticles1() throws IOException, URISyntaxException {
......
// License: GPL. For details, see LICENSE file.
package org.wikipedia.api.wikidata_action;
import static org.junit.Assert.assertEquals;
import java.util.Arrays;
import java.util.Collections;
import org.junit.Test;
public class WikidataActionApiUrlTest {
@Test(expected = IllegalArgumentException.class)
public void testCheckEntityExists_nonQId() {
WikidataActionApiUrl.checkEntityExistsUrl(Collections.singletonList("X1"));
}
@Test(expected = IllegalArgumentException.class)
public void testCheckEntityExists_nonQId2() {
WikidataActionApiUrl.checkEntityExistsUrl(Arrays.asList("Q1", "Q2", "X1"));
}
@Test(expected = IllegalArgumentException.class)
public void testCheckEntityExists_nullId() {
WikidataActionApiUrl.checkEntityExistsUrl(Arrays.asList("Q1", null, "Q3"));
}
@Test(expected = IllegalArgumentException.class)
public void testCheckEntityExists_emptyIdList() {
WikidataActionApiUrl.checkEntityExistsUrl(Collections.emptyList());
}
@Test
public void testCheckEntityExistsUrl() {
assertEquals(
"https://www.wikidata.org/w/api.php?format=json&utf8=1&formatversion=1&action=wbgetentities&sites=&props=&ids=Q1",
WikidataActionApiUrl.checkEntityExistsUrl(Collections.singletonList("Q1")).toString()
);
assertEquals(
"https://www.wikidata.org/w/api.php?format=json&utf8=1&formatversion=1&action=wbgetentities&sites=&props=&ids=Q1%7CQ42",
WikidataActionApiUrl.checkEntityExistsUrl(Arrays.asList("Q1", "Q42")).toString()
);
assertEquals(
"https://www.wikidata.org/w/api.php?format=json&utf8=1&formatversion=1&action=wbgetentities&sites=&props=&ids=Q1%7CQ42%7CQ12345",
WikidataActionApiUrl.checkEntityExistsUrl(Arrays.asList("Q1", "Q42", "Q12345")).toString()
);
assertEquals(
"https://www.wikidata.org/w/api.php?format=json&utf8=1&formatversion=1&action=wbgetentities&sites=&props=&ids=Q1%7CQ13%7CQ24%7CQ20150617%7CQ42%7CQ12345",
WikidataActionApiUrl.checkEntityExistsUrl(Arrays.asList("Q1", "Q13", "Q24", "Q20150617", "Q42", "Q12345")).toString()
);
}
}
// License: GPL. For details, see LICENSE file.
package org.wikipedia.api.wikidata_action.json;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import org.junit.Test;
public class CheckEntityExistsResultTest {
@Test
public void test() throws IOException {
final CheckEntityExistsResult enitityQueryResult = SerializationSchema.WBGETENTITIES.getMapper().readValue(
CheckEntityExistsResultTest.class.getResourceAsStream("../response/wbgetentities/Q1_Q1234567.json"),
SerializationSchema.WBGETENTITIES.getSchemaClass()
);
assertEquals(1, enitityQueryResult.getSuccess());
assertEquals(1, enitityQueryResult.getEntities().size());
assertEquals(1, enitityQueryResult.getMissingEntities().size());
assertEquals("Q1", enitityQueryResult.getEntities().get("Q1").getId());
assertEquals("item", enitityQueryResult.getEntities().get("Q1").getType());
assertEquals("Q1234567", enitityQueryResult.getMissingEntities().iterator().next().getId());
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment