Commit 79aabb4b authored by Florian Schäfer's avatar Florian Schäfer

Add validator check to ensure a Wikidata item exists and is no redirect

This validator check needs a network connection, if the API requests fail, this is noted in the 'Other' category of the validator.
It provides an auto-fix if the wikidata=* tag points to a redirect. If the Wikidata item does not exist, the users have to fix it on their own.
The class AllValidatorTests will serve as a directory where all possible TestErrors are listed.
parent 42c77062
......@@ -3,6 +3,8 @@ package org.wikipedia;
import javax.swing.JMenu;
import org.openstreetmap.josm.data.Version;
import org.openstreetmap.josm.data.validation.OsmValidator;
import org.openstreetmap.josm.gui.MainApplication;
import org.openstreetmap.josm.gui.MainMenu;
import org.openstreetmap.josm.gui.MapFrame;
......@@ -18,13 +20,19 @@ import org.wikipedia.gui.SophoxServerPreference;
import org.wikipedia.gui.WikidataItemSearchDialog;
import org.wikipedia.gui.WikidataTagCellRenderer;
import org.wikipedia.gui.WikipediaToggleDialog;
import org.wikipedia.validator.WikidataItemExists;
public class WikipediaPlugin extends Plugin {
public final class WikipediaPlugin extends Plugin {
private static String name;
private static String versionInfo;
private PreferenceSetting preferences;
public WikipediaPlugin(PluginInformation info) {
super(info);
versionInfo = String.format("JOSM/%s & JOSM-wikipedia/%s", Version.getInstance().getVersionString(), info.version);
name = info.name;
new WikipediaCopyTemplate();
JMenu dataMenu = MainApplication.getMenu().dataMenu;
MainMenu.add(dataMenu, new WikipediaAddNamesAction());
......@@ -32,6 +40,16 @@ public class WikipediaPlugin extends Plugin {
MainMenu.add(dataMenu, new WikidataItemSearchDialog.Action());
DownloadDialog.addDownloadSource(new SophoxDownloadReader());
OsmValidator.addTest(WikidataItemExists.class);
}
public static String getVersionInfo() {
return versionInfo;
}
public static String getName() {
return name;
}
@Override
......
// License: GPL. For details, see LICENSE file.
package org.wikipedia.validator;
import org.openstreetmap.josm.data.validation.Severity;
import org.openstreetmap.josm.data.validation.Test;
import org.openstreetmap.josm.data.validation.TestError;
import org.openstreetmap.josm.tools.I18n;
class AllValidationTests {
static final ValidationTest<WikidataItemExists> INVALID_QID = new ValidationTest<>(Severity.ERROR, 30_000);
static final ValidationTest<WikidataItemExists> API_REQUEST_FAILED = new ValidationTest<>(Severity.OTHER, 30_001);
static final ValidationTest<WikidataItemExists> WIKIDATA_ITEM_DOES_NOT_EXIST = new ValidationTest<>(Severity.ERROR, 30_002);
static final ValidationTest<WikidataItemExists> WIKIDATA_ITEM_IS_REDIRECT = new ValidationTest<>(Severity.WARNING, 30_003);
// i18n: Prefix for the validator messages. Note the space at the end!
static final String VALIDATOR_MESSAGE_MARKER = I18n.tr("[Wiki] ");
private AllValidationTests() {
// Private constructor to avoid instantiation
}
static class ValidationTest<T extends Test> {
private Severity severity;
private int code;
ValidationTest(final Severity severity, final int code) {
this.severity = severity;
this.code = code;
}
TestError.Builder getBuilder(final T test) {
return TestError.builder(test, severity, code);
}
}
}
// License: GPL. For details, see LICENSE file.
package org.wikipedia.validator;
import static org.wikipedia.validator.AllValidationTests.VALIDATOR_MESSAGE_MARKER;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import org.openstreetmap.josm.command.ChangePropertyCommand;
import org.openstreetmap.josm.data.osm.OsmPrimitive;
import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
import org.openstreetmap.josm.data.validation.Test;
import org.openstreetmap.josm.gui.Notification;
import org.openstreetmap.josm.tools.I18n;
import org.openstreetmap.josm.tools.ImageProvider;
import org.wikipedia.api.wikidata_action.ApiQueryClient;
import org.wikipedia.api.wikidata_action.WikidataActionApiUrl;
import org.wikipedia.api.wikidata_action.json.CheckEntityExistsResult;
import org.wikipedia.tools.RegexUtil;
/**
* Checks if for the wikidata=* tag on an {@link OsmPrimitive} a Wikidata item really exists.
* This check requires a working internet connection, because it queries the Wikidata Action API.
*/
public class WikidataItemExists extends Test.TagTest {
private static final int CHUNK_SIZE = 50;
private List<OsmPrimitive> primitivesForChunks = new ArrayList<>();
public WikidataItemExists() {
//super("wikipedia=* is interwiki link of wikidata=*", "make sure that the wikipedia=* article is connected to the wikidata=* item");
super(I18n.tr("wikidata=* item exists"), I18n.tr("Make sure the Wikidata item for the Q-ID in the wikidata=* tag actually exists"));
}
/**
* Checks one "chunk" of {@link OsmPrimitive}s at a time.
* The checks that are done here are requiring API requests and thus are grouped into these chunks.
* @param primitives the primitives that belong to the current chunk and should be checked now.
* @return {@code false} if there were any issues with downloading or decoding the API response, {@code true} otherwise
*/
private boolean checkChunk(final List<? extends OsmPrimitive> primitives) {
boolean result = true;
final List<String> qIds = primitives.stream().map(it -> {
final String wdValue = it.get("wikidata");
return RegexUtil.isValidQId(wdValue) ? wdValue : null;
}).collect(Collectors.toList());
if (qIds.stream().anyMatch(Objects::nonNull)) {
try {
final URL url = WikidataActionApiUrl.checkEntityExistsUrl(qIds.stream().filter(Objects::nonNull).collect(Collectors.toList()));
final CheckEntityExistsResult entityQueryResult = ApiQueryClient.query(url, CheckEntityExistsResult.class);
if (entityQueryResult.getSuccess() != 1) {
errors.add(AllValidationTests.API_REQUEST_FAILED.getBuilder(this).primitives(new ArrayList<>(primitives)).message(VALIDATOR_MESSAGE_MARKER + I18n.tr("The Wikidata Action API reports a failed query!")).build());
} else {
for (int i = 0; i < Math.min(qIds.size(), primitives.size()); i++) {
final OsmPrimitive primitive = primitives.get(i);
final String qId = qIds.get(i);
check(primitive, qId, entityQueryResult);
}
}
} catch (IOException e) {
result = false;
errors.add(AllValidationTests.API_REQUEST_FAILED.getBuilder(this).primitives(new ArrayList<>(primitives)).message(VALIDATOR_MESSAGE_MARKER + e.getMessage()).build());
}
}
return result;
}
/**
* Checks an {@link OsmPrimitive} against a given {@link CheckEntityExistsResult}.
* @param primitive the primitive to check
* @param qId the Wikidata ID from the tags of the given {@link OsmPrimitive}
* @param entityQueryResult the result from the Wikidata Action API
*/
private void check(final OsmPrimitive primitive, final String qId, final CheckEntityExistsResult entityQueryResult) {
if (qId != null) {
final CheckEntityExistsResult.Entity entity = entityQueryResult.getEntities().get(qId);
if (entity == null) {
errors.add(AllValidationTests.API_REQUEST_FAILED.getBuilder(this).primitives(primitive).message(VALIDATOR_MESSAGE_MARKER + I18n.tr("The Wikidata Action API did not respond with all requested entities!"), I18n.marktr("Item {0} is missing"), qId).build());
} else if (!qId.equals(entity.getId())) {
errors.add(
AllValidationTests.WIKIDATA_ITEM_IS_REDIRECT.getBuilder(this)
.primitives(primitive)
.message(VALIDATOR_MESSAGE_MARKER + I18n.tr("The Wikidata item is a redirect", qId, entity.getId()), I18n.marktr("Item {0} redirects to {1}"), qId, entity.getId())
.fix(() -> new ChangePropertyCommand(primitive, "wikidata", entity.getId()))
.build()
);
} else if (entity.getType() == null) {
errors.add(
AllValidationTests.WIKIDATA_ITEM_DOES_NOT_EXIST.getBuilder(this)
.primitives(primitive)
.message(I18n.tr("The Wikidata item does not exist! Replace the wikidata=* tag with an existing Wikidata item or remove the Wikidata tag."), I18n.marktr("Item {0} does not exist!"), qId)
.build()
);
}
}
}
@Override
public void check(OsmPrimitive osmPrimitive) {
final String wdValue = osmPrimitive.get("wikidata");
if (wdValue != null) {
if (RegexUtil.isValidQId(wdValue)) {
primitivesForChunks.add(osmPrimitive);
} else {
errors.add(
AllValidationTests.INVALID_QID.getBuilder(this)
.primitives(osmPrimitive)
.message(VALIDATOR_MESSAGE_MARKER + I18n.tr("Invalid Q-ID! Does not exist on Wikidata."), I18n.marktr("{0} is not a valid Wikidata-ID"))
.build()
);
}
}
}
@Override
public void endTest() {
boolean fullSuccess = true;
final int numPrimitives = primitivesForChunks.size();
final int numChunks = numPrimitives / CHUNK_SIZE + (numPrimitives % CHUNK_SIZE == 0 ? 0 : 1);
for (int chunkIndex = 0; chunkIndex * CHUNK_SIZE < numPrimitives; chunkIndex++) {
progressMonitor.setExtraText(I18n.tr("(chunk {0}/{1} of {2} items)", chunkIndex + 1, numChunks, numPrimitives));
fullSuccess &= checkChunk(primitivesForChunks.subList(chunkIndex * CHUNK_SIZE, Math.min(primitivesForChunks.size(), (chunkIndex + 1) * CHUNK_SIZE)));
}
if (!fullSuccess) {
new Notification(
I18n.tr("Could not validate all wikidata=* tags over the internet.") + "\n" +
(ValidatorPrefHelper.PREF_OTHER.get()
? I18n.tr("See the validator messages of the category ''Other'' for more details.")
: I18n.tr("Turn on the informational level validator messages in the preferences to see more details.")
)
)
.setIcon(ImageProvider.get("dialogs/wikipedia"))
.show();
}
primitivesForChunks.clear();
super.endTest();
}
}
\ No newline at end of file
......@@ -9,7 +9,6 @@ import java.io.IOException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Test;
import org.wikipedia.api.wikidata_action.WikidataActionApiUrlTest;
public class CheckEntityExistsResultTest {
@Test
......
// License: GPL. For details, see LICENSE file.
package org.wikipedia.validator;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import org.junit.Rule;
import org.junit.Test;
import org.openstreetmap.josm.data.osm.Way;
import org.openstreetmap.josm.data.validation.TestError;
import org.openstreetmap.josm.testutils.JOSMTestRules;
public class AllValidationTestsTest {
@Rule
public JOSMTestRules rules = new JOSMTestRules()
.preferences(); // The Severity class needs access to the preferences
@Test
public void testValidationTestFields() throws IllegalAccessException {
AllValidationTests.ValidationTest<WikidataItemExists> test = AllValidationTests.INVALID_QID;
final Field[] fields = AllValidationTests.class.getDeclaredFields();
for (Field field : fields) {
if ((field.getModifiers() & Modifier.PRIVATE) == 0) {
final Object fieldValue = field.get(null);
if (fieldValue instanceof AllValidationTests.ValidationTest) {
System.out.print("Check validation test " + field.getName());
testValidationTestField((AllValidationTests.ValidationTest) fieldValue);
}
}
}
}
@SuppressWarnings("unchecked")
private static void testValidationTestField(final AllValidationTests.ValidationTest validationTest) {
final TestError te = validationTest
.getBuilder(new org.openstreetmap.josm.data.validation.Test("DummyTest"))
.message("dummy message")
.primitives(new Way())
.build();
assertTrue("The code of a validation test must be at least 30,000", te.getCode() >= 30_000);
assertTrue("The code of a validation test must be lower than 31,000", te.getCode() < 31_000);
assertNotNull("The severity of a validation test must be non-null", te.getSeverity());
System.out.println(" \uD83D\uDDF8");
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment