Commit e2cd12c6 authored by Florian Schäfer's avatar Florian Schäfer

Make use of caching for certain API requests

At the moment it's only the `action=sitematrix` API call that will be cached (for 30 days).
The queries for which caching is enabled will be stored in the cache for a certain amount of time (configured per query). After that time passed, the query gets sent to the API again. If that request fails, the cached result will be used even longer. But if it's successful, the cache is updated and the timer restarts.
parent 69f4a630
# The minimum JOSM version this plugin is compatible with (can be any numeric version)
plugin.main.version = 13597
plugin.main.version = 13643
# The JOSM version this plugin is currently compiled against
# Please make sure this version is available at https://josm.openstreetmap.de/download
# The special values "latest" and "tested" are also possible here, but not recommended.
plugin.compile.version = 13860
plugin.compile.version = 13878
plugin.canloadatruntime = true
plugin.author = simon04
plugin.class = org.wikipedia.WikipediaPlugin
......
// License: GPL. For details, see LICENSE file.
package org.wikipedia;
import java.io.File;
import org.apache.commons.jcs.access.CacheAccess;
import org.openstreetmap.josm.data.cache.JCSCacheManager;
import org.openstreetmap.josm.spi.preferences.Config;
public final class Caches {
public static final CacheAccess<String, String> API_RESPONSES = JCSCacheManager.getCache(
"api",
1,
1_000,
new File(Config.getDirs().getCacheDirectory(false), "plugin/wikipedia").getAbsolutePath()
);
private Caches() {
// Private constructor to avoid instantiation
}
}
package org.wikipedia.api;
import java.net.URL;
import org.openstreetmap.josm.tools.HttpClient;
import org.wikipedia.WikipediaPlugin;
import org.wikipedia.api.wikidata_action.json.SerializationSchema;
public abstract class ApiQuery<T> {
private final SerializationSchema<T> schema;
private final URL url;
private final long cacheExpiryTime;
protected ApiQuery(final URL url, final SerializationSchema<T> schema) {
protected ApiQuery(final URL url, final SerializationSchema<T> schema, final long cacheExpiryTime) {
this.schema = schema;
this.url = url;
if (cacheExpiryTime >= 1) {
this.cacheExpiryTime = cacheExpiryTime;
} else {
this.cacheExpiryTime = -1;
}
}
public SerializationSchema<T> getSchema() {
/**
* @return the number of milliseconds for which the API response should be retrieved
* from the cache without trying to get it directly from the API
*/
public final long getCacheExpiryTime() {
return cacheExpiryTime;
}
/**
* A string containing the API endpoint and the query sent to that endpoint.
* Normally this is a URL, but don't rely on that.
* @return the key for which the response will be cached
*/
public String getCacheKey() {
return url.toString();
}
public abstract HttpClient getHttpClient();
/**
* @return the schema that is used to get from the JSON encoded response to its Java representation
*/
public final SerializationSchema<T> getSchema() {
return schema;
}
public URL getUrl() {
/**
* @return the URL that is queried (might not contain the whole query, e.g. when a POST request is made)
*/
public final URL getUrl() {
return url;
}
public String getCacheKey() {
return url.toString();
protected final String getUserAgent(final String[] keywords) {
final String result = String.format(
"JOSM-wikipedia (%s). Report issues at %s.",
WikipediaPlugin.getVersionInfo(),
"https://josm.openstreetmap.de/newticket?component=Plugin%20wikipedia&priority=major&keywords=api"
);
if (keywords == null || keywords.length <= 0) {
return result;
}
return result + "%20" + String.join("%20", keywords);
}
}
package org.wikipedia.api.wikidata_action;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import org.openstreetmap.josm.tools.HttpClient;
import org.openstreetmap.josm.tools.Utils;
import org.wikipedia.api.ApiQuery;
import org.wikipedia.api.ApiUrl;
......@@ -14,14 +15,19 @@ import org.wikipedia.tools.RegexUtil;
public class WikidataActionApiQuery<T> extends ApiQuery<T> {
static URL defaultUrl = ApiUrl.url("https://www.wikidata.org/w/api.php");
private static final String FORMAT_PARAMS = "format=json&utf8=1&formatversion=1";
private static final String[] TICKET_KEYWORDS = {"wikidata", "ActionAPI"};
private final String query;
private WikidataActionApiQuery(final String query, final SerializationSchema<T> schema) {
super(defaultUrl, schema);
private WikidataActionApiQuery(final String query, final SerializationSchema<T> schema, final long cacheExpiryTime) {
super(defaultUrl, schema, cacheExpiryTime);
this.query = query;
}
private WikidataActionApiQuery(final String query, final SerializationSchema<T> schema) {
this(query, schema, -1);
}
public String getQuery() {
return query;
}
......@@ -29,7 +35,8 @@ public class WikidataActionApiQuery<T> extends ApiQuery<T> {
public static WikidataActionApiQuery<SitematrixResult> sitematrix() {
return new WikidataActionApiQuery<>(
FORMAT_PARAMS + "&action=sitematrix",
SerializationSchema.SITEMATRIX
SerializationSchema.SITEMATRIX,
2_592_000_000L // = 1000*60*60*24*30 = number of ms in 30 days
);
}
......@@ -41,8 +48,8 @@ public class WikidataActionApiQuery<T> extends ApiQuery<T> {
throw new IllegalArgumentException("You must supply only Q-IDs as argument to construct a checkEntityExists URL.");
}
return new WikidataActionApiQuery<>(
FORMAT_PARAMS + "&action=wbgetentities&sites=&props=&ids=" + Utils.encodeUrl(String.join("|", qIds)),
SerializationSchema.WBGETENTITIES
FORMAT_PARAMS + "&action=wbgetentities&sites=&props=&ids=" + Utils.encodeUrl(String.join("|", qIds)),
SerializationSchema.WBGETENTITIES
);
}
......@@ -65,4 +72,14 @@ public class WikidataActionApiQuery<T> extends ApiQuery<T> {
public String getCacheKey() {
return getUrl().toString() + '?' + getQuery();
}
@Override
public HttpClient getHttpClient() {
return HttpClient.create(getUrl(), "POST")
.setAccept("application/json")
.setHeader("Content-Type", "text/plain; charset=utf-8")
.setHeader("User-Agent", getUserAgent(TICKET_KEYWORDS))
.setReasonForRequest(String.join(" ", getQuery().split("&")))
.setRequestBody(getQuery().getBytes(StandardCharsets.UTF_8));
}
}
// License: GPL. For details, see LICENSE file.
package org.wikipedia.tools;
import java.io.ByteArrayOutputStream;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
/**
* A {@link FilterInputStream} that redirects all calls to the contained {@link InputStream}
* and whenever a {@code read} method is called, the read bytes are recorded.
* You can receive all bytes that were read so far from the stream by calling the method {@link #getCapturedBytes()}.
*/
public class CapturingInputStream extends FilterInputStream {
private final ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
public CapturingInputStream(final InputStream in) {
super(in);
}
/**
* @return the bytes that the {@link InputStream} has captured so far.
*/
public byte[] getCapturedBytes() {
return byteStream.toByteArray();
}
@Override
public int read() throws IOException {
final int result = super.read();
if (result >= 0) {
byteStream.write(result);
}
return result;
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
final int result = super.read(b, off, len);
byteStream.write(b, off, result);
return result;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment