Commit 5b91be96 authored by Adam P. Goucher's avatar Adam P. Goucher

Common names from wiki

parent 7c355404
Pipeline #41051756 (#47) passed with stage
in 3 minutes and 59 seconds
......@@ -47,4 +47,5 @@ if __name__ == '__main__':
prepare_payosha(argv[1])
upload_commonnames("all-common-names.txt", "%s OVERWRITE std" % argv[1])
upload_commonnames("wiki/wiki-names.txt", "%s UPDATE std" % argv[1])
upload_commonnames("all-badges.txt", "%s OVERWRITE badges" % argv[1])
#!/bin/bash
baseurl='http://conwaylife.com/w/index.php?title=Category:Patterns&'
urlsuffix='pagefrom=0'
while [ ! -z "$urlsuffix" ]; do
curl "$baseurl$urlsuffix" > "download.txt"
cat "download.txt" | grep -o '<li><a href="/wiki/[^"]*"' | grep -v ':' | grep -o '"[^"]*"'
urlsuffix="$( cat "download.txt" | grep -o 'pagefrom=[^"]*#' | grep -o '[^#]*' | head -n 1 )"
done
import subprocess
def download_url(pname):
x = subprocess.check_output("curl 'http://conwaylife.com/wiki/" + pname + "?action=edit' 2>/dev/null | grep '^|.*='", shell=True)
return dict([tuple([z.strip() for z in y.split('=')[:2]]) for y in x.split('\n') if '=' in y])
def process_line(l):
l = l.replace('"','').strip()
if l[:6] == '/wiki/':
l = l[6:]
if len(l) == 0:
return
d = download_url(l)
if '|name' not in d:
return
if '|apgcode' not in d:
return
common_name = d['|name'].replace('"', '').replace('!','')
apgcode = d['|apgcode']
print('"%s" : "%s!%s",' % (apgcode, common_name, l))
if __name__ == '__main__':
with open('wikipages.txt', 'r') as f:
for l in f:
process_line(l)
This diff is collapsed.
This diff is collapsed.
......@@ -39,7 +39,7 @@ public class CommonNames {
datastore.put(commonnameEntity);
}
public static void getNamemap(String namespace, Map<String, String> namemap) {
public static void getNamemap(String namespace, Map<String, String> namemap, boolean mogrify) {
DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
......@@ -57,7 +57,15 @@ public class CommonNames {
if (line.length() >= 2) {
String[] parts = line.split(" ", 2);
String description = parts[1];
if (mogrify) {
String[] parts2 = description.split("!");
if (parts2.length == 2) {
description = "<a href=\"http://conwaylife.com/wiki/" + parts2[1] + "\">" + parts2[0] + "</a>";
}
}
namemap.put(parts[0], parts[1]);
}
......
......@@ -433,7 +433,7 @@ public class CensusServlet extends ExampleServlet {
try {
Map<String, String> namemap = new HashMap<String, String>();
CommonNames.getNamemap("std", namemap);
CommonNames.getNamemap("std", namemap, true);
Key tabulationKey = new KeyFactory.Builder("Census", rulestring + "/" + symmetry).addChild("Tabulation", tabprefix).getKey();
Entity tabulation = datastore.get(tabulationKey);
......
......@@ -39,7 +39,7 @@ public class CommonNamesServlet extends HttpServlet {
Map<String, String> namemap = new HashMap<String, String>();
if (command.equals("UPDATE")) {
CommonNames.getNamemap(namespace, namemap);
CommonNames.getNamemap(namespace, namemap, false);
} else if (command.equals("OVERWRITE")) {
// do nothing;
} else {
......@@ -54,7 +54,7 @@ public class CommonNamesServlet extends HttpServlet {
}
}
CommonNames.putNamemap(namespace, namemap);
CommonNames.putNamemap(namespace, namemap);
writer.println("Operation successful.");
}
......
......@@ -256,6 +256,16 @@ public class ExampleServlet extends HttpServlet {
throws IOException {
String title = getTitle(req);
String title2 = title;
String[] parts2 = title.split("!");
if (parts2.length == 2) {
title = parts2[1];
title2 = parts2[1] + parts2[0];
} else if (parts2.length >= 3) {
title = parts2[1];
title2 = "<a href=\"http://conwaylife.com/wiki/" + parts2[2] + "\">" + parts2[1] + "</a>" + parts2[0];
}
// This contains the template for Catagolue pages:
resp.setContentType("text/html");
......@@ -315,7 +325,7 @@ public class ExampleServlet extends HttpServlet {
writeNavbars(writer, req);
writer.println("<div id=\"content\">");
writer.println("<h2 style=\"overflow-wrap:break-word\">" + title + "</h2>");
writer.println("<h2 style=\"overflow-wrap:break-word\">" + title2 + "</h2>");
writeContent(writer, req);
writer.println("</div>");
......
......@@ -57,7 +57,7 @@ public class GolstatsServlet extends ExampleServlet {
String symmetry = "C1";
Map<String, String> namemap = new HashMap<String, String>();
CommonNames.getNamemap("std", namemap);
CommonNames.getNamemap("std", namemap, true);
Key censusKey = KeyFactory.createKey("Census", rulestring + "/" + symmetry);
......
......@@ -222,7 +222,7 @@ public class HaulServlet extends ExampleServlet {
writer.println("<tr><th>Rank</th>"+(includeImages ? "<th>Image</th>" : "")+"<th>Object</th><th>Occurrences</th></tr>");
Map<String, String> namemap = new HashMap<String, String>();
CommonNames.getNamemap("std", namemap);
CommonNames.getNamemap("std", namemap, true);
int i = 0;
......
......@@ -130,10 +130,10 @@ public class ObjectServlet extends ExampleServlet {
} else {
Map<String, String> namemap = new HashMap<String, String>();
CommonNames.getNamemap("std", namemap);
CommonNames.getNamemap("std", namemap, false);
if (namemap.containsKey(apgcode)) {
return namemap.get(apgcode);
return " (" + apgcode + ")!" + namemap.get(apgcode);
} else {
return apgcode;
}
......
......@@ -177,7 +177,7 @@ public class UserServlet extends ExampleServlet {
List<Entity> badges = preparedQuery.asList(FetchOptions.Builder.withLimit(1000));
Map<String, String> namemap = new HashMap<String, String>();
CommonNames.getNamemap("badges", namemap);
CommonNames.getNamemap("badges", namemap, false);
if (badges.size() > 0) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment