Commit 380b1d93 authored by Dmitry Mozzherin's avatar Dmitry Mozzherin
Browse files

Fix #33 REST and web interfaces

REST interface is added to gnparser itself, and the web interface is
implemented as a Ruby Sinatra app at
https://gitlab.com/gnames/gnparser_web
parent b6481501
......@@ -2,6 +2,8 @@
## Unreleased
- Add [#33]: Web-based user interface and REST API
## [v0.6.0]
- Add [#35]: gRPC method to preserve order in output according to input
......
......@@ -36,7 +36,7 @@ peg:
goimports -w grammar.peg.go; \
asset:
cd dict; \
cd fs; \
$(FLAGS_SHARED) go run -tags=dev assets_gen.go
build: version peg grpc asset
......
......@@ -276,6 +276,39 @@ gnparser -g 8989 -j 20
For an example how to use gRPC server check ``gnparser`` [Ruby gem][gnparser
ruby] as well as [gRPC documentation].
## Usage as a REST API Interface
Use web-server REST API as a slower, but more wide-spread alternative to
gRPC server. Web-based user interface and API are invoked by ``--web-port`` or
``-w`` flag. To start web server on ``http://0.0.0.0:9000``
```bash
gnparser -w 9000
```
Opening a browser with this address will now show an interactive interface
to parser. API calls would be accessibe on ``http://0.0.0.0:9000/api``.
Make sure to CGI-escape name-strings for GET requests. An '&' character
needs to be converted to '%26'
- ``GET /api?q=Aus+bus|Aus+bus+D.+%26+M.,+1870``
- ``POST /api`` with request body of JSON array of strings
```ruby
require 'json'
require 'net/http'
uri = URI('https://parser.globalnames.org/api')
http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true
request = Net::HTTP::Post.new(uri, 'Content-Type' => 'application/json',
'accept' => 'json')
request.body = ['Solanum mariae Särkinen & S.Knapp',
'Ahmadiago Vánky 2004'].to_json
response = http.request(request)
```
## Use as a library in Go
```go
......
// +build !dev
package dict
import (
"bufio"
"log"
"gitlab.com/gogna/gnparser/fs"
)
// Dict contains loaded dictionaries
......@@ -34,7 +34,7 @@ func readBacterialData() map[string]bool {
}
func scanFile(path string, isHomonym bool, m map[string]bool) {
f, err := assets.Open(path)
f, err := fs.Files.Open(path)
if err != nil {
log.Fatal(err)
}
......
// +build dev
package dict
package fs
import (
"net/http"
......
<?xml version="1.0" encoding="utf-8"?>
<!-- Generator: Adobe Illustrator 19.1.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
viewBox="-130 172 350 450" style="enable-background:new -130 172 350 450;" xml:space="preserve">
<style type="text/css">
.st0{stroke:#FFFFFF;stroke-width:2;stroke-miterlimit:10;}
.st1{fill:#007934;stroke:#FFFFFF;stroke-width:2;stroke-miterlimit:10;}
.st2{fill:#FDBA12;stroke:#FFFFFF;stroke-width:2;stroke-miterlimit:10;}
.st3{fill:#001689;stroke:#FFFFFF;stroke-width:2;stroke-miterlimit:10;}
</style>
<path class="st0" d="M-111.2,481.3c0-12.7,4.7-22.7,13.9-30.1c9.3-7.4,22-11,38-11h53.1v18.4h-28.1c4.7,4.8,7.9,9,9.7,12.9
c1.8,3.9,2.7,8.4,2.7,13.4c0,6.2-1.8,12.3-5.4,18.3c-3.6,6-8.1,10.6-13.7,13.8c-5.6,3.2-14.7,5.7-27.3,7.7
c-8.9,1.3-13.4,4.3-13.4,9.1c0,2.8,1.7,5,5,6.8c3.3,1.8,9.4,3.6,18.1,5.5c14.6,3.2,24,5.7,28.1,7.5c4.2,1.8,7.9,4.5,11.4,7.8
c5.7,5.7,8.6,12.8,8.6,21.4c0,11.3-5,20.3-15,27s-23.5,10.1-40.4,10.1c-16.9,0-30.5-3.3-40.7-10.1s-15.2-15.8-15.2-27.2
c0-16.1,9.9-26.4,29.9-31.1c-7.9-5.1-11.9-10.1-11.9-15.2c0-3.8,1.7-7.2,5.1-10.4c3.5-3.1,8-5.4,13.8-6.9
C-102.5,511.2-111.2,498.6-111.2,481.3z M-72.5,564.7c-7.9,0-14.4,1.7-19.5,5c-5,3.3-7.6,7.7-7.6,12.9c0,12.3,11,18.4,33.1,18.4
c10.5,0,18.5-1.6,24.3-4.6c5.7-3,8.6-7.4,8.6-12.9c0-5.5-3.6-10-10.8-13.5C-51.6,566.5-61,564.7-72.5,564.7z M-66.9,460.6
c-6.4,0-11.8,2.1-16.3,6.5c-4.5,4.3-6.7,9.5-6.7,15.5c0,6.1,2.2,11.1,6.6,15.2c4.5,4,9.9,6,16.7,6c6.7,0,12.1-2,16.6-6.1
c4.5-4.1,6.7-9.3,6.7-15.4c0-6.2-2.2-11.4-6.7-15.5C-54.6,462.7-60.2,460.6-66.9,460.6z"/>
<path class="st0" d="M22.8,440.2v15.2c10.5-11.6,22.5-17.5,35.9-17.5c7.5,0,14.3,1.9,20.7,5.8c6.4,3.8,11.3,9.1,14.6,15.8
c3.3,6.7,5,17.3,5,31.8v68.1H75.5v-67.8c0-12.1-1.9-20.8-5.6-26.1c-3.7-5.2-9.9-7.8-18.6-7.8c-11.1,0-20.6,5.6-28.3,16.7v85H-1
V440.2H22.8z"/>
<path class="st0" d="M188.6,487.7v50.6c0,4,1.3,6.1,4.1,6.1c2.8,0,7.2-2.1,13.3-6.4v14.4c-5.4,3.5-9.6,5.8-12.8,7
c-3.2,1.2-6.6,1.9-10.1,1.9c-10.1,0-16.1-3.9-17.8-11.9c-10,7.8-20.6,11.6-32,11.6c-8.2,0-15.2-2.7-20.7-8.2
c-5.6-5.5-8.2-12.4-8.2-20.6c0-7.5,2.7-14.2,8.1-20.1c5.4-5.9,13-10.6,23-14l30.1-10.4v-6.4c0-14.3-7.1-21.5-21.5-21.5
c-12.8,0-27,5.2-39.2,18.5c0,0-0.7-7-2.2-13.3c-1-4.1-3.2-9-3.2-9c9.1-10.8,29.4-18.3,46.6-18.3c12.8,0,23.2,3.3,30.9,10.1
c2.6,2.1,4.9,5,7,8.6c2.1,3.6,3.3,7.1,3.9,10.7C188.3,470.8,188.6,477.6,188.6,487.7z M165.5,535.6v-35.3l-15.8,6.1
c-8,3.2-13.7,6.4-17.1,9.6c-3.3,3.2-5,7.2-5,12.1s1.6,8.9,4.7,12c3.1,3.1,7.2,4.7,12.3,4.7C152.1,544.9,159,541.7,165.5,535.6z"/>
<path class="st1" d="M80.4,305.5c0-14-10.1-32.4-33-32.4c-22.1,0-33,15.5-33,32c0,14.2,11.9,29.6,11.9,29.6s6.4,8,13.3,26.6
c6.5,17.2,7.4,39.9,7.5,35c0.1,5.1,1-17.7,7.5-35c6.9-18.6,13.3-26.6,13.3-26.6S80.4,319.4,80.4,305.5z"/>
<path class="st2" d="M0.9,311.5c-6.9-12.3-24.6-23.4-44.6-12.3C-63,310-65,328.8-57,343.2c6.9,12.4,24.9,20.2,24.9,20.2
s9.5,3.9,24.5,16.8c13.9,11.9,25.9,31.3,23.5,27c2.6,4.5-7.7-15.9-10.5-34.2c-3-19.6-1.3-29.8-1.3-29.8S7.5,323.5,0.9,311.5z"/>
<path class="st3" d="M93.8,311.5c6.9-12.3,24.6-23.4,44.6-12.3c19.3,10.8,21.3,29.6,13.3,44c-6.9,12.4-24.9,20.2-24.9,20.2
s-9.5,3.9-24.6,16.8c-13.9,11.9-25.9,31.3-23.5,27c-2.6,4.5,7.7-15.9,10.5-34.2c3-19.6,1.3-29.8,1.3-29.8S87,323.5,93.8,311.5z"/>
<path class="st2" d="M-37.5,251.8c0,22.5-18.3,40.8-40.8,40.8s-40.8-18.3-40.8-40.8s18.3-40.8,40.8-40.8S-37.5,229.3-37.5,251.8z"/>
<circle class="st1" cx="46.3" cy="214.9" r="40.8"/>
<path class="st3" d="M211.8,249.8c0,22.5-18.3,40.8-40.8,40.8s-40.8-18.3-40.8-40.8S148.4,209,171,209
C193.5,209,211.8,227.3,211.8,249.8z"/>
</svg>
.parser textarea {
width: 100%;
height: 7em;
display: block;
margin-bottom:1em;
}
This diff is collapsed.
{{ define "content" }}
<section class="parser api">
<div class="grid">
<div class="unit whole">
<h2 id="api">Application Programming Interface (API)</h2>
<p>Web-based parser service includes a RESTful interface to parsing
functionalilty. Both GET and POST methods are supported.</p>
<h3 id="get">GET</h3>
<p>
Append a vertical line separated array of strings to your domain url.
Make sure that '&amp;' in the names are escaped as '%26',
and spaces are escaped as '+'.
</p>
<p>
<code>/api?q=Aus+bus|Aus+bus+D.+%26+M.,+1870</code>
</p>
<h3 id="post">POST</h3>
<p><code>/api</code></p>
<p>
with request body of JSON array of strings
</p>
</div>
</div>
</section>
{{ end }}
\ No newline at end of file
{{ define "content" }}
<section class='parser'>
<div class='grid'>
<div class='unit whole'>
<form action='/' method='get'>
<textarea autofocus id='names' name='q' placeholder='Add names, one per line'>{{.Input}}</textarea>
<input type='submit' value='Parse'>
</form>
</div>
</div>
</section>
{{ if .Parsed }}
<section class="parser results">
<div class="grid">
<div class="unit whole">
<h4>Results:</h4>
{{ range .Parsed }}
<p>
<code>{{ . }}</code>
</p>
{{ end }}
</div>
</div>
</section>
{{ end }}
{{ end }}
\ No newline at end of file
{{ define "layout" }}
<!DOCTYPE html>
<html>
<head>
<meta charset='UTF-8'>
<title>GNParser</title>
<meta content='width=device-width,initial-scale=1' name='viewport'>
<link href='/styles/screen.css' rel='stylesheet'>
<link href='/styles/parser.css' rel='stylesheet'>
<link href='/images/favicon.ico' rel='icon' type='image/x-icon'>
</head>
<body class='wrap'>
<header role='banner'>
<nav class='mobile-nav show-on-mobiles'>
<ul>
{{ if .HomePage }} <li class='current'> {{ else }} <li> {{ end }}
<a href='/'>Parser</a>
</li>
{{ if .HomePage }} <li> {{ else }} <li class='current'> {{ end }}
<a href='/doc/api'>API</a>
</li>
<li>
<a href='https://gitlab.com/gogna/gnparser/blob/master/README.md'><span class='hide-on-mobiles'>Doc on</span>
GitHub</a>
</li>
<li>
<a href='http://globalnames.org/apps'>Projects</a>
</li>
</ul>
</nav>
<div class='grid'>
<div class='unit one-quarter center-on-mobiles'>
<div class='logo'>
<a href='/doc/api'>
<span class='sr-only'>GlobalNames</span>
<img alt='GNA Logo' src='/images/gna.svg' width='72'>
</a>
</div>
</div>
<nav class='main-nav unit three-quarters hide-on-mobiles'>
<ul>
{{ if .HomePage }} <li class='current'> {{ else }} <li> {{ end }}
<a href='/'>Parser</a>
</li>
{{ if .HomePage }} <li> {{ else }} <li class='current'> {{ end }}
<a href='/doc/api'>API</a>
</li>
<li>
<a href='https://gitlab.com/gogna/gnparser/blob/master/README.md'><span class='hide-on-mobiles'>Doc on</span>
GitHub</a>
</li>
<li>
<a href='http://globalnames.org/apps'>Projects</a>
</li>
</ul>
</nav>
</div>
</header>
<section class='intro'>
<div class='grid'>
<div class='unit whole center-on-mobiles'>
<h1>Global Names Parser <span style="color:#aaa">({{ .Version }})</span></h1>
<h4>Scientific Names in Detail</h4>
</div>
</div>
</section>
{{ template "content" . }}
<section class='footer'>
<div class='grid'>
<div class='unit whole center-on-mobiles'>
This is a newer <em>gnparser</em> implementation
<a href="https://gitlab.com/gogna/gnparser">written in Go</a>.
To access <a href="https://github.com/GlobalNamesArchitecture/gnparser">
<em>gnparser</em> written in Scala</a> try this
<a href="scala.parser.globalnames.org">web-service</a>.
</div>
</div>
</section>
</body>
</html>
{{ end }}
\ No newline at end of file
......@@ -6,14 +6,14 @@ import (
"log"
"github.com/shurcool/vfsgen"
"gitlab.com/gogna/gnparser/dict"
"gitlab.com/gogna/gnparser/fs"
)
func main() {
err := vfsgen.Generate(dict.Assets, vfsgen.Options{
PackageName: "dict",
err := vfsgen.Generate(fs.Assets, vfsgen.Options{
PackageName: "fs",
BuildTags: "!dev",
VariableName: "assets",
VariableName: "Files",
})
if err != nil {
log.Fatalln(err)
......
This diff is collapsed.
......@@ -32,6 +32,7 @@ import (
"github.com/spf13/cobra"
"gitlab.com/gogna/gnparser"
"gitlab.com/gogna/gnparser/grpc"
"gitlab.com/gogna/gnparser/web"
)
// rootCmd represents the base command when called without any subcommands
......@@ -59,6 +60,7 @@ gnparser -j 10 -g 3355
Run: func(cmd *cobra.Command, args []string) {
versionFlag(cmd)
wn := workersNumFlag(cmd)
grpcPort := grpcFlag(cmd)
if grpcPort != 0 {
fmt.Println("Running gnparser as gRPC service:")
......@@ -66,15 +68,23 @@ gnparser -j 10 -g 3355
fmt.Printf("jobs: %d\n\n", wn)
grpc.Run(grpcPort, wn)
os.Exit(0)
} else {
f := formatFlag(cmd)
opts := []gnparser.Option{
gnparser.WorkersNum(wn),
gnparser.Format(f),
}
data := getInput(cmd, args)
parse(data, opts)
}
webPort := webFlag(cmd)
if webPort != 0 {
fmt.Println("Running gnparser as a website and REST server:")
fmt.Printf("port: %d\n", webPort)
fmt.Printf("jobs: %d\n\n", wn)
web.Run(webPort, wn)
os.Exit(0)
}
f := formatFlag(cmd)
opts := []gnparser.Option{
gnparser.WorkersNum(wn),
gnparser.Format(f),
}
data := getInput(cmd, args)
parse(data, opts)
},
}
......@@ -101,7 +111,10 @@ func init() {
rootCmd.Flags().IntP("jobs", "j", dj,
"Nubmer of threads to run. CPU's threads number is the default.")
rootCmd.Flags().IntP("grpc_port", "g", 0, "starts grpc server on the port, ignores other flags.")
rootCmd.Flags().IntP("grpc_port", "g", 0, "starts gRPC server on the port.")
rootCmd.Flags().IntP("web_port", "w", 0,
"starts web site and REST server on the port.")
}
func versionFlag(cmd *cobra.Command) {
......@@ -127,6 +140,15 @@ func grpcFlag(cmd *cobra.Command) int {
return grpcPort
}
func webFlag(cmd *cobra.Command) int {
webPort, err := cmd.Flags().GetInt("web_port")
if err != nil {
fmt.Println(err)
os.Exit(1)
}
return webPort
}
func formatFlag(cmd *cobra.Command) string {
str, err := cmd.Flags().GetString("format")
if err != nil {
......
......@@ -3,6 +3,7 @@ module gitlab.com/gogna/gnparser
require (
github.com/gnames/uuid5 v0.1.1
github.com/golang/protobuf v1.2.0
github.com/gorilla/mux v1.6.2
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/json-iterator/go v1.1.5
github.com/mitchellh/go-homedir v1.0.0 // indirect
......@@ -13,7 +14,7 @@ require (
github.com/pointlander/compress v1.1.0 // indirect
github.com/pointlander/jetset v1.0.0 // indirect
github.com/pointlander/peg v1.0.1-0.20181228211923-fa48cc294fa1 // indirect
github.com/shurcooL/httpfs v0.0.0-20181222201310-74dc9339e414 // indirect
github.com/shurcooL/httpfs v0.0.0-20181222201310-74dc9339e414
github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd // indirect
github.com/shurcool/vfsgen v0.0.0-20181202132449-6a9ea43bcacd // indirect
github.com/spf13/cobra v0.0.4-0.20190109003409-7547e83b2d85
......
package web
import (
"fmt"
"net/http"
"strings"
"sync"
"github.com/gorilla/mux"
jsoniter "github.com/json-iterator/go"
"gitlab.com/gogna/gnparser"
)
func apiEmptyRequest(w http.ResponseWriter, r *http.Request) {
res := `{"error": "Unrecognized request"}`
fmt.Fprint(w, res)
}
func apiGetParse(w http.ResponseWriter, r *http.Request) {
params := mux.Vars(r)
namesPipe, ok := params["q"]
if !ok {
fmt.Fprint(w, "[]\n")
return
}
names := strings.Split(namesPipe, "|")
parseSlice(w, names)
}
func apiPostParse(w http.ResponseWriter, r *http.Request) {
var names []string
_ = jsoniter.NewDecoder(r.Body).Decode(&names)
if names == nil || len(names) == 0 {
fmt.Fprint(w, "[]\n")
return
}
parseSlice(w, names)
}
func parseSlice(w http.ResponseWriter, ns []string) {
in := make(chan string)
out := make(chan *gnparser.ParseResult)
gnp := gnparser.NewGNparser()
var wg sync.WaitGroup
wg.Add(1)
go gnp.ParseStream(in, out)
go processResults(w, out, &wg)
for _, v := range ns {
in <- v
}
close(in)
wg.Wait()
}
func processResults(w http.ResponseWriter, out <-chan *gnparser.ParseResult, wg *sync.WaitGroup) {
defer wg.Done()
var res []string
for r := range out {
if r.Error == nil {
res = append(res, r.Output)
}
}
fmt.Fprint(w, "[\n"+strings.Join(res, ",\n")+"]\n")
}
package web
import (
"log"
"net/http"
"strconv"
"time"
"github.com/gorilla/mux"
"gitlab.com/gogna/gnparser/fs"
)
// Run starts RESTful service on /api route for both GET and
// POST verbs
func Run(port int, wn int) {
router := mux.NewRouter()
router.HandleFunc("/", home)
router.HandleFunc("/doc/api", docAPI)
router.HandleFunc("/api", apiGetParse).Methods("GET").Queries("q", "{q}")
router.HandleFunc("/api", apiPostParse).Methods("POST")
router.HandleFunc("/api", apiEmptyRequest)
router.PathPrefix("/").Handler(http.FileServer(fs.Files))
srv := &http.Server{
Handler: router,
Addr: "0.0.0.0:" + strconv.Itoa(port),
// Good practice: enforce timeouts for servers you create!
WriteTimeout: 60 * time.Second,
ReadTimeout: 15 * time.Second,
}
log.Fatal(srv.ListenAndServe())
}
package web
import (
"fmt"
"html/template"
"net/http"
"strings"
"github.com/shurcooL/httpfs/html/vfstemplate"
"gitlab.com/gogna/gnparser"
"gitlab.com/gogna/gnparser/fs"
"gitlab.com/gogna/gnparser/output"
)
type Data struct {
Input string
Parsed []string
HomePage bool
Version string
}
func NewData() *Data {
return &Data{Version: output.Version}
}
func home(w http.ResponseWriter, r *http.Request) {
data := NewData()
data.HomePage = true
params := r.URL.Query()
if txt, ok := params["q"]; ok && len(txt) > 0 {
fmt.Println(txt)
names := namesFromText(txt[0])
data.Input = txt[0]
data.HomePage = true
data.Parsed = parseForWeb(names)
}
var t *template.Template
t, err := vfstemplate.ParseFiles(fs.Files, t, "templates/layout.html",
"templates/home.html")
if err != nil {
fmt.Fprintf(w, "<html><body><h1>Error</h2><p>%s</p></body></html>",
err.Error())
return
}
err = t.ExecuteTemplate(w, "layout", data)
if err != nil {
fmt.Fprintf(w, "<html><body><h1>Error</h2><p>%s</p></body></html>",
err.Error())
return
}
}
func docAPI(w http.ResponseWriter, r *http.Request) {
var t *template.Template
t, err := vfstemplate.ParseFiles(fs.Files, t, "templates/layout.html",
"templates/doc_api.html")
if err != nil {
fmt.Fprintf(w, "<html><body><h1>Error</h2><p>%s</p></body></html>",
err.Error())
return
}
data := NewData()
t.ExecuteTemplate(w, "layout", data)
}
func parseForWeb(names []string) []string {
parsed := make([]string, len(names))
opts := []gnparser.Option{gnparser.Format("pretty")}
gnp := gnparser.NewGNparser(opts...)
for i, v := range names {
json, err := gnp.ParseAndFormat(v)
if err != nil {
parsed[i] = err.Error()
continue
}
parsed[i] = json
}
return parsed
}
func namesFromText(txt string) []string {
var names []string
names = strings.Split(txt, "|")
if len(names) > 1 {
return names
}
names = names[0:0]
strs := strings.Split(txt, "\n")
for _, v := range strs {
v = strings.TrimRight(v, "\r ")
fmt.Printf("'%s'", v)