Commit 1f9a8ce9 authored by Dmitry Mozzherin's avatar Dmitry Mozzherin
Browse files

Fix #54 - add grpc methods for cleanup

parent 4f16c9a6
Pipeline #46125171 passed with stages
in 8 minutes and 9 seconds
......@@ -2,12 +2,16 @@
## Unreleased
## [v0.7.3]
- Add [#54]: add cleaning functions to gRPC
- Add [#46]: add ``supg.`` rank
- Add [#45]: add ``natio`` rank (deprecated ICZN rank)
- Add [#44]: documentation for canonicalName fields
- Add [#42]: tests for command line app
## [v0.7.2]
- Add [#41]: parse/clean multiple names from standard input.
## [v0.7.1]
......@@ -53,12 +57,14 @@
This document follows [changelog guidelines]
[v0.7.3]: https://gitlab.com/gogna/gnparser/compare/v0.7.3...v0.7.2
[v0.7.2]: https://gitlab.com/gogna/gnparser/compare/v0.7.1...v0.7.2
[v0.7.1]: https://gitlab.com/gogna/gnparser/compare/v0.7.0...v0.7.1
[v0.7.0]: https://gitlab.com/gogna/gnparser/compare/v0.6.0...v0.7.0
[v0.6.0]: https://gitlab.com/gogna/gnparser/compare/v0.5.1...v0.6.0
[v0.5.1]: https://gitlab.com/gogna/gnparser/tree/v0.5.1
[#54]: https://gitlab.com/gogna/gnparser/issues/54
[#46]: https://gitlab.com/gogna/gnparser/issues/46
[#45]: https://gitlab.com/gogna/gnparser/issues/45
[#44]: https://gitlab.com/gogna/gnparser/issues/44
......
......@@ -3,10 +3,10 @@ module gitlab.com/gogna/gnparser
require (
cloud.google.com/go v0.35.1 // indirect
dmitri.shuralyov.com/app/changes v0.0.0-20181114035150-5af16e21babb // indirect
dmitri.shuralyov.com/service/change v0.0.0-20190130035545-6cc139383561 // indirect
dmitri.shuralyov.com/service/change v0.0.0-20190203163610-217368fe4577 // indirect
git.apache.org/thrift.git v0.12.0 // indirect
github.com/Shopify/sarama v1.20.1 // indirect
github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 // indirect
github.com/coreos/go-systemd v0.0.0-20190204112023-081494f7ee4f // indirect
github.com/gnames/uuid5 v0.1.1
github.com/go-logfmt/logfmt v0.4.0 // indirect
github.com/gogo/protobuf v1.2.0 // indirect
......@@ -16,7 +16,7 @@ require (
github.com/googleapis/gax-go v2.0.2+incompatible // indirect
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect
github.com/gorilla/mux v1.7.0
github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f // indirect
github.com/gregjones/httpcache v0.0.0-20190203031600-7a902570cb17 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.7.0 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/json-iterator/go v1.1.5
......@@ -33,14 +33,14 @@ require (
github.com/pointlander/peg v1.0.1-0.20181228211923-fa48cc294fa1 // indirect
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 // indirect
github.com/prometheus/common v0.2.0 // indirect
github.com/prometheus/procfs v0.0.0-20190129233650-316cf8ccfec5 // indirect
github.com/prometheus/procfs v0.0.0-20190203183350-488faf799f86 // indirect
github.com/rendon/testcli v0.0.0-20161027181003-6283090d169f
github.com/russross/blackfriday v2.0.0+incompatible // indirect
github.com/shurcooL/go v0.0.0-20190121191506-3fef8c783dec // indirect
github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d // indirect
github.com/shurcooL/highlight_diff v0.0.0-20181222201841-111da2e7d480 // indirect
github.com/shurcooL/highlight_go v0.0.0-20181215221002-9d8641ddf2e1 // indirect
github.com/shurcooL/home v0.0.0-20190127175526-fdd6870b8ab8 // indirect
github.com/shurcooL/home v0.0.0-20190204141146-5c8ae21d4240 // indirect
github.com/shurcooL/htmlg v0.0.0-20190120222857-1e8a37b806f3 // indirect
github.com/shurcooL/httpfs v0.0.0-20181222201310-74dc9339e414
github.com/shurcooL/issues v0.0.0-20190120000219-08d8dadf8acb // indirect
......@@ -64,9 +64,9 @@ require (
golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3
golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1 // indirect
golang.org/x/perf v0.0.0-20190124201629-844a5f5b46f4 // indirect
golang.org/x/sys v0.0.0-20190201152629-afcc84fd7533 // indirect
golang.org/x/sys v0.0.0-20190204203706-41f3e6584952 // indirect
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2
golang.org/x/tools v0.0.0-20190202202610-718ddee9569a // indirect
golang.org/x/tools v0.0.0-20190204195632-d810ce9e47b0 // indirect
google.golang.org/genproto v0.0.0-20190201180003-4b09977fb922 // indirect
google.golang.org/grpc v1.18.0
honnef.co/go/tools v0.0.0-20190128043916-71123fcbb8fe // indirect
......
......@@ -11,6 +11,7 @@ dmitri.shuralyov.com/service/change v0.0.0-20190119025054-42f2f11cf6af/go.mod h1
dmitri.shuralyov.com/service/change v0.0.0-20190124041518-7c09d163af9b/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
dmitri.shuralyov.com/service/change v0.0.0-20190127173012-184eb3bb42b2/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
dmitri.shuralyov.com/service/change v0.0.0-20190130035545-6cc139383561/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
dmitri.shuralyov.com/service/change v0.0.0-20190203163610-217368fe4577/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
git.apache.org/thrift.git v0.0.0-20181218151757-9b75e4fe745a/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
......@@ -30,6 +31,7 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd v0.0.0-20190204112023-081494f7ee4f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
......@@ -81,6 +83,7 @@ github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U=
github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/gregjones/httpcache v0.0.0-20190203031600-7a902570cb17/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
github.com/grpc-ecosystem/grpc-gateway v1.6.2/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
github.com/grpc-ecosystem/grpc-gateway v1.6.4/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
......@@ -162,6 +165,7 @@ github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190129224605-0e558783f66d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190129233650-316cf8ccfec5/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190203183350-488faf799f86/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/rendon/testcli v0.0.0-20161027181003-6283090d169f h1:onGP+qmYmjKs7pkmi9j0mwyr97/D5wki80e74aKIOxg=
github.com/rendon/testcli v0.0.0-20161027181003-6283090d169f/go.mod h1:cq57a4l475CeMvE7RRpSui1MEqCmhirIt1E7kl8BC2Q=
......@@ -186,6 +190,7 @@ github.com/shurcooL/highlight_go v0.0.0-20181215221002-9d8641ddf2e1/go.mod h1:UD
github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
github.com/shurcooL/home v0.0.0-20190120230144-cf17a69b0cc5/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
github.com/shurcooL/home v0.0.0-20190127175526-fdd6870b8ab8/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
github.com/shurcooL/home v0.0.0-20190204141146-5c8ae21d4240/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
github.com/shurcooL/htmlg v0.0.0-20190120222857-1e8a37b806f3/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y=
......@@ -316,6 +321,8 @@ golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc h1:WiYx1rIFmx8c0mXAFtv5D/mHy
golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190201152629-afcc84fd7533 h1:bLfqnzrpeG4usq5OvMCrwTdmMJ6aTmlCuo1eKl0mhkI=
golang.org/x/sys v0.0.0-20190201152629-afcc84fd7533/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190204203706-41f3e6584952 h1:FDfvYgoVsA7TTZSbgiqjAbfPbK47CNHdWl3h/PJtii0=
golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
......@@ -345,6 +352,8 @@ golang.org/x/tools v0.0.0-20190201185910-0e05534988fe h1:vnIBlz9Fwms9Qgql2K0Ar+X
golang.org/x/tools v0.0.0-20190201185910-0e05534988fe/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190202202610-718ddee9569a h1:l5aJW4keRqzZu9Ad3NtNDTtfRhCn0c0aCmFNd463Ihw=
golang.org/x/tools v0.0.0-20190202202610-718ddee9569a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190204195632-d810ce9e47b0 h1:nkN84Yt0YNBRfAvS2WmqXEk8xW2RzlhtbNx6jNfMIOg=
golang.org/x/tools v0.0.0-20190204195632-d810ce9e47b0/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
......
......@@ -27,8 +27,15 @@ enum Format {
Debug = 3;
}
message Cleaned {
string input = 1;
string output = 2;
}
service GNparser {
rpc Ver(Void) returns(Version) {}
rpc Parse(stream Input) returns(stream Output) {}
rpc ParseInOrder(stream Input) returns(stream Output) {}
rpc Clean(stream Input) returns(stream Cleaned) {}
rpc CleanInOrder(stream Input) returns(stream Cleaned) {}
}
......@@ -5,16 +5,30 @@ import (
"io"
"log"
"net"
"strings"
"sync"
"gitlab.com/gogna/gnparser"
"gitlab.com/gogna/gnparser/dict"
"gitlab.com/gogna/gnparser/output"
"gitlab.com/gogna/gnparser/preprocess"
context "golang.org/x/net/context"
"google.golang.org/grpc"
)
type parseStream interface {
Send(*Output) error
Recv() (*Input, error)
grpc.ServerStream
}
type cleanStream interface {
Send(*Cleaned) error
Recv() (*Input, error)
grpc.ServerStream
}
type gnparserServer struct {
WorkersNum int
}
......@@ -27,7 +41,16 @@ func (gnparserServer) Ver(ctx context.Context,
}
func (gnps gnparserServer) Parse(stream GNparser_ParseServer) error {
opts := []gnparser.Option{gnparser.WorkersNum(gnps.WorkersNum)}
wn := gnps.WorkersNum
return gnps.parse(stream, wn)
}
func (gnps gnparserServer) ParseInOrder(stream GNparser_ParseInOrderServer) error {
return gnps.parse(stream, 1)
}
func (gnps gnparserServer) parse(stream parseStream, wn int) error {
opts := []gnparser.Option{gnparser.WorkersNum(wn)}
gnp := gnparser.NewGNparser(opts...)
inCh := make(chan string)
outCh := make(chan *gnparser.ParseResult)
......@@ -65,43 +88,6 @@ func (gnps gnparserServer) Parse(stream GNparser_ParseServer) error {
}
}
func (gnps gnparserServer) ParseInOrder(stream GNparser_ParseInOrderServer) error {
gnp := gnparser.NewGNparser()
firstRecord := true
for {
in, err := stream.Recv()
if err == io.EOF {
return nil
}
if err != nil {
return err
}
switch c := in.Content.(type) {
case *Input_Name:
if firstRecord {
firstRecord = false
}
res, err := gnp.ParseAndFormat(c.Name)
strError := ""
if err != nil {
strError = err.Error()
}
out := &Output{Value: res, Error: strError}
err = stream.Send(out)
if err != nil {
return err
}
case *Input_Format:
if firstRecord {
firstRecord = false
f := c.Format
gnp = gnparser.NewGNparser(gnparser.Format(strFormat(f)))
}
}
}
}
func strFormat(f Format) string {
switch f {
case Format_Compact:
......@@ -116,7 +102,7 @@ func strFormat(f Format) string {
return "compact"
}
func processStream(stream GNparser_ParseServer,
func processStream(stream parseStream,
outCh <-chan *gnparser.ParseResult, wg *sync.WaitGroup) {
defer wg.Done()
for r := range outCh {
......@@ -132,6 +118,56 @@ func processStream(stream GNparser_ParseServer,
}
}
func (gnps gnparserServer) Clean(stream GNparser_CleanServer) error {
return gnps.clean(stream, gnps.WorkersNum)
}
func (gnps gnparserServer) CleanInOrder(stream GNparser_CleanInOrderServer) error {
return gnps.clean(stream, 1)
}
func (gnps gnparserServer) clean(stream cleanStream, wn int) error {
inCh := make(chan string)
outCh := make(chan string)
var wg sync.WaitGroup
firstRecord := true
wg.Add(1)
go processCleaningStream(stream, outCh, &wg)
for {
in, err := stream.Recv()
if err == io.EOF {
close(inCh)
wg.Wait()
return nil
}
if err != nil {
return err
}
switch c := in.Content.(type) {
case *Input_Name:
if firstRecord {
firstRecord = false
go preprocess.CleanupStream(inCh, outCh, wn)
}
inCh <- c.Name
}
}
}
func processCleaningStream(stream cleanStream,
outCh <-chan string, wg *sync.WaitGroup) {
defer wg.Done()
for r := range outCh {
res := strings.Split(r, "|")
out := &Cleaned{
Input: res[0],
Output: res[1],
}
stream.Send(out)
}
}
var dictionary *dict.Dictionary
func Run(port int, workersNum int) {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment