Commit 2f2cd0ea authored by Çagdas's avatar Çagdas

New user interface and more data in the crawler

local: 6522fc3b419d0f1bbe322dbaeb2f32d520f6c6d8
parent 5a1cae6b
......@@ -4,9 +4,6 @@
[submodule "libs/ez-pgocaml"]
path = libs/ez-pgocaml
url = https://github.com/OCamlPro/ez-pgocaml
[submodule "libs/ocplib-json-typed"]
path = libs/ocplib-json-typed
url = https://github.com/OCamlPro/ocplib-json-typed
[submodule "libs/ocplib-jsutils"]
path = libs/ocplib-jsutils
url = https://github.com/OCamlPro/ocplib-jsutils
......
......@@ -6,10 +6,8 @@ HAS_PGOCAML:=true
API_PORT:=
API_HOST:=
AUTO_UPGRADE:=true
NEED_PARSEXP:=false
# Keep this flag until the ezpg table has been updated
DBOLDINFO=--old-info
NEED_PARSEXP:=true
BASE64_3:=true
# for betanet, create a file Makefile.database with the (uncommented) line:
# DATABASE=alphanet
......@@ -30,9 +28,6 @@ DBOLDINFO=--old-info
#API_PORT:=8000
#API_HOST:=localhost
# Keep this flag until the ezpg table has been updated
DBOLDINFO=--old-info
-include Makefile.database
DEST_DIR ?= ${CURDIR}/www
......@@ -41,6 +36,7 @@ DEST_DIR ?= ${CURDIR}/www
# otherwise, it is impossible to compile without committing first the
# submodules
all: _obuild may-db-update
$(MAKE) BASE64_3=$(BASE64_3) -C libs/ocplib-jsutils base64-conf
PGDATABASE=$(DATABASE) ocp-build
cp -f _obuild/tzscan-crawler/tzscan-crawler.asm tzscan-crawler
cp -f _obuild/tzscan-api-server/tzscan-api-server.asm tzscan-api-server
......@@ -91,6 +87,7 @@ build-deps: tezos-explorer.opam
DBUPDATER=tzscan-db-updater
DBWITNESS=--witness db-version.txt
include libs/ez-pgocaml/libs/ez-pgocaml/Makefile.ezpg
include libs/ocplib-jsutils/ocp-autoconf.d/Makefile
_obuild:
ocp-build init
......
......@@ -56,72 +56,89 @@ sudo apt-get install\
curl zlib1g-dev libgeoip-dev
```
On CentOS 7 (utdrmac):
Base packages:
```
yum install epel-release
yum install gcc m4 make libsodium libsodium-devel geoip geoip-devel zlib libcurl libcurl-devel
```
PostgreSQL 10:
```
rpm -Uvh https://yum.postgresql.org/10/redhat/rhel-7-x86_64/pgdg-centos10-10-2.noarch.rpm
yum install postgresql10 postgresql10-server postgresql10-libs
```
#### OCaml dependencies
They can (and should) be installed with opam through
Before installing and compiling all dependencies, you need to disable
the sandbox mode of opam. Check the file `~/.opam/config`, and comment
the lines below if there are still there:
```
opam install
wrap-build-commands:
["%{hooks}%/sandbox.sh" "build"] {os = "linux" | os = "macos"}
wrap-install-commands:
["%{hooks}%/sandbox.sh" "install"] {os = "linux" | os = "macos"}
wrap-remove-commands:
["%{hooks}%/sandbox.sh" "remove"] {os = "linux" | os = "macos"}
```
When you finished the compilation, you can reenable the sandboxing
mode of opam.
Then the dependencies can (and should) be installed with opam through
```
opam install\
base64.2.3.0 \
camlp4 \
cohttp-lwt \
opam install \
base64 \
cohttp-lwt-unix \
csv-lwt \
ezjsonm \
geoip \
js_of_ocaml \
js_of_ocaml-lwt \
js_of_ocaml-ppx \
js_of_ocaml-camlp4 \
js_of_ocaml-tyxml \
js_of_ocaml-ppx \
lwt_log \
nocrypto \
ocamlfind \
ocplib-endian \
ocurl \
omd \
sodium \
zarith \
ocp-build \
ocplib-json-typed \
re \
calendar \
alcotest
alcotest \
pgocaml \
extunix
```
One last dependency is pgocaml.
The OPAM version is not compatible with tzscan, also you should install from
the sources:
https://github.com/darioteixeira/pgocaml
You also need to install the development version of httpaf and httpaf-lwt-unix:
```
./configure --enable-p4 --prefix $OPAM_SWITCH_PREFIX --docdir $OPAM_SWITCH_PREFIX/doc
make
make doc
make install
./build-httpaf.sh
```
If `make install` fails, you might retry it after a `ocamlfind remove pgocaml`.
Optional:
If you don't want to disable the sandbox mode in opam, you can also build pgocaml from the sources using:
```
./rebuild-pgocaml.sh
```
### Configuring
#### Configuring Postgresql
For TzScan to compile successfully, you will need your user to be able to
create tables in Postgresql. This is done with the following commands:
```
$ sudo -i -u postgres
$ psql
CREATE USER <user>;
ALTER ROLE <user> CREATEDB;
```
where `<user>` should be replaced by your login.
For TzScan to compile successfully, you will need your user to be able
to create tables in Postgresql. This is done with the following
commands: ``` $ sudo -i -u postgres $ psql CREATE USER <user>; ALTER
ROLE <user> CREATEDB; ``` where `<user>` should be replaced by your
login.
#### Configuring TzScan
Then, configure tz-scan:
Then, configure TzScan:
```
./configure
```
......@@ -135,7 +152,7 @@ WITH_VERSION=true
API_HOST:=localhost
API_PORT:=8080
NEED_PARSEXP=true
BASE64_3:=false
BASE64_3:=true
```
The options are:
......@@ -156,8 +173,8 @@ make
```
If no error happened, you should get:
* A command called `tzscan-crawler`
* A command called `tzscan-api-server`
* A binary called `tzscan-crawler`
* A binary called `tzscan-api-server`
* A directory `www` containing at least an `explorer-main.js` file
## Usage
......@@ -220,13 +237,23 @@ Create a file `config.json` containing :
```
[
{
"name": "betanet-main",
"name": "mainnet-main",
"crawler" : [
{"url" : "http://your-tezos-node", "port" : 8132}
{"url" : "http://tezos-node", "port" : 8732}
],
"api" : [
{"url" : "http://another-tezos-node", "port" : 8132 },
{"url" : "http://tezos-node", "port" : 8732},
]
} ,
{
"name" : "mainnet-extra-data",
"crawler" : [
{"url" : "http://tezos-node", "port" : 8732}
],
"api" : [ ],
"max_default_blocks" : 10,
"max_default_operations" : 10
}
]
```
......@@ -241,6 +268,16 @@ to register blocks from the genesis block to the latest one. Then, it
should wait and connect to the Tezos node every 3 seconds to query for new
blocks.
To get extra non-critical data like marketcap data or voting data, you
should run another instance of the crawler with `--extra-data` option
```
./tzscan-crawler --extra-data config.json
```
This instance of the crawler will read the `$NETWORK-extra-data` field
from your config.json
### Running the API Server Locally
You need first to have started the Tezos crawler as explained in the
......
#!/bin/sh
git clone https://github.com/inhabitedtype/httpaf.git
(cd httpaf;
opam pin add httpaf .
opam pin add httpaf-lwt-unix .)
rm -rf httpaf
oflags = [ "-g"; "-w"; "+a-48-4-44"; "-warn-error"; "+a-44" ];
nowarnings = [ "-warn-error"; "-a"; "-w"; "-a" ];
nowarnings = [ "-warn-error"; "-a" (* bad idea ; "-w"; "-a" *) ];
ocaml.asmlink = ["-g"];
ocaml.bytecomp = oflags;
ocaml.asmcomp = oflags;
if( need_parsexp ){
sexplib0 = [
sexplib0 = [
"%{sexplib_FULL_DST_DIR}%0/sexplib0.cma";
"%{parsexp_FULL_DST_DIR}%/parsexp.cma";
];
......@@ -28,7 +28,7 @@ if( with_version = true ){
"commit";
"en_date";
];
}
}
OCaml.library("tzscan-config", ocaml + {
......@@ -68,12 +68,13 @@ pp_pgocaml = [
"%{re_FULL_DST_DIR}%/re.cma"; "-I"; "%{csv_FULL_DST_DIR}%";
"%{csv_FULL_DST_DIR}%/csv.cma";
"-I"; "%{sexplib_FULL_DST_DIR}%"; "-I"; "%{sexplib_FULL_DST_DIR}%/0"; "-I"; "%{sexplib_FULL_DST_DIR}%/unix" ]
"-I"; "%{sexplib_FULL_DST_DIR}%"; "-I"; "%{sexplib_FULL_DST_DIR}%/0"; "-I"; "%{sexplib_FULL_DST_DIR}%/unix" ]
+ sexplib0 +
[
"%{sexplib_FULL_DST_DIR}%/sexplib.cma";
"%{sexplib_FULL_DST_DIR}%/unix/sexplib_unix.cma";
"-I"; "%{cstruct_FULL_DST_DIR}%"; "%{cstruct_FULL_DST_DIR}%/cstruct.cma";
"-I"; "%{bigarray-compat_FULL_DST_DIR}%"; "%{bigarray-compat_FULL_DST_DIR}%/bigarray_compat.cma";
"-I"; "%{hex_FULL_DST_DIR}%"; "%{hex_FULL_DST_DIR}%/hex.cma";
"-I"; "%{pgocaml_FULL_DST_DIR}%";
"%{pgocaml_FULL_DST_DIR}%/pgocaml.cma"; "-I";
......
......@@ -50,29 +50,4 @@ try {
build_common_loaded = build_common_loaded ;
} catch ("unknown-variable",x){
include "build.common" ;
}
submodules_dir = "libs/";
(* We have our own version of ocplib-json-typed to avoid having a dev version
in our environment. *)
OCaml.library("ocplib-json-typed",
ocaml + {
bytecomp = ocaml.bytecomp + nowarnings;
asmcomp = ocaml.asmcomp + nowarnings;
files = [
submodules_dir + "ocplib-json-typed/src/json_repr.ml";
submodules_dir + "ocplib-json-typed/src/json_query.ml";
submodules_dir + "ocplib-json-typed/src/json_schema.ml";
submodules_dir + "ocplib-json-typed/src/json_encoding.ml";
];
requires = '[
ezjsonm
ocplib-endian
uri
];
}
);
}
\ No newline at end of file
(* Uses JS library blockies https://github.com/ethereum/blockies *)
open Js_of_ocaml
open Ocp_js
open Js_utils
let alphabet =
......@@ -79,7 +79,7 @@ let create ?(size=8) ?(scale=8) tz1 =
) in
let icon =
Js.Unsafe.(meth_call (variable "blockies")) "create" [|o|]
|> Tyxml_js.Of_dom.of_canvas
|> Of_dom.of_canvas
in
Manip.SetCss.borderRadius icon "3px";
icon
......
Subproject commit 9e73dfdd56846fc62bb3cd002d2b4c63cf1daae3
Subproject commit a48bd8bea3307dc2b0ad47625d1c4f7e3efcb993
Subproject commit acfb41f711cbda67680b84d0f8292e2a1849c711
Subproject commit 60afbdae1226b6f2120a1fff73c8264d6a9f5ef9
Subproject commit 049e23142322e99977043c265710f91ffcb79412
......@@ -6,7 +6,7 @@
(* *)
(**************************************************************)
opam-version: "1.2"
opam-version: "1.3"
maintainer: "OCamlPro <[email protected]>"
authors: [
"Cagdas Bozman <[email protected]>"
......@@ -24,51 +24,30 @@ build: [
[ make ]
]
install: [
[ make "install" ]
[ make ]
]
remove: [
]
available: [ocaml-version >= "4.06.1"]
available: [ocaml-version >= "4.07.1"]
depends: [
"base64"
"camlp4"
"cohttp-lwt" {>= "0.99.0" }
"cohttp-lwt-unix"
"csv-lwt"
"ezjsonm"
"geoip"
"js_of_ocaml"
"js_of_ocaml-camlp4"
"js_of_ocaml-lwt"
"js_of_ocaml-ppx"
"js_of_ocaml-camlp4"
"js_of_ocaml-tyxml"
"js_of_ocaml-ppx"
"lwt_log"
"nocrypto"
"ocamlfind"
"ocp-build"
"ocplib-endian"
"ocurl"
"omd"
"pgocaml"
"sodium"
"zarith"
]
(**************************************************************)
(* *)
(* From opam.trailer: *)
(* *)
(**************************************************************)
(* Currently, there is no way to tell ocp-autoconf that we want an exact
version, not a minimal version. *)
depends: [
"cohttp-lwt" {= "0.99.0" }
"ocp-build"
"ocplib-json-typed"
"alcotest"
"pgocaml"
"extunix"
]
pin-depends: [
[ "pgocaml.2.3+dev" "git+https://github.com/darioteixeira/pgocaml.git" ]
[ "ocplib-json-typed.0.6+dev"
"git+https://github.com/ocamlpro/ocplib-json-typed.git#428b9b4ac47a79810b65c4aef961d45e6688d13a" ]
]
\ No newline at end of file
opam list -i :
alcotest 0.8.5 Alcotest is a lightweight and colourful test framework
angstrom 0.11.2 Parser combinators built for speed and memory-efficiency
astring 0.8.3 Alternative String module for OCaml
base v0.12.1 Full standard library replacement for OCaml
base64 3.2.0 Base64 encoding for OCaml
bigarray-compat 1.0.0 Compatibility library to use Stdlib.Bigarray when possible
bigstringaf 0.5.0 Bigstring intrinsics and fast blits based on memcpy/memmove
biniou 1.2.0 Binary data format designed for speed, safety, ease of use and backward compatibility as protocols evolve
calendar 2.04 Library for handling dates and times in your program
camlp4 4.07+1 Camlp4 is a system for writing extensible parsers for programming languages
cmdliner 1.0.3 Declarative definition of command line interfaces for OCaml
cohttp 2.1.2 An OCaml library for HTTP clients and servers
cohttp-lwt 2.0.0 CoHTTP implementation using the Lwt concurrency library
cohttp-lwt-unix 2.0.0 CoHTTP implementation for Unix and Windows using Lwt
conduit 1.4.0 A network connection establishment library
conduit-lwt 1.4.0 A portable network connection establishment library using Lwt
conduit-lwt-unix 1.4.0 A network connection establishment library for Lwt_unix
conf-gmp 1 Virtual package relying on a GMP lib system installation
conf-libcurl 1 Virtual package relying on a libcurl system installation
conf-libsodium 1 Virtual package relying on a libsodium system installation
conf-m4 1 Virtual package relying on m4
conf-perl 1 Virtual package relying on perl
conf-pkg-config 1.1 Virtual package relying on pkg-config installation
conf-which 1 Virtual package relying on which
cppo 1.6.5 Equivalent of the C preprocessor for OCaml programs
cppo_ocamlbuild 1.6.0 ocamlbuild support for cppo, OCaml-friendly source preprocessor
cpuid 0.1.2 Detect CPU features
cstruct 4.0.0 Access C-like structures directly from OCaml
cstruct-lwt 4.0.0 Access C-like structures directly from OCaml
csv 2.2 A pure OCaml library to read and write CSV files
csv-lwt 2.2 A pure OCaml library to read and write CSV files, LWT version
ctypes 0.14.0 Combinators for binding to C libraries without writing any C.
dune 1.9.3 Fast, portable and opinionated build system
easy-format 1.3.1 High-level and functional interface to the Format module of the OCaml standard library
extunix 0.1.6 Collection of thin bindings to various low-level system API
ezjsonm 1.1.0 Simple interface on top of the Jsonm JSON library
faraday 0.7.0 A library for writing fast and memory-efficient serializers.
faraday-lwt 0.7.0 Lwt support for Faraday
faraday-lwt-unix 0.7.0 Lwt_unix support for Faraday
fieldslib v0.12.0 Syntax extension to define first class values representing record fields, to get and set record fields, iterate and fold over all fields of a record and create new record val
fmt 0.8.6 OCaml Format pretty-printer combinators
geoip 0.0.3 Bindings to GeoIP database library.
hex 1.4.0 Library providing hexadecimal converters
httpaf 0.6.0 pinned to version 0.6.0 at git+file:///home/...
httpaf-lwt-unix 0.6.0 pinned to version 0.6.0 at git+file:///home/...
integers 0.2.2 Various signed and unsigned integer types for OCaml
ipaddr 3.1.0 A library for manipulation of IP (and MAC) address representations
jbuilder transition This is a transition package, jbuilder is now named dune. Use the dune
js_of_ocaml 3.4.0 Compiler from OCaml bytecode to Javascript
js_of_ocaml-camlp4 3.1.0 Compiler from OCaml bytecode to Javascript
js_of_ocaml-compiler 3.4.0 Compiler from OCaml bytecode to Javascript
js_of_ocaml-lwt 3.4.0 Compiler from OCaml bytecode to Javascript
js_of_ocaml-ppx 3.4.0 Compiler from OCaml bytecode to Javascript
js_of_ocaml-tyxml 3.4.0 Compiler from OCaml bytecode to Javascript
jsonm 1.0.1 Non-blocking streaming JSON codec for OCaml
logs 0.6.3 Logging infrastructure for OCaml
lwt 4.2.1 Promises and event-driven I/O
lwt_log 1.1.0 Lwt logging library (deprecated)
macaddr 3.1.0 A library for manipulation of MAC address representations
magic-mime 1.1.1 Map filenames to common MIME types
mirage-no-solo5 1 Virtual package conflicting with mirage-solo5
mirage-no-xen 1 Virtual package conflicting with mirage-xen
mmap 1.1.0 File mapping functionality
nocrypto 0.5.4-1 Simpler crypto
num 1.1 The legacy Num library for arbitrary-precision integer and rational arithmetic
ocaml 4.07.1 The OCaml compiler (virtual package)
ocaml-base-compiler 4.07.1 Official release 4.07.1
ocaml-compiler-libs v0.11.0 OCaml compiler libraries repackaged
ocaml-config 1 OCaml Switch Configuration
ocaml-migrate-parsetree 1.3.1 Convert OCaml parsetrees between different versions
ocamlbuild 0.14.0 OCamlbuild is a build system with builtin rules to easily build most OCaml projects.
ocamlfind 1.8.0 A library manager for OCaml
ocb-stubblr 0.1.1-1 OCamlbuild plugin for C stubs
ocp-build 1.99.20-beta Project builder for OCaml
ocplib-endian 1.0 Optimised functions to read and write int16/32/64 from strings and bigarrays, based on new primitives added in version 4.01.
ocplib-json-typed 0.7.1 Type-aware JSON and JSON schema utilities
ocurl 0.8.2 Bindings to libcurl
omd 1.3.1 A Markdown frontend in pure OCaml.
parsexp v0.12.0 S-expression parsing library
pgocaml 3.2 Interface to PostgreSQL databases
ppx_derivers 1.2.1 Shared [@@deriving] plugin registry
ppx_deriving 4.2.1 Type-driven code generation for OCaml >=4.02
ppx_fields_conv v0.12.0 Generation of accessor and iteration functions for ocaml records
ppx_sexp_conv v0.12.0 [@@deriving] plugin to generate S-expression conversion functions
ppx_tools 5.1+4.06.0 Tools for authors of ppx rewriters and other syntactic tools
ppx_tools_versioned 5.2.2 A variant of ppx_tools based on ocaml-migrate-parsetree
ppxlib 0.6.0 Base library and tools for ppx rewriters
re 1.9.0 RE is a regular expression library for OCaml
react 1.2.1 Declarative events and signals for OCaml
reactiveData 0.2.2 Declarative events and signals for OCaml
result 1.3 Compatibility Result module
rresult 0.6.0 Result value combinators for OCaml
sexplib v0.12.0 Library for serializing OCaml values to and from S-expressions
sexplib0 v0.12.0 Library containing the definition of S-expressions and some base converters
sodium 0.6.0 Binding to libsodium UNAUDITED
stdio v0.12.0 Standard IO library for OCaml
stdlib-shims 0.1.0 Backport some of the new stdlib features to older compiler
stringext 1.5.0 Extra string functions for OCaml
topkg 1.0.0 The transitory OCaml software packager
tyxml 4.3.0 TyXML is a library for building correct HTML and SVG documents
uchar 0.0.2 Compatibility library for OCaml's Uchar module
uri 2.2.0 An RFC3986 URI/URL parsing library
uuidm 0.9.7 Universally unique identifiers (UUIDs) for OCaml
uutf 1.0.2 Non-blocking streaming Unicode codec for OCaml
yojson 1.7.0 Yojson is an optimized parsing and printing library for the JSON format
zarith 1.7 Implements arithmetic and logical operations over arbitrary-precision integers
opam list -e :
debianutils
libcurl4-gnutls-dev
libgeoip-dev
libgmp-dev
libsodium-dev
m4
perl
pkg-config
#!/bin/sh
opam pin -y --dev pgocaml
opam source pgocaml
opam remove pgocaml
(cd pgocaml.3.2;
./configure --enable-p4 --prefix $OPAM_SWITCH_PREFIX --docdir $OPAM_SWITCH_PREFIX/doc &&
make &&
make doc &&
make install)
opam install pgocaml --fake
rm -rf pgocaml.3.2
......@@ -13,8 +13,7 @@
(* GNU General Public License for more details. *)
(* *)
(************************************************************************)
open EzAPIServer
open EzAPIServerUtils
module MakeRegisterer(S: module type of Service.V1)(H:module type of Handler.V1) =
struct
......@@ -22,12 +21,13 @@ struct
let register dir =
dir
(* Block services *)
|> EzAPIServer.register S.block H.block
|> EzAPIServerUtils.register S.block H.block
|> register S.blocks H.blocks
|> register S.heads H.heads
|> register S.nb_heads H.nb_heads
|> register S.nb_uncles H.nb_uncles
|> register S.head H.head
|> register S.head_cycle H.head_cycle
|> register S.nb_cycle H.nb_cycle
|> register S.nb_snapshot_blocks H.nb_snapshot_blocks
|> register S.snapshot_blocks H.snapshot_blocks
......@@ -149,6 +149,7 @@ struct
|> register S.next_baking_and_endorsement H.next_baking_and_endorsement
|> register S.blocks_with_pred_fitness H.blocks_with_pred_fitness
|> register S.heads_with_pred_fitness H.heads_with_pred_fitness
|> register S.current_period_sum H.current_period_sum
(* not used in website *)
|> register S.roll_number H.roll_number
......@@ -164,6 +165,10 @@ struct
|> register S.balance_break_down H.balance_break_down
|> register S.alias H.alias
|> register S.account_from_alias H.account_from_alias
|> register S.nb_exchange H.nb_exchange
|> register S.exchange_info H.exchange_info
|> register S.get_services H.get_services
end
module V1 = MakeRegisterer(Service.V1)(Handler.V1)
......@@ -171,4 +176,4 @@ module V2 = MakeRegisterer(Service.V2)(Handler.V1)
module V3 = MakeRegisterer(Service.V3)(Handler.V1)
let services =
EzAPIServer.empty |> V1.register |> V2.register |> V3.register
EzAPIServerUtils.empty |> V1.register |> V2.register |> V3.register
......@@ -42,8 +42,9 @@ OCaml.library("tzscan-api-lib", ocaml+
"data-types-lib";
"tzscan-config";
"ez-api-cohttp";
"ez-api-server";
"ez-api-curl";
"ez-api-server-httpaf";
"ez-api-server";
"tezos-explorer-services";