Commit 5a1cae6b authored by Çagdas's avatar Çagdas

Support for amendment

local: 112e9f5cb5f611470a5411003e9599dd9aa60a08
parent 2e195410
......@@ -18,12 +18,12 @@ for the latest sources, issues and contributions:
## Copyright and license
Copyright OCamlPro 2017-2018. This code is licensed under the terms
Copyright OCamlPro 2017-2019. This code is licensed under the terms
of the GNU Public License version 3 (GPL v3).
## Building
You need OCaml 4.06.1.
You need OCaml 4.07.1.
Update submodules:
......@@ -65,40 +65,34 @@ opam install
```
opam install\
camlp4\
re.1.7.3\
ocplib-json-typed\
ocurl\
js_of_ocaml\
js_of_ocaml-ppx\
js_of_ocaml-camlp4\
js_of_ocaml-tyxml\
js_of_ocaml-lwt\
omd\
cohttp-lwt\
cohttp-lwt-unix\
base64\
ezjsonm\
ocplib-endian\
geoip\
ocp-build\
nocrypto\
sodium\
lru\
alcotest\
calendar \
base64.2.3.0 \
camlp4 \
cohttp-lwt \
cohttp-lwt-unix \
csv-lwt \
ezjsonm \
geoip \
js_of_ocaml \
js_of_ocaml-lwt \
js_of_ocaml-ppx \
js_of_ocaml-tyxml \
lwt_log \
csv-lwt
```
You will also need some development versions:
```
opam pin --dev ocplib-json-typed
opam install ocplib-json-typed
nocrypto \
ocamlfind \
ocplib-endian \
ocurl \
omd \
sodium \
zarith \
ocp-build \
ocplib-json-typed \
re \
calendar \
alcotest
```
One last dependency is pgocaml.
The OPAM version is not compatible with tz-scan, also you should install from
The OPAM version is not compatible with tzscan, also you should install from
the sources:
https://github.com/darioteixeira/pgocaml
```
......@@ -141,6 +135,7 @@ WITH_VERSION=true
API_HOST:=localhost
API_PORT:=8080
NEED_PARSEXP=true
BASE64_3:=false
```
The options are:
......@@ -151,6 +146,7 @@ The options are:
* WITH_VERSION: set to false to avoid recompiling everything everytime
* API_HOST & API_PORT: if you want to use localhost instead of api.tzscan.io
* NEED_PARSEXP: if you use sexplib (>= v0.11), then set it to true
* BASE64_3: if you use base64 (v>=3.0.0), then set it to true
### Building
......@@ -237,7 +233,7 @@ Create a file `config.json` containing :
Start the crawler:
```
./_tzscan-crawler config.json
./tzscan-crawler config.json
```
It should immediately connect to the Tezos node, and rewind the chain
......
(* Uses JS library blockies https://github.com/ethereum/blockies *)
open Js_of_ocaml
open Js_utils
let alphabet =
......
Subproject commit 3e037b801f865221a9fa534b9a7cffe8d3a5b72b
Subproject commit 9e73dfdd56846fc62bb3cd002d2b4c63cf1daae3
Subproject commit f9f144a10744f45c47c064aed7cb1cf87a322521
Subproject commit 1e4995e5051acece10fb41ab1f04ee769e7641b6
Subproject commit 428b9b4ac47a79810b65c4aef961d45e6688d13a
Subproject commit acfb41f711cbda67680b84d0f8292e2a1849c711
Subproject commit f9a94efce90f4a933ac0ebf5c6c435f322256394
Subproject commit 60afbdae1226b6f2120a1fff73c8264d6a9f5ef9
......@@ -36,6 +36,7 @@ struct
|> register S.accounts H.accounts
|> register S.nb_accounts H.nb_accounts
|> register S.account_bonds_rewards H.bonds_rewards
|> register S.extra_bonds_rewards H.extra_bonds_rewards
|> register S.rolls_history H.rolls_history
|> register S.account_status H.account_status
(* Operation services *)
......@@ -46,6 +47,10 @@ struct
|> register S.bakings H.bakings
|> register S.nb_bakings_endorsement H.nb_bakings_endorsement
|> register S.bakings_endorsement H.bakings_endorsement
|> register S.nb_cycle_bakings H.nb_cycle_bakings
|> register S.cycle_bakings H.cycle_bakings
|> register S.nb_cycle_endorsements H.nb_cycle_endorsements
|> register S.cycle_endorsements H.cycle_endorsements
|> register S.nb_bakings_history H.nb_bakings_history
|> register S.bakings_history H.bakings_history
|> register S.nb_endorsements_history H.nb_endorsements_history
......@@ -111,9 +116,23 @@ struct
|> register S.rewards_split_fast H.rewards_split_fast
|> register S.nb_cycle_delegator_rewards H.nb_cycle_delegator_rewards
|> register S.delegator_rewards H.delegator_rewards
|> register S.delegator_rewards_with_details H.delegator_rewards_with_details
|> register S.h24_stats H.h24_stats
|> register S.nb_protocol H.nb_protocol
|> register S.protocols H.protocols
|> register S.voting_period_info H.voting_period_info
|> register S.nb_proposals H.nb_proposals
|> register S.proposals H.proposals
|> register S.testing_proposal H.testing_proposal
|> register S.ballots H.ballots
|> register S.votes_account H.votes_account
|> register S.vote_graphs_account H.vote_graphs_account
|> register S.nb_proposal_votes H.nb_proposal_votes
|> register S.proposal_votes H.proposal_votes
|> register S.total_proposal_votes H.total_proposal_votes
|> register S.nb_ballot_votes H.nb_ballot_votes
|> register S.ballot_votes H.ballot_votes
|> register S.total_voters H.total_voters
|> register S.transaction_account_csv H.transaction_account_csv
|> register S.market_prices H.market_prices
|> register S.api_server_info H.api_server_info
......@@ -128,15 +147,11 @@ struct
|> register S.balance_ranking H.balance_ranking
|> register S.last_baking_and_endorsement H.last_baking_and_endorsement
|> register S.next_baking_and_endorsement H.next_baking_and_endorsement
|> register S.blocks_with_pred_fitness H.blocks_with_pred_fitness
|> register S.heads_with_pred_fitness H.heads_with_pred_fitness
(* not used in website *)
|> register S.roll_number H.roll_number
|> register S.deleguees_count H.deleguees_count
|> register S.deleguees H.deleguees
|> register S.deleguees_count_by_cycle_count H.deleguees_count_by_cycle_count
|> register S.deleguees_count_by_cycle H.deleguees_count_by_cycle
|> register S.all_deleguees_count_by_cycle_count H.all_deleguees_count_by_cycle_count
|> register S.all_deleguees_count_by_cycle H.all_deleguees_count_by_cycle
|> register S.total_bakings H.total_bakings
|> register S.total_endorsements H.total_endorsements
|> register S.cycle_baker_rights H.cycle_baker_rights
......@@ -153,6 +168,7 @@ end
module V1 = MakeRegisterer(Service.V1)(Handler.V1)
module V2 = MakeRegisterer(Service.V2)(Handler.V1)
module V3 = MakeRegisterer(Service.V3)(Handler.V1)
let services =
EzAPIServer.empty |> V1.register |> V2.register
EzAPIServer.empty |> V1.register |> V2.register |> V3.register
......@@ -38,12 +38,15 @@ OCaml.library("tzscan-api-lib", ocaml+
];
requires = [
"database-reader-lib";
"database-writer-lib";
"data-types-lib";
"tzscan-config";
"ez-api-cohttp";
"ez-api-server";
"ez-api-curl";
"tezos-explorer-services";
"csv-lwt";
"ez-recaptcha";
];
});
......@@ -56,3 +59,14 @@ OCaml.program("tzscan-api-server", ocaml+
"tzscan-api-lib";
];
});
OCaml.program("tzscan-openapi", ocaml+
{
files = [
"openapi_specs.ml";
];
requires = [
"tezos-explorer-services";
"ez-api-server";
];
});
\ No newline at end of file
This diff is collapsed.
(************************************************************************)
(* TzScan *)
(* *)
(* Copyright 2017-2018 OCamlPro *)
(* *)
(* This file is distributed under the terms of the GNU General Public *)
(* License as published by the Free Software Foundation; either *)
(* version 3 of the License, or (at your option) any later version. *)
(* *)
(* TzScan is distributed in the hope that it will be useful, *)
(* but WITHOUT ANY WARRANTY; without even the implied warranty of *)
(* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *)
(* GNU General Public License for more details. *)
(* *)
(************************************************************************)
let str_opt s = Arg.String (fun x -> s := Some x)
let () =
let output_file = ref "tzscan_openapi.json" in
let descr, version, title, terms, contact, license, servers =
ref None, ref None, ref None, ref None, ref None, ref None, ref None in
let speclist =
[ "-o", Arg.String (fun s -> output_file := s), "Optional name (path) of output file";
"--descr", str_opt descr, "Optional API description";
"--version", str_opt version, "Optional API version";
"--title", str_opt title, "Optional API title";
"--terms", str_opt terms, "Optional API terms";
"--contact", str_opt contact, "Optional API contact";
"--license", Arg.Tuple [
Arg.String (fun s -> license := Some (s,""));
Arg.String (fun s -> match !license with
| None -> license := None
| Some (a, _) -> license := Some (a, s))], "Optional API license";
"--servers", Arg.Tuple [
Arg.String (fun s ->
let before = match !servers with
| None -> []
| Some l -> l in
servers := Some ((s,"") :: before));
Arg.String (fun s -> match !servers with
| Some ((a, _) :: t) -> servers := Some ((a, s) :: t)
| _ -> servers := None)], "Optional API license"
] in
let usage_msg = "Create a swagger json file with the services of the API" in
Arg.parse speclist (fun _ -> ()) usage_msg;
let sections = Service.V3.sections in
let str = EzSwagger.to_string
?descr:!descr ?version:!version ?title:!title ?terms:!terms ?license:!license
?servers:!servers ~docs:Service_doc.doc sections in
let oc = open_out !output_file in
output_string oc str;
close_out oc
......@@ -29,8 +29,10 @@ let root = ref None
let default = ref (Some "index.html")
let node_config = ref None
let api_config = ref None
let alias_latency = ref 300.
let csv_lifespan = ref (24. *. 3600.)
(* sofar fixed at one day, need to change handler to put it variable *)
let local_services_file = ref None
(* Command-line arguments *)
let speclist = [
......@@ -51,7 +53,77 @@ let speclist = [
" No default for file-server (return 404)";
"--time-alias", Arg.Set_float alias_latency,
"Latency for alias table update";
]
"--verbose-db", Arg.Set Reader.verbose_mode,
" Verbosity mode for DB requests";
"--local-services", Arg.String (fun s -> local_services_file := Some s),
"Choose a local services.json instead of the one on the web server";
"--csv-lifespan", Arg.Float (fun f -> csv_lifespan := f *. 3600.),
"Lifespan of csv"
]
let services_encoding = ref None
let encoding_hook encoding =
services_encoding := Some encoding
let update_aliases () =
Lwt.return @@
match !services_encoding with
| None -> Printf.printf "services encoding not loaded\n%!"
| Some services_encoding ->
match !local_services_file with
| None ->
begin
match Config.get_services_filename () with
| None ->
Printf.eprintf "No services, no aliases to update.\n%!" ;
| Some url ->
try
EzCurl.get "" (EzAPI.TYPES.URL url)
~error:(fun i s ->
let msg = match s with None -> "" | Some s -> s in
Printf.eprintf "Downloading services from %S... \
[ FAIL ] [ %d ] [ %S ]\n%!" url i msg)
(fun content ->
Printf.eprintf "Downloading services from %S... [ OK ]\n%!" url ;
let json = Ezjsonm.from_string content in
Alias.change_aliases_from_json services_encoding json)
with Curl.CurlException (_code, i, s) ->
Printf.eprintf "[Service request] [%d] %s Curl exception: %s\n%!" i url s
end
| Some filename ->
try
Printf.eprintf "Reading services from %S\n%!" filename ;
let ic = open_in filename in
begin
try
let json = Ezjsonm.from_channel ic in
Alias.change_aliases_from_json services_encoding json
with exn ->
Printf.eprintf "Fatal error while destructing %S:\n %s\n%!"
filename (Printexc.to_string exn)
end;
close_in ic
with exn ->
Printf.eprintf "Fatal error while reading %S:\n %s\n%!"
filename (Printexc.to_string exn)
let clear_csv () =
match Config.get_csv_dir () with
| Some csv_dir when (String.length csv_dir) > 10 ->
begin
try
let a = Sys.readdir csv_dir in
Printf.eprintf "Cleaning CSV directory at %S\n%!" csv_dir ;
Array.iter (fun s ->
Sys.remove @@ Printf.sprintf "%s%s" csv_dir s) a
with Sys_error _ ->
Printf.eprintf "Directory to store csv files not found at %S.\
\nMaybe you should try `mkdir %s` first."
csv_dir csv_dir ;
exit 1
end
| _ -> ()
(* Main *)
let server services =
......@@ -81,21 +153,19 @@ let server services =
Lwt_main.run (
Stats.init () >>= fun () ->
Printf.eprintf "Starting servers on ports [%s]\n%!"
(String.concat ","
(List.map (fun (port,_) ->
string_of_int port) servers));
let update_table () =
Dbr.all_aliases () >>= fun l ->
Alias.reset ();
List.iter (fun (hash, alias) -> Alias.change_alias hash alias) l;
Lwt.return_unit
in
(String.concat ","
(List.map (fun (port,_) ->
string_of_int port) servers));
let rec update_loop () =
Printf.printf "Updating alias table [in progress]...\n%!";
update_table () >>= fun () ->
Printf.printf "Updating alias table [OK]...\n%!";
update_aliases () >>= fun () ->
Printf.printf "Updating alias table [done]...\n%!";
Lwt_unix.sleep !alias_latency >>= fun () ->
update_loop () in
let rec csv_gc () =
clear_csv ();
Lwt_unix.sleep !csv_lifespan >>= fun () ->
csv_gc () in
Lwt_list.iter_p (fun x -> x)
[EzAPIServer.server servers; update_loop ()]
[EzAPIServer.server servers; update_loop (); csv_gc ()]
)
......@@ -15,3 +15,5 @@
(************************************************************************)
val server : EzAPI.request EzAPIServer.directory -> unit
val encoding_hook : Data_types.service list Json_encoding.encoding -> unit
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -31,34 +31,42 @@ let address_encoding =
let all_addresses = ref []
let api_addresses = ref [||]
type config =
{
name : string ;
crawler : address list ;
api : address list ;
data_path : string option ;
}
let services_filename = ref None
let secret_key = ref None
let csv_dir = ref None
type config = {
name : string ;
crawler : address list ;
api : address list ;
services_filename : string option ;
data_path : string option ;
recaptcha_secret_key : string option;
csv_dir : string option
}
let config_encoding =
conv
(fun
{ name ; crawler ; api ; data_path }
-> ( name , crawler , api , data_path , None , None )
)
(fun
( name , crawler , api , data_path,
_max_default_blocks , _max_default_operations )
-> { name ; crawler ; api ; data_path }
(fun { name ; crawler ; api ; services_filename ; data_path ;
recaptcha_secret_key ; csv_dir }
-> ( name , crawler , api , services_filename, data_path , None , None ,
recaptcha_secret_key, csv_dir ))
(fun ( name , crawler , api , services_filename , data_path,
_max_default_blocks , _max_default_operations, recaptcha_secret_key ,
csv_dir )
-> { name ; crawler ; api ; services_filename ; data_path ;
recaptcha_secret_key ; csv_dir }
)
( obj6
( obj9
(req "name" string)
(req "crawler" (list address_encoding))
(req "api" (list address_encoding))
(opt "services" string)
(opt "data" string)
(opt "max_default_blocks" int)
(opt "max_default_operations" int)
)
(opt "recaptcha_secret_key" string)
(opt "csv_dir" string))
let configs_encoding = list config_encoding
......@@ -68,7 +76,10 @@ let configs_example = [
api = [
{ url = "http://another-host"; port = 18732 };
];
services_filename = None ;
data_path = None;
recaptcha_secret_key = None;
csv_dir = None
}
]
......@@ -134,7 +145,10 @@ let load_config_api file =
| [] -> crawler_addresses
| _ -> addresses
in
services_filename := c.services_filename ;
api_addresses := Array.of_list addresses;
secret_key := c.recaptcha_secret_key;
csv_dir := c.csv_dir;
begin
match !api_addresses with
| [||] -> error "No node addresses for balance queries"
......@@ -162,10 +176,14 @@ let get_config_crawler ~crawler file =
}
let get_addresses () = !all_addresses
let get_services_filename () = !services_filename
let get_api_address () =
let api_addresses = !api_addresses in
snd api_addresses.(Random.int (Array.length api_addresses))
let get_secret_key () = !secret_key
let get_csv_dir () = !csv_dir
(*
let get_address_pref ?fpref () =
let rec aux = function
......
......@@ -17,7 +17,10 @@
val load_config_api : string option -> unit
val get_addresses : unit -> (string * string) list
val get_api_address : unit -> string
val get_services_filename : unit -> string option
val get_config_crawler :
crawler:string -> string -> Data_types.config
val load_json_config : 'a Json_encoding.encoding -> string -> 'a
val error : string -> 'a
val get_secret_key : unit -> string option
val get_csv_dir : unit -> string option
......@@ -33,7 +33,7 @@ let choose_fields header_all fields_all header =
let transaction_header =
["transaction"; "block"; "network"; "source"; "destination"; "amount"; "fee";
"parameters"; "failed"; "internal"; "burned tez"; "counter"; "gas limit";
"date"; "parameters"; "failed"; "internal"; "burned tez"; "counter"; "gas limit";
"storage limit"]
let transaction header o =
......@@ -49,6 +49,7 @@ let transaction header o =
tr.tr_dst.tz;
Int64.to_string tr.tr_amount;
Int64.to_string tr.tr_fee;
tr.tr_timestamp;
Misc.unopt "" tr.tr_parameters;
string_of_bool tr.tr_failed;
string_of_bool tr.tr_internal;
......
......@@ -23,7 +23,7 @@ type account_name = { tz : account_hash; alias : string option }
type config =
{ nodes : url list ; data_path : string option }
type nonces = int * (operation_hash * (int list)) list
type nonces = int * (operation_hash option * int * block_hash) list
type block = {
hash : block_hash ;
......@@ -142,6 +142,8 @@ and origination = {
or_failed : bool ;
or_internal : bool ;
or_burn : int64 ;
or_op_level : int ;
or_timestamp : string
}
and delegation = {
......@@ -153,6 +155,8 @@ and delegation = {
del_storage_limit : Z.t ;
del_failed : bool ;
del_internal : bool ;
del_op_level : int ;
del_timestamp : string
}
and reveal = {
......@@ -164,11 +168,16 @@ and reveal = {
rvl_storage_limit : Z.t ;
rvl_failed : bool ;
rvl_internal : bool ;
rvl_op_level : int ;
rvl_timestamp : string
}
and activation = {
act_pkh : account_name ;
act_secret : string ;
act_balance : int64 option ;
act_op_level : int ;
act_timestamp : string
}
and endorsement = {
......@@ -392,6 +401,16 @@ type account_bonds_rewards = {
acc_e_deposits : int64
}
type account_extra_rewards ={
acc_dn_gain: int64;
acc_dn_deposit: int64;
acc_dn_rewards: int64;
acc_dn_fees: int64;
acc_rv_rewards: int64;
acc_rv_lost_rewards: int64;
acc_rv_lost_fees:int64
}
type account_status = {
account_status_hash : account_name ;
account_status_revelation : operation_hash option ;
......@@ -472,11 +491,13 @@ type service = {
srv_url : string;
srv_logo : string; (* file name, related to /images/ *)
srv_logo2 : string option; (* file name, related to /images/ *)
srv_logo_payout : string option; (* file name, related to /images/ *)
srv_descr : string option;
srv_sponsored : string option;
srv_page : string option;
srv_delegations_page : bool ;
srv_account_page : bool ;
srv_aliases: account_name list option
}
type crawler_activity = {
......@@ -540,6 +561,19 @@ type delegator_reward = {
dor_status: rewards_status
}
type delegator_reward_details = {
dor_block_rewards: int64;
dor_end_rewards: int64;
dor_fees: int64;
dor_rv_rewards: int64;
dor_dn_gain: int64;
dor_rv_lost_rewards: int64;
dor_rv_lost_fees: int64;
dor_dn_lost_deposit: int64;
dor_dn_lost_rewards: int64;
dor_dn_lost_fees: int64;
}
type snapshot = {
snap_cycle : int ;
snap_index : int ;
......@@ -720,6 +754,9 @@ type www_server_info = {
mutable www_logo : string ;
mutable www_footer : string ;
mutable www_networks : ( string * string ) list ; (* name x www_url *)
mutable www_themes : (string * string) list ;
mutable www_recaptcha_key : string option ;
mutable www_csv_server : (string * string) option ;
}
type gk_coin = {gk_usd: float; gk_btc: float}
......@@ -759,3 +796,14 @@ type exchange_info = {
ex_total_volume: float;
ex_tickers: ex_ticker list
}
type proposal = {
prop_period: int;
prop_period_kind: voting_period_kind;
prop_hash: proposal_hash;
prop_count: int;
prop_votes: int;
prop_source: account_name;
prop_op: operation_hash option;
prop_ballot: ballot_type option
}
......@@ -96,6 +96,9 @@ let www = {
www_logo = "tzscan-logo.png" ;
www_footer = "/footer.html" ;
www_networks = [] ;
www_themes = [ "Light", "default"; "Dark", "slate" ] ;
www_recaptcha_key = None;
www_csv_server = None;
}
let save_api_config filename =
......
......@@ -65,6 +65,10 @@ let unopt def = function
| None -> def
| Some x -> x
let convopt f = function
| None -> None
| Some x -> Some (f x)
let list_index f =
List.fold_left (fun acc x ->
let i = fst acc in
......
This diff is collapsed.
......@@ -18,7 +18,10 @@ open Data_types
let alias_table : (string, string) Hashtbl.t = Hashtbl.create 1000
let change_alias tz alias = Hashtbl.add alias_table tz alias
let add_alias {tz; alias} =
match alias with
| None -> ()
| Some alias -> Hashtbl.add alias_table tz alias
let get_alias_tbl tz = Hashtbl.find_opt alias_table tz
......@@ -29,3 +32,35 @@ let to_name ?alias tz =
else
let alias = if alias = None then get_alias_tbl tz else alias in
{ tz ; alias }
let change_aliases_from_json ?(with_table=true) ?update_db encoding json =
let alias_added = ref 0 in
let alias_removed = ref 0 in
try
let services = Json_encoding.destruct encoding json in
reset ();
List.iter (fun s ->
match s.srv_aliases, s.srv_sponsored with
| None, _ | _, None -> ()
| Some aliases, Some timestamp ->
let date =
CalendarLib.Printer.Calendar.from_fstring
"%Y-%m-%dT%H:%M:%SZ" timestam