Commit 2e195410 authored by Fabrice Le Fessant's avatar Fabrice Le Fessant

Support for protocol 003

local:46678c6ac69769b12ebd784251e626de1b478d2f
parent 9f02015d
......@@ -7,3 +7,4 @@ ChangeLog
* Use `api-config.json` to configure the API server
* User `info.json` to configure the Webapp interface
2018-12-10: Support for protocol 003
......@@ -45,7 +45,7 @@ On Debian 9:
sudo apt-get install\
postgresql libsodium-dev libgeoip1 \
geoip1-database libcurl4-gnutls-dev \
curl zlib1g-dev
curl zlib1g-dev libgeoip-dev
```
On Ubuntu:
......@@ -53,13 +53,15 @@ On Ubuntu:
sudo apt-get install\
postgresql libsodium-dev libgeoip1 \
geoip-database libcurl4-gnutls-dev \
curl zlib1g-dev
curl zlib1g-dev libgeoip-dev
```
#### OCaml dependencies
They can (and should) be installed with opam through
```opam install``
```
opam install
```
```
opam install\
......
Subproject commit 594e64dd03d0b5c5b21c73cdf0521832fc9b0ddc
Subproject commit f9a94efce90f4a933ac0ebf5c6c435f322256394
......@@ -28,7 +28,6 @@ struct
|> register S.nb_heads H.nb_heads
|> register S.nb_uncles H.nb_uncles
|> register S.head H.head
|> register S.genesis H.genesis
|> register S.nb_cycle H.nb_cycle
|> register S.nb_snapshot_blocks H.nb_snapshot_blocks
|> register S.snapshot_blocks H.snapshot_blocks
......@@ -37,18 +36,7 @@ struct
|> register S.accounts H.accounts
|> register S.nb_accounts H.nb_accounts
|> register S.account_bonds_rewards H.bonds_rewards
|> register S.roll_number H.roll_number
|> register S.rolls_history H.rolls_history
|> register S.deleguees_count H.deleguees_count
|> register S.deleguees H.deleguees
|> register S.deleguees_count_by_cycle_count
H.deleguees_count_by_cycle_count
|> register S.deleguees_count_by_cycle
H.deleguees_count_by_cycle
|> register S.all_deleguees_count_by_cycle_count
H.all_deleguees_count_by_cycle_count
|> register S.all_deleguees_count_by_cycle
H.all_deleguees_count_by_cycle
|> register S.account_status H.account_status
(* Operation services *)
|> register S.operation H.operation
......@@ -60,45 +48,36 @@ struct
|> register S.bakings_endorsement H.bakings_endorsement
|> register S.nb_bakings_history H.nb_bakings_history
|> register S.bakings_history H.bakings_history
|> register S.total_bakings H.total_bakings
|> register S.nb_endorsements_history H.nb_endorsements_history
|> register S.endorsements_history H.endorsements_history
|> register S.total_endorsements H.total_endorsements
|> register S.nb_cycle_rights H.nb_cycle_rights
|> register S.cycle_rights H.cycle_rights
|> register S.nb_baker_rights H.nb_baker_rights
|> register S.baker_rights H.baker_rights
|> register S.cycle_baker_rights H.cycle_baker_rights
|> register S.cycle_all_rights H.cycle_all_rights
|> register S.nb_endorser_rights H.nb_endorser_rights
|> register S.endorser_rights H.endorser_rights
|> register S.cycle_endorser_rights H.cycle_endorser_rights
|> register S.required_balance H.required_balance
|> register S.nb_operations_hash H.nb_operations_hash
|> register S.nb_operations H.nb_operations
|> register S.endorsements_level H.endorsements_level
(* Misc services *)
|> register S.block_succ H.block_succ
|> register S.block_pred H.block_pred
|> register S.block_hash_level H.block_hash_level
|> register S.timestamp H.timestamp
|> register S.level H.level
|> register S.nb_network_peers H.nb_network_peers
|> register S.network H.network
|> register S.priority H.priority
|> register S.volume H.volume
|> register S.block_level H.block_level
|> register S.nonces H.nonces
|> register S.marketcap H.marketcap
|> register S.network_stats H.network_stats
|> register S.country_stats H.country_stats
|> register S.baker H.baker_stats
|> register S.bakers H.bakers_stats
|> register S.blocks_per_day H.blocks_per_day
|> register S.bakers_per_day H.bakers_per_day
|> register S.priorities_per_day H.priorities_per_day
|> register S.operations_per_day H.operations_per_day
|> register S.operations_per_block_per_day
H.operations_per_block_per_day
|> register S.operations_per_block_per_day H.operations_per_block_per_day
|> register S.fees_per_day H.fees_per_day
|> register S.volume_per_day H.volume_per_day
|> register S.version H.version
......@@ -118,20 +97,13 @@ struct
|> register S.nb_search_account H.nb_search_account
|> register S.activated_balances H.activated_balances
|> register S.supply H.supply
|> register S.balance_break_down H.balance_break_down
|> register S.node_timestamps H.node_timestamps
|> register S.node_account H.node_account
|> register S.node_deactivated_grace_period
H.node_deactivated_grace_period
|> register S.node_delegated_contracts
H.node_delegated_contracts
|> register S.node_nb_delegated_contracts
H.node_nb_delegated_contracts
|> register S.node_staking_balance
H.node_staking_balance
|> register S.node_deactivated_grace_period H.node_deactivated_grace_period
|> register S.node_delegated_contracts H.node_delegated_contracts
|> register S.node_nb_delegated_contracts H.node_nb_delegated_contracts
|> register S.node_staking_balance H.node_staking_balance
|> register S.date H.date
|> register S.alias H.alias
|> register S.account_from_alias H.account_from_alias
|> register S.nb_delegators H.nb_delegators
|> register S.nb_cycle_rewards H.nb_cycle_rewards
|> register S.rewards_split_cycles H.rewards_split_cycles
......@@ -145,6 +117,38 @@ struct
|> register S.transaction_account_csv H.transaction_account_csv
|> register S.market_prices H.market_prices
|> register S.api_server_info H.api_server_info
|> register S.balance_updates_number H.balance_updates_number
|> register S.balance_updates H.balance_updates
|> register S.active_balance_updates H.active_balance_updates
|> register S.cycle_frozen H.cycle_frozen
|> register S.balance H.balance
|> register S.balance_from_balance_updates H.balance_from_balance_updates
|> register S.balance_history H.balance_history
|> register S.balance_number H.balance_number
|> register S.balance_ranking H.balance_ranking
|> register S.last_baking_and_endorsement H.last_baking_and_endorsement
|> register S.next_baking_and_endorsement H.next_baking_and_endorsement
(* not used in website *)
|> register S.roll_number H.roll_number
|> register S.deleguees_count H.deleguees_count
|> register S.deleguees H.deleguees
|> register S.deleguees_count_by_cycle_count H.deleguees_count_by_cycle_count
|> register S.deleguees_count_by_cycle H.deleguees_count_by_cycle
|> register S.all_deleguees_count_by_cycle_count H.all_deleguees_count_by_cycle_count
|> register S.all_deleguees_count_by_cycle H.all_deleguees_count_by_cycle
|> register S.total_bakings H.total_bakings
|> register S.total_endorsements H.total_endorsements
|> register S.cycle_baker_rights H.cycle_baker_rights
|> register S.cycle_endorser_rights H.cycle_endorser_rights
|> register S.block_pred H.block_pred
|> register S.priority H.priority
|> register S.network H.network
|> register S.block_level H.block_level
|> register S.baker H.baker_stats
|> register S.balance_break_down H.balance_break_down
|> register S.alias H.alias
|> register S.account_from_alias H.account_from_alias
end
module V1 = MakeRegisterer(Service.V1)(Handler.V1)
......
......@@ -86,6 +86,14 @@ module V1 = struct
let tops_kind_params params =
EzAPI.find_param Service.param_kind params
let spendable_params params =
match EzAPI.find_param Service.param_spendable params with
| None -> Some true
| Some b_str -> Some (bool_of_string b_str)
let block_hash_params params =
EzAPI.find_param Service.param_block_hash params
(** Block *)
(* /block/BHASH *)
let block (params, hash) () =
......@@ -265,12 +273,6 @@ module V1 = struct
let page, page_size = pagination_params params in
Dbr.delegator_rewards ?page ?page_size hash >>= EzAPIServer.return
let rewards_stats (params, hash) () =
let cycle = cycle_param params in
Lwt_io.printf "rewards_split/%s\n%!" hash >>= fun () ->
Dbr.rewards_stats hash ?cycle
>>= EzAPIServer.return
let account_status (_params, hash) () =
Lwt_io.printf "account_status/%s\n%!" hash >>= fun () ->
Dbr.account_status hash >>= EzAPIServer.return
......@@ -285,9 +287,10 @@ module V1 = struct
(** Operation *)
(* /operation/OHASH *)
let operation (_params, ohash) () =
let operation (params, ohash) () =
Lwt_io.printf "operation/%s\n%!" ohash >>= fun () ->
Dbr.operation ohash >>= function
let block_hash = block_hash_params params in
Dbr.operation ?block_hash ohash >>= function
| None -> Lwt.fail EzAPI.ResultNotfound
| Some op -> EzAPIServer.return op
......@@ -407,6 +410,12 @@ module V1 = struct
Lwt_io.printf "Request: 'cycle_baker_rights/%s'\n%!'" hash >>= fun () ->
Dbr.cycle_baker_rights hash >>= EzAPIServer.return
let cycle_all_rights (params,hash) () =
Lwt_io.printf "Request: 'cycle_all_rights/%s'\n%!" hash >>= fun () ->
let cycle = cycle_param params in
let prio = number_params params in
Dbr.cycle_all_rights ?cycle ?prio hash >>= EzAPIServer.return
(* /number_bakings/HASH *)
let nb_endorser_rights (params, hash) () =
Lwt_io.printf "Request: 'nb_endorser_rights/%s'\n%!" hash >>= fun () ->
......@@ -845,6 +854,76 @@ module V1 = struct
api_date ; api_versions } in
EzAPIServer.return api_server_info
(* Balance updates *)
let balance_updates_number (params,hash) () =
Lwt_io.printf "nu_balance_updates/\n%!" >>= fun () ->
let from = cycle_param params in
Dbr.nb_balance_updates ?from hash >>= EzAPIServer.return
let balance (_params,hash) () =
Lwt_io.printf "balance/\n%!" >>= fun () ->
Dbr.balance hash >>= EzAPIServer.return
let balance_updates (params,hash) () =
Lwt_io.printf "balance_updates/\n%!" >>= fun () ->
let from = cycle_param params in
let page, page_size = pagination_params params in
Dbr.balance_updates ?page ?page_size ?from hash >>= EzAPIServer.return
let active_balance_updates (params,hash) () =
Lwt_io.printf "active_balance_updates/\n%!" >>= fun () ->
let cycle_opt = cycle_param params in
match cycle_opt with
Some cycle -> Dbr.active_balance_updates cycle hash >>= EzAPIServer.return
| None -> EzAPIServer.return []
let balance_from_balance_updates (_params,hash) () =
Lwt_io.printf "balance_from_balance_updates/\n%!" >>= fun () ->
Dbr.balance_from_balance_updates hash >>= EzAPIServer.return
let balance_history (_params,hash) () =
Lwt_io.printf "balance_history/\n%!" >>= fun () ->
Dbr.balance_history hash >>= EzAPIServer.return
let cycle_frozen (params,hash) () =
Lwt_io.printf "cycle_frozen/\n%!" >>= fun () ->
match cycle_param params with
None -> EzAPIServer.return
{b_spendable = Int64.zero;
b_frozen = Int64.zero;
b_rewards = Int64.zero;
b_fees = Int64.zero;
b_deposits = Int64.zero}
| Some cycle ->
(Dbr.cycle_frozen cycle hash) >>= EzAPIServer.return
let balance_number params () =
Lwt_io.printf "balance_number/\n%!" >>= fun () ->
match cycle_param params with
None -> EzAPIServer.return 0
| Some cycle ->
(Dbr.nb_balance (Int32.of_int cycle)) >>= EzAPIServer.return
let balance_ranking params () =
Lwt_io.printf "balance_ranking/\n%!" >>= fun () ->
let page, page_size = pagination_params params in
match cycle_param params with
None -> EzAPIServer.return []
| Some cycle ->
match spendable_params params with
None -> EzAPIServer.return []
| Some spendable ->
Dbr.balance_ranking ?page ?page_size (Int32.of_int cycle) spendable >>= EzAPIServer.return
let last_baking_and_endorsement (_params, hash) () =
Lwt_io.printf "last baking and endorsement of %s\n%!" hash >>= fun () ->
Dbr.last_baking_and_endorsement hash >>= EzAPIServer.return
let next_baking_and_endorsement (_params, hash) () =
Lwt_io.printf "next baking and endorsement of %s\n%!" hash >>= fun () ->
Dbr.next_baking_and_endorsement hash >>= EzAPIServer.return
end
module V2 = V1
......@@ -85,15 +85,15 @@ let server services =
(List.map (fun (port,_) ->
string_of_int port) servers));
let update_table () =
Lwt.bind (Dbr.all_aliases ())
(fun l ->
Alias.reset ();
List.iter (fun (hash, alias) -> Alias.change_alias hash alias) l;
Lwt.return_unit)
Dbr.all_aliases () >>= fun l ->
Alias.reset ();
List.iter (fun (hash, alias) -> Alias.change_alias hash alias) l;
Lwt.return_unit
in
let rec update_loop () =
Printf.printf "update of alias table\n%!";
Printf.printf "Updating alias table [in progress]...\n%!";
update_table () >>= fun () ->
Printf.printf "Updating alias table [OK]...\n%!";
Lwt_unix.sleep !alias_latency >>= fun () ->
update_loop () in
Lwt_list.iter_p (fun x -> x)
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -219,10 +219,12 @@ type baking = {
bk_level : int;
bk_cycle : int;
bk_priority : int;
bk_missed_priority : int option;
bk_distance_level : int;
bk_fees : int64;
bk_bktime : int;
bk_baked : bool;
bk_tsp: string
}
type baking_endorsement = {
......@@ -233,7 +235,8 @@ type baking_endorsement = {
ebk_priority : int option;
ebk_dist : int option;
ebk_slots: int list option;
ebk_lr_nslot: int
ebk_lr_nslot: int;
ebk_tsp: string option
}
type bk_count = {
......@@ -322,6 +325,7 @@ type baker_stats = {
nb_blocks : int ;
volume_total : int64 ;
fees_total : int64 ;
nb_endorsements : int
}
type 'a per_day = {
......@@ -470,6 +474,9 @@ type service = {
srv_logo2 : string option; (* file name, related to /images/ *)
srv_descr : string option;
srv_sponsored : string option;
srv_page : string option;
srv_delegations_page : bool ;
srv_account_page : bool ;
}
type crawler_activity = {
......@@ -533,14 +540,6 @@ type delegator_reward = {
dor_status: rewards_status
}
type rewards_stats = {
rstats_staking_balance : int64 ;
rstats_delegators_nb : int ;
rstats_rewards : int64 ;
rstats_pc_blocks : float ;
rstats_pc_endorsements : float ;
}
type snapshot = {
snap_cycle : int ;
snap_index : int ;
......@@ -569,20 +568,40 @@ type h24_stats = {
type balance_update_info =
{bu_account : account_hash;
bu_block_hash : block_hash;
bu_diff : int64;
bu_date : Date.t;
bu_update_type : string;
bu_op_type : string;
bu_internal : bool;
bu_level : int32;
bu_frozen : bool;
bu_level : int32;
mutable bu_burn : bool
}
type balance =
{
b_spendable : int64;
b_frozen : int64;
b_rewards : int64;
b_fees : int64;
b_deposits : int64;
}
type chart_axis_type =
| Cat_Tz
| Cat_MuTz
| Cat_Other
type chart_file = {
chart_period : string;
chart_period_kind : string;
chart_name : string;
chart_pretty : string option;
chart_unit : chart_axis_type option;
chart_values : (string * float) array;
chart_abstract : string option;
chart_axis_title : string option;
}
type context_top_kind =
......@@ -598,12 +617,20 @@ type context_top_kind =
| TTotal_frozen_fees
| TUsed_bytes
(* top accounts at the beginning of this period *)
type context_top_accounts = {
context_top_period : string;
context_top_kind : string;
context_top_hash : string;
context_top_list : (string * int64) list ;
}
(* top accounts at the beginning of this period *)
type top_accounts = {
top_period : string;
top_kind : string;
top_hash : string;
top_list : (string * int64) list ;
top_list : (account_name * int64) list ;
}
type context_file_with_diff = {
......@@ -666,6 +693,7 @@ type api_server_config = {
conf_constants : ( int * constants ) list ; (* cycle x constants *)
conf_ico : ico_constants ;
conf_rampup_cycles : int ;
conf_start_reward_cycle : int ;
conf_has_delegation : bool ;
conf_has_marketcap : bool ;
}
......@@ -693,3 +721,41 @@ type www_server_info = {
mutable www_footer : string ;
mutable www_networks : ( string * string ) list ; (* name x www_url *)
}
type gk_coin = {gk_usd: float; gk_btc: float}
type gk_market_data = {
gk_price: gk_coin;
gk_market_volume: gk_coin;
gk_1h: gk_coin;
gk_24h: gk_coin;
gk_7d: gk_coin
}
type gk_ticker = {
gk_last: float;
gk_target: string;
gk_tsp: string;
gk_anomaly: bool;
gk_converted_last: gk_coin;
gk_volume: float;
gk_stale: bool;
gk_base: string;
gk_converted_volume: gk_coin;
gk_market: string
}
type gk_shell = {
gk_last_updated: string;
gk_market_data: gk_market_data;
gk_tickers: gk_ticker list
}
type ex_ticker = {
ex_base: string;
ex_target: string;
ex_volume: float;
ex_conversion: float;
ex_price_usd: float
}
type exchange_info = {
ex_name: string;
ex_total_volume: float;
ex_tickers: ex_ticker list
}
......@@ -67,6 +67,7 @@ let api =
] ;
conf_ico = betanet;
conf_rampup_cycles = 64;
conf_start_reward_cycle = 7;
conf_has_delegation = false ;
conf_has_marketcap = false ;
}
......@@ -121,3 +122,6 @@ let rampup ~cycle deposit =
if cycle >= rampup_cycles then deposit
else
Int64.(div (mul deposit (of_int cycle)) (of_int rampup_cycles))
let cycle_from_level ~cst level =
(level - 1) / cst.blocks_per_cycle
......@@ -32,3 +32,5 @@ val rampup : cycle:int -> int64 -> int64
(* `save_api_config filename` save current configuration into `filename` *)
val save_api_config : string -> unit
val cycle_from_level : cst:Tezos_types.constants -> int -> int
This diff is collapsed.
......@@ -28,6 +28,22 @@ OCaml.library("tezos-cache-lib",
}
);
OCaml.program("tezos-balance-checker",
ocaml+{
files = ["request.ml";"balance_checker.ml"];
has_byte = false;
requires = [
"data-types-lib";
"tezos-cache-lib";
"database-writer-lib";
"blake2b";
"ez-api-curl";
"database-reader-lib";
"database-writer-lib"
]
});
if ( has_pgocaml ){
OCaml.library("tezos-crawler-lib",
......
......@@ -64,10 +64,24 @@ let rec register main config hash =
let level = Request.level config hash in
debug "[Crawler] Found level %d\n%!" level.node_lvl_level ;
let block = Request.block config hash in
let date = block.node_header.header_timestamp in
let operations = List.flatten @@ block.node_operations in
debug "[Crawler] Found operations %d\n%!" @@ List.length operations ;
let t1 = Unix.gettimeofday () in
debug "[Crawler] [%d] Registering block %s\n%!" block.node_header.header_level hash ;
begin
if level.node_lvl_level = 1
then
let first_contracts = Request.node_contracts config hash in
List.iter
(fun (h,b) ->
Dbw.register_init_balance
h
b
date
level.node_lvl_level)
first_contracts
end;
Dbw.register_all block level operations ;
if main then
Dbw.register_main_chain !count block;
......@@ -105,7 +119,10 @@ let main_chain config =
=========\n%!" hash level current_level;
register main url hash)
(catchup_levels url hash current_level)
end in
end
(* Now we are up to date, we can register alt heads *)
else
alternative_heads_flag := true in
(* Try to register the current head (if predecessor doesn't exist,
try to get it first recursively). If until_mode is enabled, use
the hash given in CLI as new head. *)
......@@ -114,16 +131,21 @@ let main_chain config =
| None ->
Request.get_head_hash ?block:(!new_head) url
| Some hash -> hash in
(* Wait to be up to date before registering alt head (Cf. issue
when catching new chain from scratch) *)
alternative_heads_flag := false ;
register_or_not ~main:true url head_hash ;
debug "[Crawler] Registered block %s\n%!" head_hash ;
if !until_mode then exit 1 ;
if !alternative_heads_flag then begin
(* Registering alternative heads and operations in their branches *)
let heads = Request.get_alternative_heads_hashes url in
debug "[Crawler] Found heads %d\n%!" @@ List.length heads ;
List.iter (register_or_not url) @@ List.flatten heads ;
debug "[Crawler] Registered heads\n%!" ;
match Request.get_alternative_heads_hashes url with
| [] -> ()
| _ :: heads -> (* Ignore the first block *)
debug "[Crawler] Found heads %d\n%!" @@ List.length heads ;
List.iter (register_or_not url) @@ List.flatten heads ;
debug "[Crawler] Registered heads\n%!" ;
end;
if !pending_operations_flag then begin
......
......@@ -64,9 +64,6 @@ let get_data_path config =
(* Command-line arguments *)
let speclist = ref [
"--no-alternative-heads",
Arg.Clear alternative_heads_flag,
" Do not pull for alt-heads";
"--pending-operations",
Arg.Set pending_operations_flag,
" Pull for pending operations (expensive)";
......
......@@ -116,3 +116,30 @@ let get_alternative_heads_hashes url =
(* Shortcuts on [head] *)
let current_level ?(block=head) url = level url block
let head_block ?(block_hash=head) url = block url block_hash
let node_balance url block hash =
let path =
spf
"%s/blocks/%s/context/contracts/%s" root block hash in
(* debug "[crawler] node_balance %s\n%!" path ; *)
EzEncoding.destruct
Api_encoding.V1.Account_details.node_encoding @@
request ~cachable:(block <> head) url path
let node_contracts url bhash : (string * int64) list =
let contracts =
let path =
spf
"%s/blocks/%s/context/contracts/" root bhash in
EzEncoding.destruct
Tezos_encoding.Encoding.Contracts.encoding @@ request ~cachable:(bhash<>head) url path
in
List.map
(fun contract ->
let (_,b,_,_,_,_,_) = node_balance url bhash contract in contract,b)
contracts
let db_balance url account =
let path= spf "/v1/balance_from_balance_updates/%s" account in
EzEncoding.destruct (Api_encoding.V1.Balance.encoding) @@
request ~cachable:false url path
......@@ -44,5 +44,8 @@ val get_alternative_heads_hashes : url -> Tezos_types.block_hash list list
val current_level : ?block:string -> url -> Tezos_types.node_level
val pending_operations : url -> Tezos_types.pending_operation
val node_contracts : Data_types.url -> string -> (string * int64) list
val db_balance : Data_types.url -> string -> Data_types.balance
val request : ?post:bool -> cachable:bool -> Data_types.url -> string -> string
......@@ -16,34 +16,16 @@
open Data_types
let alias_table : (string, string) Hashtbl.t = Hashtbl.create 10000
let alias_table : (string, string) Hashtbl.t = Hashtbl.create 1000
let change_alias = Hashtbl.add alias_table
let change_alias tz alias = Hashtbl.add alias_table tz alias
let get_alias_tbl = Hashtbl.find_opt alias_table
let get_alias_tbl tz = Hashtbl.find_opt alias_table tz
let reset () = Hashtbl.reset alias_table
let dbh = PGOCaml.connect ~database:TzscanConfig.database ()
let get_alias_db hash =
let query =
PGSQL(dbh) "SELECT alias FROM user_alias WHERE tz = $hash" in
match query with
| [ alias ] -> Some alias
| _ -> None
let to_name ?(db=false) ?alias tz =
let to_name ?alias tz =
if tz = "" then {tz = ""; alias = None}
else
let alias =
if alias = None then
begin
if db then
get_alias_db tz
else
get_alias_tbl tz
end
else
alias in
{tz; alias}
let alias = if alias = None then get_alias_tbl tz else alias in
{ tz ; alias