Commit e78c685a authored by Sharon Urban's avatar Sharon Urban

refactoring,

added auto activation and extension capabilities
parent ab295b8b
......@@ -144,11 +144,12 @@
<PackageName>$([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[0])</PackageName>
<PackageVersion>$([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[1])</PackageVersion>
<AllPrivateAssets>$([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[4])</AllPrivateAssets>
<CopyLocal>$([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[5])</CopyLocal>
</PaketReferencesFileLinesInfo>
<PackageReference Include="%(PaketReferencesFileLinesInfo.PackageName)">
<Version>%(PaketReferencesFileLinesInfo.PackageVersion)</Version>
<PrivateAssets Condition=" ('%(PaketReferencesFileLinesInfo.AllPrivateAssets)' == 'true') Or ('$(PackAsTool)' == 'true') ">All</PrivateAssets>
<ExcludeAssets Condition="%(PaketReferencesFileLinesInfo.AllPrivateAssets) == 'exclude'">runtime</ExcludeAssets>
<ExcludeAssets Condition="%(PaketReferencesFileLinesInfo.CopyLocal) == 'false'">runtime</ExcludeAssets>
<Publish Condition=" '$(PackAsTool)' == 'true' ">true</Publish>
</PackageReference>
</ItemGroup>
......
......@@ -13,10 +13,13 @@ nuget NetMQ >= 4.0.0.175-pre
nuget FsNetMQ >= 0.2.8.0
nuget MongoDB.Driver
nuget MongoDB.FSharp
nuget Logary
nuget NodaTime 1.3.2 // specific version needed by Logary
source https://www.myget.org/F/zenprotocol/api/v2
nuget Zulib 0.3.36
nuget Infrastructure >= 0.9.14
nuget Consensus >= 0.9.14
nuget Api >= 0.9.14
nuget Zulib
nuget Infrastructure
nuget Consensus
nuget Api
nuget Wallet
\ No newline at end of file
......@@ -2,18 +2,19 @@ GENERATE-LOAD-SCRIPTS: ON
RESTRICTION: == net47
NUGET
remote: https://www.myget.org/F/zenprotocol/api/v2
Api (0.9.14)
Consensus (>= 0.9.14)
Api (0.9.21)
Consensus (>= 0.9.21)
FsBech32 (>= 0.1.0.7)
FSharp.Control.Reactive
FSharp.Core (4.3.4)
FSharp.Data
FsNetMQ (>= 0.2.8)
Infrastructure (>= 0.9.14)
Infrastructure (>= 0.9.21)
Logary
Newtonsoft.Json
Zulib (0.3.36)
Consensus (0.9.14)
Wallet (>= 0.9.21)
Zulib (0.3.37)
Consensus (0.9.21)
BouncyCastle
FsBech32 (>= 0.1.0.7)
FSharp.Compatibility.OCaml
......@@ -21,10 +22,10 @@ NUGET
FSharpx.Extras
FsNetMQ (>= 0.2.8)
FsPickler
Infrastructure (>= 0.9.14)
Infrastructure (>= 0.9.21)
Logary
Zulib (>= 0.0.1)
Infrastructure (0.9.14)
Infrastructure (0.9.21)
BouncyCastle
FSharp.Compiler.Tools (>= 10.0.2)
FSharp.Control.Reactive
......@@ -36,12 +37,24 @@ NUGET
Logary
System.Reactive.Compatibility
Zen.FSharp.Compiler.Service (17.0.2)
ZFS-Tools (0.0.23)
ZFStar (0.0.25)
ZFS-Tools (0.0.24)
ZFStar (0.0.26)
Wallet (0.9.21)
Base58Check
BouncyCastle
Consensus (>= 0.9.21)
FsBech32 (>= 0.1.0.7)
FSharp.Control.Reactive
FSharp.Core (4.3.4)
FsNetMQ (>= 0.2.8)
Infrastructure (>= 0.9.21)
Logary
NBitcoin
Zulib (0.3.37)
zen_z3_linux (4.5.1.8-v1f29cebd4df)
zen_z3_osx (4.5.1.8-v1f29cebd4df6)
zen_z3_windows (4.5.1.7-v1f29cebd4df)
Zulib (0.3.36)
Zulib (0.3.37)
BouncyCastle
FAKE (>= 4.0 < 5.0)
FsBech32 (>= 0.1.0.7)
......@@ -53,9 +66,10 @@ NUGET
zen_z3_linux (4.5.1.8-v1f29cebd4df)
zen_z3_osx (4.5.1.8-v1f29cebd4df6)
zen_z3_windows (4.5.1.7-v1f29cebd4df)
ZFStar (0.0.25)
ZFStar (0.0.26)
remote: https://www.nuget.org/api/v2
AsyncIO (0.1.40)
Base58Check (0.2)
BouncyCastle (1.8.2)
DnsClient (1.2)
System.Buffers (>= 4.4)
......@@ -108,13 +122,22 @@ NUGET
MongoDB.Bson (>= 2.7)
System.Runtime.InteropServices.RuntimeInformation (>= 4.0)
MongoDB.FSharp (0.1)
NBitcoin (4.1.1.46)
Newtonsoft.Json (>= 11.0.2)
System.Buffers (>= 4.5)
System.Net.Http (>= 4.3.3)
System.Net.Requests (>= 4.3)
NetMQ (4.0.0.175-pre)
AsyncIO (>= 0.1.40)
Newtonsoft.Json (11.0.2)
NodaTime (2.3)
NodaTime (1.3.2)
NUnit (3.10.1)
System.Buffers (4.5)
System.Collections.Immutable (1.5)
System.IO (4.3)
System.Net.Http (4.3.3)
System.Security.Cryptography.X509Certificates (>= 4.3)
System.Net.Requests (4.3)
System.Reactive (4.1)
System.Threading.Tasks.Extensions (>= 4.5.1)
System.ValueTuple (>= 4.4)
......@@ -157,13 +180,24 @@ NUGET
System.Threading.Tasks.Extensions (>= 4.5.1)
System.Reflection.Metadata (1.6)
System.Collections.Immutable (>= 1.5)
System.Runtime (4.3)
System.Runtime.CompilerServices.Unsafe (4.5.1)
System.Runtime.InteropServices.RuntimeInformation (4.3)
System.Security.Cryptography.Algorithms (4.3.1)
System.IO (>= 4.3)
System.Runtime (>= 4.3)
System.Security.Cryptography.Encoding (>= 4.3)
System.Security.Cryptography.Primitives (>= 4.3)
System.Security.Cryptography.Encoding (4.3)
System.Security.Cryptography.Primitives (4.3)
System.Security.Cryptography.X509Certificates (4.3.2)
System.Security.Cryptography.Algorithms (>= 4.3)
System.Security.Cryptography.Encoding (>= 4.3)
System.Threading.Tasks.Extensions (4.5.1)
System.Runtime.CompilerServices.Unsafe (>= 4.5)
System.ValueTuple (4.5)
Zen.FSharp.Compiler.Service (17.0.2)
System.Collections.Immutable (>= 1.3.1)
System.Reflection.Metadata (>= 1.4.2)
ZFS-Tools (0.0.23)
ZFStar (0.0.25)
ZFS-Tools (0.0.24)
ZFStar (0.0.26)
module Config
open System
let getEnv key =
let value = Environment.GetEnvironmentVariable(key)
if String.IsNullOrEmpty value then
failwith <| sprintf "missing environment variable: %s" key
else
value
type YamlConfig = FSharp.Configuration.YamlConfig<"config.yaml">
type Config = {
yaml: YamlConfig
password: string
}
let yamlConfig = new YamlConfig()
yamlConfig.Load("config.yaml")
let config = {
yaml = yamlConfig
password = getEnv "zen_account_password"
}
module Contract
open System
open System.IO
open Consensus
open Infrastructure
open Result
open FSharp.Data
open Util
open Json
open Config
let private contracCode =
Exception.resultWrap<String> (fun _ ->
(Platform.workingDirectory, config.yaml.contract)
|> Path.Combine
|> File.ReadAllText
) "error reading contract file"
let contractId =
contracCode
<@> Contract.makeContractId Types.Version0
|> get
let private activate() = result {
let! code = contracCode
let! activatedContractId =
getResponseBody (fun _ ->
"wallet/contract/activate"
|> getUri
|> (contractActivateRequestJson code config.yaml.acs.activate).Request
) "error activating contract"
>>= parseContractActivateResponseJson
if activatedContractId <> contractId then
return! Error "error activating contract"
else
info "contract activated"
}
let private extend() = result {
let address =
contractId
|> Wallet.Address.Contract
|> Wallet.Address.encode chain
do! getResponseBody (fun _ ->
"wallet/contract/extend"
|> getUri
|> (contractExtendRequestJson address config.yaml.acs.extend).Request
) "error extending contract"
<@> fun _ -> info "contract executed"
}
let private getTip() =
getResponseBody (fun _ ->
Http.Request(
"blockchain/info"
|> getUri,
httpMethod = "GET"
)) "error getting blockchain info"
>>= parseBlockChainInfoJson
<@> fun info -> info.Blocks
let ensureActive() = result {
let getActivationState() =
getResponseBody (fun _ ->
Http.Request(
getUri "contract/active",
httpMethod = "GET"
))
"error getting ACS"
>>= parseActiveContractsResponseJson
<@> List.tryFind (fst >> (=) contractId)
<@> Option.map snd
let! activationState = getActivationState()
match activationState with
| None ->
info "contract is not active - activating..."
do! activate()
| Some activationState ->
let! tip = getTip()
if activationState - tip < config.yaml.acs.threshold then
info <| sprintf "contract active for %d blocks, extending..." (activationState - tip)
do! extend()
}
let execute action json =
getResponseBody (fun _ ->
getUri action
|> (json:JsonValue).Request
) "error executing contract"
<@> fun _ -> info "contract executed"
\ No newline at end of file
......@@ -31,13 +31,14 @@ let insert data =
data = data
}
|> collection.InsertOne
data
let find date =
let records = collection.Find(fun x -> x.date = date).ToEnumerable()
if Seq.length records = 1 then
(Seq.head records).data
|> Some
else
None
\ No newline at end of file
let list() =
collection.Find(fun _ -> true).ToEnumerable()
let count =
list >> Seq.length
let find (date:DateTime) =
list()
|> Seq.tryFind (fun x -> x.date >= date)
|> Option.map (fun x -> x.data)
\ No newline at end of file
module Intrinio
open System
open FSharp.Data
open Util
open Json
open Config
open Infrastructure.Result
let basicAuthHeader =
sprintf "%s:%s" (getEnv "intrinio_user_name") (getEnv "intrinio_password")
|> getBytes
|> Convert.ToBase64String
let fetch() =
getResponseBody (fun _ ->
Http.Request(
"https://api.intrinio.com/data_point",
httpMethod = "GET",
query = [
"identifier", String.Join(",", config.yaml.tickers)
"item", "close_price"
],
headers = [
"Authorization", basicAuthHeader
]
)
) "error getting provider data"
>>= parseRawResultJson
module Json
open FSharp.Data
open Api.Types
open Infrastructure
open Result
open Consensus
open Util
open Wallet
open Config
type RawResultJson = JsonProvider<"""
{
......@@ -18,3 +25,50 @@ type AuditPathResponseJson = JsonProvider<"""
"abcd123"
]
}""">
let parseRawResultJson raw =
Exception.resultWrap<RawResultJson.Root> (fun _ ->
RawResultJson.Parse(raw))
"error parsing provider data"
let parseActiveContractsResponseJson raw =
Exception.resultWrap<ActiveContractsResponseJson.Root[]> (fun _ ->
ActiveContractsResponseJson.Parse(raw))
"error parsing ACS response"
<@> Array.map (fun root -> ContractId.fromString root.ContractId, root.Expire)
<@> Array.toList
>>= traverseResultM (fun (contractId, exiry) ->
ofOption "error parsing contractid" contractId
<@> fun contractId -> contractId, exiry
)
let parseContractActivateResponseJson raw =
Exception.resultWrap<ContractActivateResponseJson.Root> (fun _ ->
ContractActivateResponseJson.Parse(raw))
"error parsing contract activation response"
<@> fun root -> root.ContractId
<@> ContractId.fromString
>>= ofOption "error parsing contractid of activation response"
let parseBlockChainInfoJson raw =
Exception.resultWrap<BlockChainInfoJson.Root> (fun _ ->
BlockChainInfoJson.Parse(raw))
"error parsing blockchain info response"
let contractExecuteRequestJson contractId command provideReturnAddress signingKeyDerivationPath spends messageBody =
(new ContractExecuteRequestJson.Root(
contractId
|> Address.Contract
|> Address.encode chain,
command,
messageBody,
new ContractExecuteRequestJson.Options(provideReturnAddress, signingKeyDerivationPath),
spends,
config.password
)).JsonValue
let contractActivateRequestJson code numberOfBlocks =
(new ContractActivateRequestJson.Root(code, numberOfBlocks, config.password)).JsonValue
let contractExtendRequestJson address numberOfBlocks =
(new ContractExtendRequestJson.Root(address, numberOfBlocks, config.password)).JsonValue
module Main
open System
open FSharp.Configuration
open FSharp.Data
open Infrastructure
open Result
open Consensus
open Serialization
open FsBech32
open Json
open Http
open Util
open Server
open Api.Types
open Contract
open Config
type Config = YamlConfig<"config.yaml">
[<EntryPoint>]
let main _ =
let intrinioUserName = getEnv "intrinio_user_name"
let intrinioPassword = getEnv "intrinio_password"
let zenPassword = getEnv "zen_account_password"
let basicAuthHeader =
sprintf "%s:%s" intrinioUserName intrinioPassword
|> getBytes
|> Convert.ToBase64String
let config = new Config()
config.Load("config.yaml")
initHttpServer config.api
while true do
printfn "fetching..."
Server.init config.yaml.api
Exception.resultWrap<String> (fun _ ->
Http.RequestString (
"https://api.intrinio.com/data_point",
httpMethod = "GET",
query = [
"identifier", String.Join(",", config.tickers)
"item", "close_price"
],
headers = [
"Authorization", basicAuthHeader
]
)) "error getting provider data"
>>= (fun raw ->
printfn "parsing..."
Exception.resultWrap<RawResultJson.Root> (
fun _ -> RawResultJson.Parse(raw)) "error parsing provider data")
<@> fun x -> x.Data
<@> Array.map (fun x -> x.Identifier, x.Value)
<@> (fun data ->
data
|> Array.map Merkle.hashLeaf
|> Array.toList
|> MerkleTree.computeRoot
|> Hash.bytes
|> Zen.Types.Data.data.Hash
|> Data.serialize
|> Base16.encode
|> (fun messageBody ->
(new ContractExecuteRequestJson.Root(
config.contract,
"Add",
messageBody,
new ContractExecuteRequestJson.Options(false, "m/44'/258'/0'/3/0"),
Array.empty,
zenPassword
)))
|> (fun json ->
printfn "making a commitment on the blockchain..."
while true do
result {
do! Contract.ensureActive()
let! data = Intrinio.fetch()
let data =
data.Data
|> Array.map (fun root -> root.Identifier, root.Value)
Exception.resultWrap<HttpResponse> (fun _ ->
(sprintf "http://%s/wallet/contract/execute" config.nodeApi)
|> json.JsonValue.Request) "error communicating with zen-node"
)
>>= (fun response ->
if response.StatusCode <> 200 then
sprintf "could not execute contract: %A" response.Body
|> Error
else
Ok data)
<@> DataAccess.insert
)
|> Result.mapError (printfn "%A")
do! data
|> Array.map Merkle.hashLeaf
|> Array.toList
|> MerkleTree.computeRoot
|> Hash.bytes
|> Zen.Types.Data.data.Hash
|> Data.serialize
|> Base16.encode
|> contractExecuteRequestJson contractId "Add" false "m/44'/258'/0'/3/0" Array.empty
|> execute "wallet/contract/execute"
DataAccess.insert data
info <| sprintf "data successfully persisted. %d record(s) found" (DataAccess.count())
}
|> mapError error
|> ignore
printfn "waiting..."
Threading.Thread.Sleep (1000 * 60 * 60 * config.interval)
info "waiting..."
Threading.Thread.Sleep (1000 * 60 * 60 * config.yaml.interval)
0
\ No newline at end of file
......@@ -6,10 +6,12 @@ open FSharp.Control.Reactive
open Infrastructure.Http
open Consensus
open FSharp.Data
open Logary
open Util
module Actor = FsNetMQ.Actor
let initHttpServer address =
let init address =
Actor.create (fun shim ->
use poller = Poller.create ()
use observer = Poller.registerEndMessage poller shim
......@@ -51,6 +53,26 @@ let initHttpServer address =
reply StatusCode.BadRequest (TextContent "ticker not found or bad date format")
| _ ->
reply StatusCode.BadRequest (TextContent "missing timestamp and ticker")
| Get ("/list", _) ->
DataAccess.list()
|> Seq.map (fun item ->
JsonValue.Record [|
"date", item.date
|> Util.stringifyDate
|> JsonValue.String
"data", item.data
|> Array.map (fun (ticker, value) ->
JsonValue.Record [|
"ticker", JsonValue.String ticker
"value", JsonValue.Number value
|])
|> Seq.toArray
|> JsonValue.Array
|])
|> Seq.toArray
|> JsonValue.Array
|> JsonContent
|> reply StatusCode.OK
| _ ->
reply StatusCode.NotFound NoContent
) (fun error ->
......
......@@ -2,20 +2,64 @@ module Util
open System
open System.Text
open Logary.Message
open Infrastructure
open Result
open FSharp.Data
open Config
let result = new ResultBuilder<string>()
let getBytes value =
ASCIIEncoding.ASCII.GetBytes (value:string)
let getEnv key =
let value = Environment.GetEnvironmentVariable(key)
if String.IsNullOrEmpty value then
failwith <| sprintf "missing environment variable: %s" key
else
value
[<Literal>]
let DateFormat = "yyyy-MM-dd"
let parseDate date =
try
DateTime.ParseExact(date, "yyyy-MM-dd", Globalization.CultureInfo.InvariantCulture)
DateTime.ParseExact(date, DateFormat, Globalization.CultureInfo.InvariantCulture)
|> Some
with _ ->
None
\ No newline at end of file
None
let stringifyDate (date : DateTime) =
date.ToString(DateFormat)
let debug message =
eventX message
|> Log.debug
message
let info message =
eventX message
|> Log.info
let error message =
eventX "{message}"
>> setField "message" (message:String)
|> Log.error
let getUri action =
sprintf "http://%s/%s" config.yaml.nodeApi action
|> debug
let getResponseBody request errorMessage = result {