Commit d38338ca authored by David Vorick's avatar David Vorick

Merge branch 'master' into testing

parents 82f74584 548d9396
......@@ -7,7 +7,7 @@ Errors are sent as plaintext, accompanied by an appropriate status code.
| Path | Params | Response |
|:------------------|:---------------------------------|:-----------------------------|
| /host/config | | See HostInfo |
| /host/setconfig | See HostInfo | |
| /host/setconfig | See HostSettings | |
| /miner/start | `threads` | |
| /miner/status | | See MinerInfo |
| /miner/stop | | |
......@@ -27,21 +27,32 @@ Errors are sent as plaintext, accompanied by an appropriate status code.
| /stop | | |
| /sync | | |
HostInfo comprises the following values:
HostInfo is a JSON object containing the following values:
```
totalstorage
minfile
maxfile
mintolerance
minduration
maxduration
minwin
maxwin
freezeduration
price
penalty
freezevolume
TotalStorage"
"Minfilesize"
"Maxfilesize"
"Minwindow"
"Minduration"
"Maxduration"
"Price"
"Collateral"
"StorageRemaining"
"NumContracts"
``
HostSettings: the following parameters can be set via `/host/setconfig`:
```
totalstorage
minfilesize
maxfilesize
minwindow
minduration
maxduration
price
collateral
```
Only the values specified in the call will be updated.
WalletInfo is a JSON object containing the following fields:
```
......
......@@ -6,9 +6,15 @@ all: install
fmt:
go fmt ./...
# REBUILD touches all of the build-dependent source files, forcing them to be
# rebuilt. This is necessary because the go tool is not smart enough to trigger
# a rebuild when build tags have been changed.
REBUILD:
@touch consensus/build*.go
# install builds and installs developer binaries.
install: fmt
go install -a -tags=dev ./...
install: fmt REBUILD
go install -tags=dev ./...
# clean removes all directories that get automatically created during
# development.
......@@ -24,24 +30,23 @@ clean:
# flag results in a multi-second compile time, which is undesirable. Leaving
# out both the touch and the '-a' means that sometimes the tests will be run
# using the developer constants, which is very slow.
test: clean fmt
@touch consensus/blocknode.go
test: clean fmt REBUILD
go test -short -tags=test ./...
# test-long does a forced rebuild of all packages, and then runs both the
# short and long tests with the race libraries enabled. test-long aims to be
# thorough.
test-long: clean fmt
go test -a -v -race -short -tags=test ./...
go test -a -v -race -tags=test ./...
test-long: clean fmt REBUILD
go test -v -race -short -tags=test ./...
go test -v -race -tags=test ./...
# cover runs the long tests and creats html files that show you which lines
# have been hit during testing and how many times each line has been hit.
coverpackages = consensus crypto encoding hash modules/hostdb network siad
cover: clean
cover: clean REBUILD
@mkdir -p cover/modules
@for package in $(coverpackages); do \
go test -a -v -tags=test -covermode=atomic -coverprofile=cover/$$package.out ./$$package ; \
go test -v -tags=test -covermode=atomic -coverprofile=cover/$$package.out ./$$package ; \
go tool cover -html=cover/$$package.out -o=cover/$$package.html ; \
rm cover/$$package.out ; \
done
......@@ -67,15 +72,14 @@ dependencies:
go get -u golang.org/x/tools/cmd/cover
# release builds and installs release binaries.
release: dependencies test-long
go install -a ./...
release: dependencies test-long REBUILD
go install ./...
# xc builds and packages release binaries for all systems by using goxc.
# Cross Compile - makes binaries for windows, linux, and mac, 32 and 64 bit.
xc: dependencies test-long
xc: dependencies test-long REBUILD
goxc -arch="amd64" -bc="linux windows darwin" -d=release -pv=0.2.0 \
-br=release -pr=beta -include=example-config,LICENSE*,README* \
-tasks-=deb,deb-dev,deb-source,go-test
# Need some command here to make sure that the release constants got used.
.PHONY: all fmt install clean test test-long cover whitepaper dependencies release xc
.PHONY: all fmt install clean test test-long cover whitepaper dependencies release xc REBUILD
......@@ -60,7 +60,7 @@ func (s *State) validProof(sp StorageProof) error {
if err != nil {
return err
}
verified := hash.VerifyReaderProof(
verified := hash.VerifySegment(
sp.Segment,
sp.HashSet,
hash.CalculateSegments(contract.FileSize),
......
......@@ -29,6 +29,8 @@ type (
Target hash.Hash
)
var ZeroAddress = CoinAddress{0}
// A Block contains all of the changes to the state that have occurred since
// the previous block. There are constraints that make it difficult and
// rewarding to find a block.
......@@ -89,7 +91,7 @@ type Output struct {
// stores a file.
type FileContract struct {
FileMerkleRoot hash.Hash
FileSize uint64 // in bytes
FileSize uint64
Start, End BlockHeight
Payout Currency
ValidProofAddress CoinAddress
......
......@@ -7,8 +7,7 @@ import (
)
const (
HashSize = 32
SegmentSize = 64 // Size of smallest piece of a file which gets hashed when building the Merkle tree.
HashSize = 32
)
type (
......
......@@ -64,7 +64,7 @@ func TestStorageProof(t *testing.T) {
numSegments := uint64(7)
data := make([]byte, numSegments*SegmentSize)
rand.Read(data)
rootHash, err := ReaderMerkleRoot(bytes.NewReader(data), numSegments)
rootHash, err := BytesMerkleRoot(data)
if err != nil {
t.Fatal(err)
}
......@@ -76,7 +76,7 @@ func TestStorageProof(t *testing.T) {
t.Error(err)
continue
}
if !VerifyReaderProof(baseSegment, hashSet, numSegments, i, rootHash) {
if !VerifySegment(baseSegment, hashSet, numSegments, i, rootHash) {
t.Error("Proof", i, "did not pass verification")
}
}
......
package hash
const (
SegmentSize = 64 // number of bytes that are hashed to form each base leaf of the Merkle tree
)
// Helper function for Merkle trees; takes two hashes, concatenates them,
// and hashes the result.
func JoinHash(left, right Hash) Hash {
......
......@@ -6,31 +6,46 @@ import (
"io"
)
// BytesMerkleRoot takes a byte slice and returns the merkle root created by
// Calculates the number of segments in the file when building a Merkle tree.
// Should probably be renamed to CountLeaves() or something.
//
// TODO: Why is this in package hash?
func CalculateSegments(fileSize uint64) (numSegments uint64) {
numSegments = fileSize / SegmentSize
if fileSize%SegmentSize != 0 {
numSegments++
}
return
}
// BytesMerkleRoot takes a byte slice and returns the Merkle root created by
// splitting the slice into small pieces and then treating each piece as an
// element of the tree.
func BytesMerkleRoot(data []byte) (hash Hash, err error) {
reader := bytes.NewReader(data)
numSegments := CalculateSegments(uint64(len(data)))
return ReaderMerkleRoot(reader, numSegments)
func BytesMerkleRoot(data []byte) (Hash, error) {
return ReaderMerkleRoot(bytes.NewReader(data), uint64(len(data)))
}
// ReaderMerkleRoot splits the provided data into segments. It then recursively
// transforms these segments into a Merkle tree, and returns the root hash.
// See MerkleRoot for a diagram of how Merkle trees are constructed.
func ReaderMerkleRoot(reader io.Reader, numSegments uint64) (hash Hash, err error) {
func ReaderMerkleRoot(r io.Reader, size uint64) (Hash, error) {
return readerMerkleRoot(r, CalculateSegments(size))
}
func readerMerkleRoot(r io.Reader, numSegments uint64) (hash Hash, err error) {
if numSegments == 0 {
err = errors.New("no data")
return
}
if numSegments == 1 {
data := make([]byte, SegmentSize)
n, _ := reader.Read(data)
if n == 0 {
err = errors.New("no data")
} else {
hash = HashBytes(data)
_, err = io.ReadFull(r, data)
// early EOF is an acceptable error. Actually, it's guaranteed to
// occur unless the filesize is an exact multiple of SegmentSize.
if err == io.ErrUnexpectedEOF {
err = nil
}
hash = HashBytes(data)
return
}
......@@ -41,21 +56,12 @@ func ReaderMerkleRoot(reader io.Reader, numSegments uint64) (hash Hash, err erro
}
// since we always read "left to right", no extra Seeking is necessary
left, err := ReaderMerkleRoot(reader, mid)
right, err := ReaderMerkleRoot(reader, numSegments-mid)
hash = JoinHash(left, right)
return
}
// Calculates the number of segments in the file when building a merkle tree.
// Should probably be renamed to CountLeaves() or something.
//
// TODO: Why is this in package hash?
func CalculateSegments(fileSize uint64) (numSegments uint64) {
numSegments = fileSize / SegmentSize
if fileSize%SegmentSize != 0 {
numSegments++
left, err := readerMerkleRoot(r, mid)
if err != nil {
return
}
right, err := readerMerkleRoot(r, numSegments-mid)
hash = JoinHash(left, right)
return
}
......@@ -64,7 +70,7 @@ func CalculateSegments(fileSize uint64) (numSegments uint64) {
// to the root. On each level of the tree, we must provide the hash of the
// "sister" node. (Since this is a binary tree, the sister node is the other
// node with the same parent as us.) To obtain this hash, we call
// ReaderMerkleRoot on the segment of data corresponding to the sister. This
// readerMerkleRoot on the segment of data corresponding to the sister. This
// segment will double in size on each iteration until we reach the root.
//
// TODO: Gain higher certianty of correctness.
......@@ -73,12 +79,12 @@ func BuildReaderProof(rs io.ReadSeeker, numSegments, proofIndex uint64) (baseSeg
if _, err = rs.Seek(int64(proofIndex)*int64(SegmentSize), 0); err != nil {
return
}
if _, err = rs.Read(baseSegment[:]); err != nil {
if _, err = io.ReadFull(rs, baseSegment[:]); err != nil && err != io.ErrUnexpectedEOF {
return
}
// Construct the hash set that proves the base segment is a part of the
// merkle tree of the reader. (Verifier needs to know the merkle root of
// Merkle tree of the reader. (Verifier needs to know the Merkle root of
// the file in advance.)
for size := uint64(1); size < numSegments; size <<= 1 {
// determine sister index
......@@ -103,7 +109,7 @@ func BuildReaderProof(rs io.ReadSeeker, numSegments, proofIndex uint64) (baseSeg
// calculate and append hash
var h Hash
h, err = ReaderMerkleRoot(rs, truncSize)
h, err = readerMerkleRoot(rs, truncSize)
if err != nil {
return
}
......@@ -113,7 +119,7 @@ func BuildReaderProof(rs io.ReadSeeker, numSegments, proofIndex uint64) (baseSeg
return
}
// verifyProof traverses a StorageProof, hashing elements together to produce
// VerifySegment traverses a hash set, hashing elements together to produce
// the root-level hash, which is then checked against the expected result.
// Care must be taken to ensure that the correct ordering is used when
// concatenating hashes.
......@@ -127,7 +133,7 @@ func BuildReaderProof(rs io.ReadSeeker, numSegments, proofIndex uint64) (baseSeg
// indicates "keep." I don't know why this works, I just noticed the pattern.
//
// TODO: Gain higher certainty of correctness.
func VerifyReaderProof(baseSegment [SegmentSize]byte, hashSet []Hash, numSegments, proofIndex uint64, expectedRoot Hash) bool {
func VerifySegment(baseSegment [SegmentSize]byte, hashSet []Hash, numSegments, proofIndex uint64, expectedRoot Hash) bool {
h := HashBytes(baseSegment[:])
depth := uint64(0)
......
......@@ -4,29 +4,54 @@ import (
"net"
"github.com/NebulousLabs/Sia/consensus"
"github.com/NebulousLabs/Sia/network"
)
const (
AcceptContractResponse = "accept"
)
// ContractTerms are the parameters agreed upon by a client and a host when
// forming a FileContract.
type ContractTerms struct {
FileSize uint64
StartHeight consensus.BlockHeight
WindowSize consensus.BlockHeight // how many blocks a host has to submit each proof
NumWindows uint64
Price consensus.Currency // client contribution towards payout each window
Collateral consensus.Currency // host contribution towards payout each window
ValidProofAddress consensus.CoinAddress
MissedProofAddress consensus.CoinAddress
}
type HostInfo struct {
HostSettings
StorageRemaining int64
NumContracts int
}
type Host interface {
// Announce puts an annoucement out so that clients can find the host.
AnnounceHost(freezeVolume consensus.Currency, freezeUnlockHeight consensus.BlockHeight) (consensus.Transaction, error)
// Announce announces the host on the blockchain. A host announcement
// requires two things: the host's address, and a volume of "frozen"
// (time-locked) coins, used to mitigate Sybil attacks.
Announce(addr network.Address, freezeValue consensus.Currency, freezeDuration consensus.BlockHeight) error
// NegotiateContract is a strict function that enables a client to
// communicate with the host to propose a contract.
//
// TODO: enhance this documentataion. For now, see the host package for a
// reference implementation.
NegotiateContract(conn net.Conn) error
NegotiateContract(net.Conn) error
// RetrieveFile is a strict function that enables a client to download a
// file from a host.
RetrieveFile(conn net.Conn) error
RetrieveFile(net.Conn) error
// Returns the number of contracts being managed by the host.
//
// TODO: Switch all of this to a status struct.
NumContracts() int
// SetConfig sets the hosting parameters of the host.
SetConfig(HostSettings)
// Info returns info about the host, including its hosting parameters, the
// amount of storage remaining, and the number of active contracts.
Info() HostInfo
}
......@@ -2,46 +2,65 @@ package host
import (
"github.com/NebulousLabs/Sia/consensus"
// "github.com/NebulousLabs/Sia/encoding"
// "github.com/NebulousLabs/Sia/sia/components"
"github.com/NebulousLabs/Sia/encoding"
"github.com/NebulousLabs/Sia/modules"
"github.com/NebulousLabs/Sia/network"
)
// HostAnnounceSelf creates a host announcement transaction, adding
// information to the arbitrary data and then signing the transaction.
func (h *Host) AnnounceHost(freezeVolume consensus.Currency, freezeUnlockHeight consensus.BlockHeight) (t consensus.Transaction, err error) {
// Announce creates a host announcement transaction, adding information to the
// arbitrary data, signing the transaction, and submitting it to the
// transaction pool.
func (h *Host) Announce(addr network.Address, freezeVolume consensus.Currency, freezeDuration consensus.BlockHeight) (err error) {
h.mu.Lock()
defer h.mu.Unlock()
/*
// Get the encoded announcement based on the host settings.
info := h.announcement
// Fill out the transaction.
id, err := h.wallet.RegisterTransaction(t)
if err != nil {
return
}
err = h.wallet.FundTransaction(id, freezeVolume)
if err != nil {
return
}
info.SpendConditions, info.FreezeIndex, err = h.wallet.AddTimelockedRefund(id, freezeVolume, freezeUnlockHeight)
if err != nil {
return
}
announcement := string(encoding.MarshalAll(components.HostAnnouncementPrefix, info))
err = h.wallet.AddArbitraryData(id, announcement)
if err != nil {
return
}
// TODO: Have the wallet manually add a fee? How should this be managed?
t, err = h.wallet.SignTransaction(id, true)
if err != nil {
return
}
h.state.AcceptTransaction(t)
*/
// get current state height
h.state.RLock()
freezeUnlockHeight := h.state.Height() + freezeDuration
h.state.RUnlock()
// create the transaction that will hold the announcement
var t consensus.Transaction
id, err := h.wallet.RegisterTransaction(t)
if err != nil {
return
}
err = h.wallet.FundTransaction(id, freezeVolume)
if err != nil {
return
}
spendHash, spendConditions, err := h.wallet.TimelockedCoinAddress(freezeUnlockHeight)
if err != nil {
return
}
output := consensus.Output{
Value: freezeVolume,
SpendHash: spendHash,
}
freezeIndex, err := h.wallet.AddOutput(id, output)
if err != nil {
return
}
// create and encode the announcement
announcement := encoding.Marshal(modules.HostAnnouncement{
IPAddress: addr,
FreezeIndex: freezeIndex,
SpendConditions: spendConditions,
})
// add announcement to arbitrary data field
err = h.wallet.AddArbitraryData(id, modules.HostAnnouncementPrefix+string(announcement))
if err != nil {
return
}
// TODO: Have the wallet manually add a fee? How should this be managed?
t, err = h.wallet.SignTransaction(id, true)
if err != nil {
return
}
h.tpool.AcceptTransaction(t)
return
}
This diff is collapsed.
......@@ -5,8 +5,8 @@ import (
"io"
"net"
"os"
"path/filepath"
"sync"
"time"
"github.com/NebulousLabs/Sia/consensus"
"github.com/NebulousLabs/Sia/encoding"
......@@ -23,20 +23,23 @@ const (
)
type contractObligation struct {
filename string // Where on disk the file is stored.
path string // Where on disk the file is stored.
}
type Host struct {
state *consensus.State
tpool modules.TransactionPool
wallet modules.Wallet
latestBlock consensus.BlockID
hostDir string
// announcement modules.HostAnnouncement
// our HostSettings, embedded for convenience
modules.HostSettings
hostDir string
spaceRemaining int64
fileCounter int
contracts map[consensus.ContractID]contractObligation // The string is filepath of the file being stored.
contracts map[consensus.ContractID]contractObligation
mu sync.RWMutex
}
......@@ -52,7 +55,7 @@ func New(state *consensus.State, wallet modules.Wallet) (h *Host, err error) {
return
}
// addr, _, err := wallet.CoinAddress()
addr, _, err := wallet.CoinAddress()
if err != nil {
return
}
......@@ -60,16 +63,15 @@ func New(state *consensus.State, wallet modules.Wallet) (h *Host, err error) {
state: state,
wallet: wallet,
/*
announcement: modules.HostAnnouncement{
MaxFilesize: 4 * 1000 * 1000,
MaxDuration: 1008, // One week.
MinWindow: 20,
Price: 1,
Burn: 1,
CoinAddress: addr,
},
*/
// default host settings
HostSettings: modules.HostSettings{
MaxFilesize: 4 * 1000 * 1000,
MaxDuration: 1008, // One week.
MinWindow: 20,
Price: 1,
Collateral: 1,
CoinAddress: addr,
},
contracts: make(map[consensus.ContractID]contractObligation),
}
......@@ -96,19 +98,20 @@ func (h *Host) RetrieveFile(conn net.Conn) (err error) {
// Verify the file exists, using a mutex while reading the host.
h.mu.RLock()
contractObligation, exists := h.contracts[contractID]
h.mu.RUnlock()
if !exists {
h.mu.RUnlock()
return errors.New("no record of that file")
}
h.mu.RUnlock()
// Open the file.
fullname := filepath.Join(h.hostDir, contractObligation.filename)
file, err := os.Open(fullname)
file, err := os.Open(contractObligation.path)
if err != nil {
return
}
defer file.Close()
info, _ := file.Stat()
conn.SetDeadline(time.Now().Add(time.Duration(info.Size()) * 8 * time.Microsecond))
// Transmit the file.
_, err = io.Copy(conn, file)
......@@ -119,9 +122,29 @@ func (h *Host) RetrieveFile(conn net.Conn) (err error) {
return
}
// TODO: Deprecate this function.
func (h *Host) NumContracts() int {
// SetConfig updates the host's internal HostSettings object. To modify
// a specific field, use a combination of Info and SetConfig
func (h *Host) SetConfig(settings modules.HostSettings) {
h.mu.Lock()
defer h.mu.Unlock()
return len(h.contracts)
h.HostSettings = settings
}
// Settings is an RPC used to request the settings of a host.
func (h *Host) Settings() (modules.HostSettings, error) {
// TODO: return an error if we haven't announced yet
return h.HostSettings, nil
}
func (h *Host) Info() modules.HostInfo {
h.mu.RLock()
defer h.mu.RUnlock()
info := modules.HostInfo{
HostSettings: h.HostSettings,
StorageRemaining: h.spaceRemaining,
NumContracts: len(h.contracts),
}
return info
}
package host
import (
// "github.com/NebulousLabs/Sia/modules"
)
type HostInfo struct {
// Announcement modules.HostEntry
StorageRemaining int
ContractCount int
}
func (h *Host) Info() (info HostInfo, err error) {
h.mu.RLock()
defer h.mu.RUnlock()
info = HostInfo{
// Announcement: h.announcement,
StorageRemaining: int(h.spaceRemaining),
ContractCount: len(h.contracts),
}
return
}
......@@ -3,12 +3,20 @@ package host
import (
"errors"
"os"
"path/filepath"
"github.com/NebulousLabs/Sia/consensus"
"github.com/NebulousLabs/Sia/hash"
)
// ContractEntry houses a single contract with its id - you cannot derive the
// id of a contract without having the transaction. Rather than keep the whole
// transaction, we store only the id.
// TODO: is this needed?
type ContractEntry struct {
ID consensus.ContractID
Contract consensus.FileContract
}
// TODO: Hold off on both storage proofs and deleting files for a few blocks
// after the first possible opportunity to reduce risk of loss due to
// blockchain reorganization.
......@@ -123,27 +131,23 @@ func (h *Host) threadedConsensusListen(updateChan chan struct{}) {
// Create a proof of storage for a contract, using the state height to
// determine the random seed. Create proof must be under a host and state lock.
func (h *Host) createStorageProof(entry ContractEntry, heightForProof consensus.BlockHeight) (sp consensus.StorageProof, err error) {
// Get the file associated with the contract.
contractObligation, exists := h.contracts[entry.ID]
if !exists {
err = errors.New("no record of that file")
return
}
fullname := filepath.Join(h.hostDir, contractObligation.filename)
// Open the file.
file, err := os.Open(fullname)
file, err := os.Open(contractObligation.path)
if err != nil {
return
}
defer file.Close()
// Build the proof using the hash library.
numSegments := hash.CalculateSegments(entry.Contract.FileSize)
segmentIndex, err := h.state.StorageProofSegment(entry.ID)
if err != nil {
return
}
numSegments := hash.CalculateSegments(entry.Contract.FileSize)
base, hashSet, err := hash.BuildReaderProof(file, numSegments, segmentIndex)
if err != nil {
return
......
......@@ -6,24 +6,45 @@ import (
)
const (
HostAnnouncementPrefix = 1
// Denotes a host announcement in the Arbitrary Data section.
HostAnnouncementPrefix = "HostAnnouncement"
)
// the Host struct is kept in the client package because it's what the client
// uses to weigh hosts and pick them out when storing files.
// HostAnnouncements are stored in the Arbitrary Data section of transactions
// on the blockchain. They announce the willingness of a node to host files.
// Renters can contact the host privately to obtain more detailed hosting
// parameters (see HostSettings). To mitigate Sybil attacks, HostAnnouncements
// are paired with a volume of 'frozen' coins. The FreezeIndex indicates which
// output in the transaction contains the frozen coins, and the
// SpendConditions indicate the number of blocks the coins are frozen for.
type HostAnnouncement struct {
IPAddress network.Address
FreezeIndex uint64 // the index of the output that froze coins
SpendConditions consensus.SpendConditions
}
// HostSettings are the parameters advertised by the host. These are the
// values that the HostDB will request from the host in order to build its
// database.
type HostSettings struct {
TotalStorage int64 // Can go negative.
MinFilesize uint64
MaxFilesize uint64
MinDuration consensus.BlockHeight
MaxDuration consensus.BlockHeight
MinWindow consensus.BlockHeight
Price consensus.Currency
Collateral consensus.Currency
CoinAddress consensus.CoinAddress
}
// A HostEntry is an entry in the HostDB. It contains the HostSettings, as
// well as the IP address where the host can be found, and the value of the
// coins frozen in the host's announcement transaction.
type HostEntry struct {
ID string
IPAddress network.Address
MinFilesize uint64
MaxFilesize uint64