renter.go 20.8 KB
Newer Older
David Vorick's avatar
David Vorick committed
1
package modules
2

3
import (
4
	"encoding/json"
5
	"fmt"
6
	"io"
7 8
	"time"

9
	"gitlab.com/NebulousLabs/Sia/build"
10 11
	"gitlab.com/NebulousLabs/Sia/crypto"
	"gitlab.com/NebulousLabs/Sia/types"
12

13
	"gitlab.com/NebulousLabs/errors"
14 15
)

16 17 18 19 20 21
var (
	// DefaultAllowance is the set of default allowance settings that will be
	// used when allowances are not set or not fully set
	DefaultAllowance = Allowance{
		Funds:       types.SiacoinPrecision.Mul64(500),
		Hosts:       uint64(PriceEstimationScope),
22 23
		Period:      types.BlockHeight(3 * types.BlocksPerMonth),
		RenewWindow: types.BlockHeight(types.BlocksPerMonth),
24

25 26 27 28
		ExpectedStorage:    1e12,                                 // 1 TB
		ExpectedUpload:     uint64(200e9) / types.BlocksPerMonth, // 200 GB per month
		ExpectedDownload:   uint64(100e9) / types.BlocksPerMonth, // 100 GB per month
		ExpectedRedundancy: 3.0,                                  // default is 10/30 erasure coding
29 30
	}
	// ErrHostFault indicates if an error is the host's fault.
31 32 33 34 35 36 37 38 39 40 41 42
	ErrHostFault = errors.New("host has returned an error")

	// PriceEstimationScope is the number of hosts that get queried by the
	// renter when providing price estimates. Especially for the 'Standard'
	// variable, there should be congruence with the number of contracts being
	// used in the renter allowance.
	PriceEstimationScope = build.Select(build.Var{
		Standard: int(50),
		Dev:      int(12),
		Testing:  int(4),
	}).(int)
)
43

44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89
// FilterMode is the helper type for the enum constants for the HostDB filter
// mode
type FilterMode int

// HostDBFilterError HostDBDisableFilter HostDBActivateBlacklist and
// HostDBActiveWhitelist are the constants used to enable and disable the filter
// mode of the renter's hostdb
const (
	HostDBFilterError FilterMode = iota
	HostDBDisableFilter
	HostDBActivateBlacklist
	HostDBActiveWhitelist
)

// String returns the string value for the FilterMode
func (fm FilterMode) String() string {
	switch fm {
	case HostDBFilterError:
		return "error"
	case HostDBDisableFilter:
		return "disable"
	case HostDBActivateBlacklist:
		return "blacklist"
	case HostDBActiveWhitelist:
		return "whitelist"
	default:
		return ""
	}
}

// FromString assigned the FilterMode from the provide string
func (fm *FilterMode) FromString(s string) error {
	switch s {
	case "disable":
		*fm = HostDBDisableFilter
	case "blacklist":
		*fm = HostDBActivateBlacklist
	case "whitelist":
		*fm = HostDBActiveWhitelist
	default:
		*fm = HostDBFilterError
		return fmt.Errorf("Could not assigned FilterMode from string %v", s)
	}
	return nil
}

90 91 92 93 94
// IsHostsFault indicates if a returned error is the host's fault.
func IsHostsFault(err error) bool {
	return errors.Contains(err, ErrHostFault)
}

95
const (
96 97
	// RenterDir is the name of the directory that is used to store the
	// renter's persistent data.
98
	RenterDir = "renter"
99 100 101 102 103 104 105

	// EstimatedFileContractTransactionSetSize is the estimated blockchain size
	// of a transaction set between a renter and a host that contains a file
	// contract. This transaction set will contain a setup transaction from each
	// the host and the renter, and will also contain a file contract and file
	// contract revision that have each been signed by all parties.
	EstimatedFileContractTransactionSetSize = 2048
106 107

	// EstimatedFileContractRevisionAndProofTransactionSetSize is the
108
	// estimated blockchain size of a transaction set used by the host to
109 110
	// provide the storage proof at the end of the contract duration.
	EstimatedFileContractRevisionAndProofTransactionSetSize = 5000
111 112
)

113 114
// An ErasureCoder is an error-correcting encoder and decoder.
type ErasureCoder interface {
Luke Champine's avatar
Luke Champine committed
115
	// NumPieces is the number of pieces returned by Encode.
116 117
	NumPieces() int

118 119 120 121
	// MinPieces is the minimum number of pieces that must be present to
	// recover the original data.
	MinPieces() int

Luke Champine's avatar
Luke Champine committed
122 123 124
	// Encode splits data into equal-length pieces, with some pieces
	// containing parity data.
	Encode(data []byte) ([][]byte, error)
125

126 127 128 129
	// EncodeShards encodes the input data like Encode but accepts an already
	// sharded input.
	EncodeShards(data [][]byte) ([][]byte, error)

130 131 132 133 134
	// Recover recovers the original data from pieces and writes it to w.
	// pieces should be identical to the slice returned by Encode (length and
	// order must be preserved), but with missing elements set to nil. n is
	// the number of bytes to be written to w; this is necessary because
	// pieces may have been padded with zeros during encoding.
135
	Recover(pieces [][]byte, n uint64, w io.Writer) error
136 137
}

138 139 140 141 142 143 144
// An Allowance dictates how much the Renter is allowed to spend in a given
// period. Note that funds are spent on both storage and bandwidth.
type Allowance struct {
	Funds       types.Currency    `json:"funds"`
	Hosts       uint64            `json:"hosts"`
	Period      types.BlockHeight `json:"period"`
	RenewWindow types.BlockHeight `json:"renewwindow"`
145 146 147 148

	// ExpectedStorage is the amount of data that we expect to have in a contract.
	ExpectedStorage uint64 `json:"expectedstorage"`

Christopher Schinnerl's avatar
Christopher Schinnerl committed
149 150 151 152 153 154 155
	// ExpectedUpload is the expected amount of data uploaded through the API,
	// before redundancy, per block.
	ExpectedUpload uint64 `json:"expectedupload"`

	// ExpectedDownload is the expected amount of data downloaded through the
	// API per block.
	ExpectedDownload uint64 `json:"expecteddownload"`
156 157 158

	// ExpectedRedundancy is the average redundancy of files being uploaded.
	ExpectedRedundancy float64 `json:"expectedredundancy"`
159 160
}

161 162 163 164 165
// ContractUtility contains metrics internal to the contractor that reflect the
// utility of a given contract.
type ContractUtility struct {
	GoodForUpload bool
	GoodForRenew  bool
166
	Locked        bool // Locked utilities can only be set to false.
167 168
}

169 170 171
// DownloadInfo provides information about a file that has been requested for
// download.
type DownloadInfo struct {
172 173 174 175 176 177
	Destination     string `json:"destination"`     // The destination of the download.
	DestinationType string `json:"destinationtype"` // Can be "file", "memory buffer", or "http stream".
	Length          uint64 `json:"length"`          // The length requested for the download.
	Offset          uint64 `json:"offset"`          // The offset within the siafile requested for the download.
	SiaPath         string `json:"siapath"`         // The siapath of the file used for the download.

178 179 180 181
	Completed            bool      `json:"completed"`            // Whether or not the download has completed.
	EndTime              time.Time `json:"endtime"`              // The time when the download fully completed.
	Error                string    `json:"error"`                // Will be the empty string unless there was an error.
	Received             uint64    `json:"received"`             // Amount of data confirmed and decoded.
Matthew Sevey's avatar
Matthew Sevey committed
182 183
	StartTime            time.Time `json:"starttime"`            // The time when the download was started.
	StartTimeUnix        int64     `json:"starttimeunix"`        // The time when the download was started in unix format.
184
	TotalDataTransferred uint64    `json:"totaldatatransferred"` // Total amount of data transferred, including negotiation, etc.
Robin Nabel's avatar
Robin Nabel committed
185 186
}

187 188 189
// FileUploadParams contains the information used by the Renter to upload a
// file.
type FileUploadParams struct {
190 191
	Source      string
	SiaPath     string
192
	ErasureCode ErasureCoder
193
	Force       bool
David Vorick's avatar
David Vorick committed
194 195
}

196 197
// FileInfo provides information about a file.
type FileInfo struct {
198
	SiaPath        string            `json:"siapath"`
Ava Howell's avatar
Ava Howell committed
199
	LocalPath      string            `json:"localpath"`
200 201
	Filesize       uint64            `json:"filesize"`
	Available      bool              `json:"available"`
202
	Renewing       bool              `json:"renewing"`
203
	Redundancy     float64           `json:"redundancy"`
204
	UploadedBytes  uint64            `json:"uploadedbytes"`
205
	UploadProgress float64           `json:"uploadprogress"`
206
	Expiration     types.BlockHeight `json:"expiration"`
207 208
	OnDisk         bool              `json:"ondisk"`
	Recoverable    bool              `json:"recoverable"`
209 210
}

211
// A HostDBEntry represents one host entry in the Renter's host DB. It
212
// aggregates the host's external settings and metrics with its public key.
213 214
type HostDBEntry struct {
	HostExternalSettings
215

Ava Howell's avatar
Ava Howell committed
216
	// FirstSeen is the last block height at which this host was announced.
David Vorick's avatar
David Vorick committed
217
	FirstSeen types.BlockHeight `json:"firstseen"`
218 219 220 221 222

	// Measurements that have been taken on the host. The most recent
	// measurements are kept in full detail, historic ones are compressed into
	// the historic values.
	HistoricDowntime time.Duration `json:"historicdowntime"`
223 224
	HistoricUptime   time.Duration `json:"historicuptime"`
	ScanHistory      HostDBScans   `json:"scanhistory"`
225

226
	// Measurements that are taken whenever we interact with a host.
227 228 229 230
	HistoricFailedInteractions     float64 `json:"historicfailedinteractions"`
	HistoricSuccessfulInteractions float64 `json:"historicsuccessfulinteractions"`
	RecentFailedInteractions       float64 `json:"recentfailedinteractions"`
	RecentSuccessfulInteractions   float64 `json:"recentsuccessfulinteractions"`
231

232
	LastHistoricUpdate types.BlockHeight `json:"lasthistoricupdate"`
233

234
	// Measurements related to the IP subnet mask.
235 236
	IPNets          []string  `json:"ipnets"`
	LastIPNetChange time.Time `json:"lastipnetchange"`
237

238 239 240
	// The public key of the host, stored separately to minimize risk of certain
	// MitM based vulnerabilities.
	PublicKey types.SiaPublicKey `json:"publickey"`
241

242 243 244
	// Filtered says whether or not a HostDBEntry is being filtered out of the
	// filtered hosttree due to the filter mode of the hosttree
	Filtered bool `json:"filtered"`
245 246 247 248
}

// HostDBScan represents a single scan event.
type HostDBScan struct {
David Vorick's avatar
David Vorick committed
249 250 251 252 253 254 255 256 257 258 259 260
	Timestamp time.Time `json:"timestamp"`
	Success   bool      `json:"success"`
}

// HostScoreBreakdown provides a piece-by-piece explanation of why a host has
// the score that they do.
//
// NOTE: Renters are free to use whatever scoring they feel appropriate for
// hosts. Some renters will outright blacklist or whitelist sets of hosts. The
// results provided by this struct can only be used as a guide, and may vary
// significantly from machine to machine.
type HostScoreBreakdown struct {
261 262
	Score          types.Currency `json:"score"`
	ConversionRate float64        `json:"conversionrate"`
263

David Vorick's avatar
David Vorick committed
264 265 266
	AgeAdjustment              float64 `json:"ageadjustment"`
	BurnAdjustment             float64 `json:"burnadjustment"`
	CollateralAdjustment       float64 `json:"collateraladjustment"`
267
	InteractionAdjustment      float64 `json:"interactionadjustment"`
David Vorick's avatar
David Vorick committed
268 269 270 271
	PriceAdjustment            float64 `json:"pricesmultiplier"`
	StorageRemainingAdjustment float64 `json:"storageremainingadjustment"`
	UptimeAdjustment           float64 `json:"uptimeadjustment"`
	VersionAdjustment          float64 `json:"versionadjustment"`
272 273
}

274
// RenterPriceEstimation contains a bunch of files estimating the costs of
275 276 277
// various operations on the network.
type RenterPriceEstimation struct {
	// The cost of downloading 1 TB of data.
278 279 280 281
	DownloadTerabyte types.Currency `json:"downloadterabyte"`

	// The cost of forming a set of contracts using the defaults.
	FormContracts types.Currency `json:"formcontracts"`
282 283

	// The cost of storing 1 TB for a month, including redundancy.
284
	StorageTerabyteMonth types.Currency `json:"storageterabytemonth"`
285 286 287

	// The cost of consuming 1 TB of upload bandwidth from the host, including
	// redundancy.
288
	UploadTerabyte types.Currency `json:"uploadterabyte"`
289 290 291 292
}

// RenterSettings control the behavior of the Renter.
type RenterSettings struct {
293 294 295 296 297
	Allowance         Allowance `json:"allowance"`
	IPViolationsCheck bool      `json:"ipviolationcheck"`
	MaxUploadSpeed    int64     `json:"maxuploadspeed"`
	MaxDownloadSpeed  int64     `json:"maxdownloadspeed"`
	StreamCacheSize   uint64    `json:"streamcachesize"`
298 299
}

Luke Champine's avatar
Luke Champine committed
300 301 302 303 304 305 306
// HostDBScans represents a sortable slice of scans.
type HostDBScans []HostDBScan

func (s HostDBScans) Len() int           { return len(s) }
func (s HostDBScans) Less(i, j int) bool { return s[i].Timestamp.Before(s[j].Timestamp) }
func (s HostDBScans) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344
// MerkleRootSet is a set of Merkle roots, and gets encoded more efficiently.
type MerkleRootSet []crypto.Hash

// MarshalJSON defines a JSON encoding for a MerkleRootSet.
func (mrs MerkleRootSet) MarshalJSON() ([]byte, error) {
	// Copy the whole array into a giant byte slice and then encode that.
	fullBytes := make([]byte, crypto.HashSize*len(mrs))
	for i := range mrs {
		copy(fullBytes[i*crypto.HashSize:(i+1)*crypto.HashSize], mrs[i][:])
	}
	return json.Marshal(fullBytes)
}

// UnmarshalJSON attempts to decode a MerkleRootSet, falling back on the legacy
// decoding of a []crypto.Hash if that fails.
func (mrs *MerkleRootSet) UnmarshalJSON(b []byte) error {
	// Decode the giant byte slice, and then split it into separate arrays.
	var fullBytes []byte
	err := json.Unmarshal(b, &fullBytes)
	if err != nil {
		// Encoding the byte slice has failed, try decoding it as a []crypto.Hash.
		var hashes []crypto.Hash
		err := json.Unmarshal(b, &hashes)
		if err != nil {
			return err
		}
		*mrs = MerkleRootSet(hashes)
		return nil
	}

	umrs := make(MerkleRootSet, len(fullBytes)/32)
	for i := range umrs {
		copy(umrs[i][:], fullBytes[i*crypto.HashSize:(i+1)*crypto.HashSize])
	}
	*mrs = umrs
	return nil
}

345 346
// A RenterContract contains metadata about a file contract. It is read-only;
// modifying a RenterContract does not modify the actual file contract.
347
type RenterContract struct {
348 349
	ID            types.FileContractID
	HostPublicKey types.SiaPublicKey
350
	Transaction   types.Transaction
351

352 353 354 355 356 357 358 359 360 361 362 363
	StartHeight types.BlockHeight
	EndHeight   types.BlockHeight

	// RenterFunds is the amount remaining in the contract that the renter can
	// spend.
	RenterFunds types.Currency

	// The FileContract does not indicate what funds were spent on, so we have
	// to track the various costs manually.
	DownloadSpending types.Currency
	StorageSpending  types.Currency
	UploadSpending   types.Currency
364

365 366 367
	// Utility contains utility information about the renter.
	Utility ContractUtility

368 369 370 371
	// TotalCost indicates the amount of money that the renter spent and/or
	// locked up while forming a contract. This includes fees, and includes
	// funds which were allocated (but not necessarily committed) to spend on
	// uploads/downloads/storage.
372 373
	TotalCost types.Currency

374 375 376 377 378 379 380 381 382 383 384
	// ContractFee is the amount of money paid to the host to cover potential
	// future transaction fees that the host may incur, and to cover any other
	// overheads the host may have.
	//
	// TxnFee is the amount of money spent on the transaction fee when putting
	// the renter contract on the blockchain.
	//
	// SiafundFee is the amount of money spent on siafund fees when creating the
	// contract. The siafund fee that the renter pays covers both the renter and
	// the host portions of the contract, and therefore can be unexpectedly high
	// if the the host collateral is high.
385 386 387
	ContractFee types.Currency
	TxnFee      types.Currency
	SiafundFee  types.Currency
388 389
}

390 391 392
// ContractorSpending contains the metrics about how much the Contractor has
// spent during the current billing period.
type ContractorSpending struct {
393 394 395 396
	// ContractFees are the sum of all fees in the contract. This means it
	// includes the ContractFee, TxnFee and SiafundFee
	ContractFees types.Currency `json:"contractfees"`
	// DownloadSpending is the money currently spent on downloads.
397
	DownloadSpending types.Currency `json:"downloadspending"`
398 399 400
	// StorageSpending is the money currently spent on storage.
	StorageSpending types.Currency `json:"storagespending"`
	// ContractSpending is the total amount of money that the renter has put
Christopher Schinnerl's avatar
Christopher Schinnerl committed
401
	// into contracts, whether it's locked and the renter gets that money
402 403 404 405 406 407
	// back or whether it's spent and the renter won't get the money back.
	TotalAllocated types.Currency `json:"totalallocated"`
	// UploadSpending is the money currently spent on uploads.
	UploadSpending types.Currency `json:"uploadspending"`
	// Unspent is locked-away, unspent money.
	Unspent types.Currency `json:"unspent"`
408
	// ContractSpendingDeprecated was renamed to TotalAllocated and always has the
409
	// same value as TotalAllocated.
410
	ContractSpendingDeprecated types.Currency `json:"contractspending"`
411 412 413 414 415 416 417 418 419 420
	// WithheldFunds are the funds from the previous period that are tied up
	// in contracts and have not been released yet
	WithheldFunds types.Currency `json:"withheldfunds"`
	// ReleaseBlock is the block at which the WithheldFunds should be
	// released to the renter, based on worst case.
	// Contract End Height + Host Window Size + Maturity Delay
	ReleaseBlock types.BlockHeight `json:"releaseblock"`
	// PreviousSpending is the total spend funds from old contracts
	// that are not included in the current period spending
	PreviousSpending types.Currency `json:"previousspending"`
421 422
}

423 424
// A Renter uploads, tracks, repairs, and downloads a set of files for the
// user.
425
type Renter interface {
426 427
	// ActiveHosts provides the list of hosts that the renter is selecting,
	// sorted by preference.
428
	ActiveHosts() []HostDBEntry
429 430

	// AllHosts returns the full list of hosts known to the renter.
431
	AllHosts() []HostDBEntry
432

433 434 435
	// Close closes the Renter.
	Close() error

436 437 438
	// CancelContract cancels a specific contract of the renter.
	CancelContract(id types.FileContractID) error

439
	// Contracts returns the staticContracts of the renter's hostContractor.
440 441
	Contracts() []RenterContract

442
	// OldContracts returns the oldContracts of the renter's hostContractor.
443 444
	OldContracts() []RenterContract

445 446
	// ContractUtility provides the contract utility for a given host key.
	ContractUtility(pk types.SiaPublicKey) (ContractUtility, bool)
447

448 449 450 451
	// CurrentPeriod returns the height at which the current allowance period
	// began.
	CurrentPeriod() types.BlockHeight

452 453 454 455
	// PeriodSpending returns the amount spent on contracts in the current
	// billing period.
	PeriodSpending() ContractorSpending

456
	// DeleteFile deletes a file entry from the renter.
457
	DeleteFile(path string) error
458

459 460 461
	// Download performs a download according to the parameters passed, including
	// downloads of `offset` and `length` type.
	Download(params RenterDownloadParameters) error
462

463 464 465 466
	// Download performs a download according to the parameters passed without
	// blocking, including downloads of `offset` and `length` type.
	DownloadAsync(params RenterDownloadParameters) error

467
	// ClearDownloadHistory clears the download history of the renter
468
	// inclusive for before and after times.
469
	ClearDownloadHistory(after, before time.Time) error
470

471 472
	// DownloadHistory lists all the files that have been scheduled for download.
	DownloadHistory() []DownloadInfo
473

474 475 476
	// File returns information on specific file queried by user
	File(siaPath string) (FileInfo, error)

477 478 479
	// FileList returns information on all of the files stored by the renter.
	FileList() []FileInfo

480 481
	// SetFilterMode sets the renter's hostdb filter mode
	SetFilterMode(fm FilterMode, hosts []types.SiaPublicKey) error
482

David Vorick's avatar
David Vorick committed
483
	// Host provides the DB entry and score breakdown for the requested host.
484
	Host(pk types.SiaPublicKey) (HostDBEntry, bool)
David Vorick's avatar
David Vorick committed
485

486 487 488 489
	// InitialScanComplete returns a boolean indicating if the initial scan of the
	// hostdb is completed.
	InitialScanComplete() (bool, error)

490
	// LoadSharedFiles loads a '.sia' file into the renter. A .sia file may
491 492
	// contain multiple files. The paths of the added files are returned.
	LoadSharedFiles(source string) ([]string, error)
493

494
	// LoadSharedFilesASCII loads an ASCII-encoded '.sia' file into the
495
	// renter.
496
	LoadSharedFilesASCII(asciiSia string) ([]string, error)
497

498
	// PriceEstimation estimates the cost in siacoins of performing various
499
	// storage and data operations.
500
	PriceEstimation(allowance Allowance) (RenterPriceEstimation, Allowance, error)
501

502
	// RenameFile changes the path of a file.
503
	RenameFile(path, newPath string) error
504

505 506
	// EstimateHostScore will return the score for a host with the provided
	// settings, assuming perfect age and uptime adjustments
507
	EstimateHostScore(entry HostDBEntry, allowance Allowance) HostScoreBreakdown
508

David Vorick's avatar
David Vorick committed
509 510 511 512
	// ScoreBreakdown will return the score for a host db entry using the
	// hostdb's weighting algorithm.
	ScoreBreakdown(entry HostDBEntry) HostScoreBreakdown

513 514 515 516 517
	// Settings returns the Renter's current settings.
	Settings() RenterSettings

	// SetSettings sets the Renter's settings.
	SetSettings(RenterSettings) error
518

519 520 521 522
	// SetFileTrackingPath sets the on-disk location of an uploaded file to a
	// new value. Useful if files need to be moved on disk.
	SetFileTrackingPath(siaPath, newPath string) error

523
	// ShareFiles creates a '.sia' file that can be shared with others.
524
	ShareFiles(paths []string, shareDest string) error
Luke Champine's avatar
Luke Champine committed
525

526
	// ShareFilesAscii creates an ASCII-encoded '.sia' file.
527
	ShareFilesASCII(paths []string) (asciiSia string, err error)
528

529
	// Streamer creates a io.ReadSeeker that can be used to stream downloads
530 531
	// from the Sia network and also returns the fileName of the streamed
	// resource.
532 533
	Streamer(siaPath string) (string, io.ReadSeeker, error)

534
	// Upload uploads a file using the input parameters.
535
	Upload(FileUploadParams) error
536
}
Robin Nabel's avatar
Robin Nabel committed
537

538 539
// RenterDownloadParameters defines the parameters passed to the Renter's
// Download method.
Robin Nabel's avatar
Robin Nabel committed
540
type RenterDownloadParameters struct {
541
	Async       bool
542
	Httpwriter  io.Writer
543 544
	Length      uint64
	Offset      uint64
545
	SiaPath     string
546
	Destination string
Robin Nabel's avatar
Robin Nabel committed
547
}