Commit cbc46d8a authored by Luke Champine's avatar Luke Champine Committed by GitHub

Merge pull request #1596 from NebulousLabs/scan-tuning

Scan tuning
parents 9693817b f6ac16e2
......@@ -286,7 +286,6 @@ func TestInitialBlockchainDownloadDoneRules(t *testing.T) {
if testing.Short() {
t.SkipNow()
}
t.Parallel()
testdir := build.TempDir(modules.ConsensusDir, "TestInitialBlockchainDownloadDoneRules")
// Create a gateway that can be forced to return errors when its RPC method
......@@ -420,7 +419,7 @@ func TestInitialBlockchainDownloadDoneRules(t *testing.T) {
}
select {
case <-doneChan:
case <-time.After(minIBDWaitTime + ibdLoopDelay):
case <-time.After(2 * (minIBDWaitTime + ibdLoopDelay)):
t.Fatal("threadedInitialBlockchainDownload never finished with 2 synced peers and 1 non-synced peer")
}
......
......@@ -9,11 +9,11 @@ import (
const (
// maxHostDowntime specifies the maximum amount of time that a host is
// allowed to be offline while still being in the hostdb.
maxHostDowntime = 30 * 24 * time.Hour
maxHostDowntime = 10 * 24 * time.Hour
// minScans specifies the number of scans that a host should have before the
// scans start getting compressed.
minScans = 20
minScans = 12
// maxSettingsLen indicates how long in bytes the host settings field is
// allowed to be before being ignored as a DoS attempt.
......@@ -43,7 +43,7 @@ var (
// scanningThreads is the number of threads that will be probing hosts for
// their settings and checking for reliability.
scanningThreads = build.Select(build.Var{
Standard: int(40),
Standard: int(20),
Dev: int(4),
Testing: int(3),
}).(int)
......@@ -61,7 +61,7 @@ var (
// maxScanSleep is the maximum amount of time that the hostdb will sleep
// between performing scans of the hosts.
maxScanSleep = build.Select(build.Var{
Standard: time.Hour * 4,
Standard: time.Hour * 8,
Dev: time.Minute * 10,
Testing: time.Second * 15,
}).(time.Duration)
......
......@@ -107,7 +107,9 @@ func newHostDB(g modules.Gateway, cs modules.ConsensusSet, persistDir string, de
hdb.hostTree = hosttree.New(hdb.calculateHostWeight)
// Load the prior persistence structures.
hdb.mu.Lock()
err = hdb.load()
hdb.mu.Unlock()
if err != nil && !os.IsNotExist(err) {
return nil, err
}
......@@ -134,16 +136,20 @@ func newHostDB(g modules.Gateway, cs modules.ConsensusSet, persistDir string, de
// If the block height has loaded as zero, the most recent consensus change
// needs to be set to perform a full rescan. This will also help the hostdb
// to pick up any hosts that it has incorrectly dropped in the past.
hdb.mu.Lock()
if hdb.blockHeight == 0 {
hdb.lastChange = modules.ConsensusChangeBeginning
}
hdb.mu.Unlock()
err = cs.ConsensusSetSubscribe(hdb, hdb.lastChange)
if err == modules.ErrInvalidConsensusChangeID {
// Subscribe again using the new ID. This will cause a triggered scan
// on all of the hosts, but that should be acceptable.
hdb.mu.Lock()
hdb.blockHeight = 0
hdb.lastChange = modules.ConsensusChangeBeginning
hdb.mu.Unlock()
err = cs.ConsensusSetSubscribe(hdb, hdb.lastChange)
}
if err != nil {
......@@ -156,7 +162,9 @@ func newHostDB(g modules.Gateway, cs modules.ConsensusSet, persistDir string, de
} else {
// During testing, the hostdb is just always assumed to be online, since
// the online check of having nonlocal peers will always fail.
hdb.mu.Lock()
hdb.online = true
hdb.mu.Unlock()
}
for i := 0; i < scanningThreads; i++ {
go hdb.threadedProbeHosts()
......
......@@ -47,18 +47,11 @@ var (
// requiredStorage indicates the amount of storage that the host must be
// offering in order to be considered a valuable/worthwhile host.
requiredStorage = func() uint64 {
switch build.Release {
case "dev":
return 1e6
case "standard":
return 5e9
case "testing":
return 1e3
default:
panic("incorrect/missing value for requiredStorage constant")
}
}()
requiredStorage = build.Select(build.Var{
Standard: uint64(20e9),
Dev: uint64(1e6),
Testing: uint64(1e3),
}).(uint64)
)
// collateralAdjustments improves the host's weight according to the amount of
......@@ -162,20 +155,29 @@ func storageRemainingAdjustments(entry modules.HostDBEntry) float64 {
if entry.RemainingStorage < 200*requiredStorage {
base = base / 2 // 2x total penalty
}
if entry.RemainingStorage < 50*requiredStorage {
base = base / 3 // 6x total penalty
if entry.RemainingStorage < 150*requiredStorage {
base = base / 2 // 4x total penalty
}
if entry.RemainingStorage < 100*requiredStorage {
base = base / 3 // 12x total penalty
}
if entry.RemainingStorage < 80*requiredStorage {
base = base / 3 // 36x total penalty
}
if entry.RemainingStorage < 40*requiredStorage {
base = base / 4 // 144x total penalty
}
if entry.RemainingStorage < 25*requiredStorage {
base = base / 4 // 24x total penalty
if entry.RemainingStorage < 20*requiredStorage {
base = base / 5 // 720x total penalty
}
if entry.RemainingStorage < 10*requiredStorage {
base = base / 5 // 96x total penalty
base = base / 5 // 3,600x total penalty
}
if entry.RemainingStorage < 5*requiredStorage {
base = base / 10 // 960x total penalty
base = base / 5 // 14,400x total penalty
}
if entry.RemainingStorage < requiredStorage {
base = base / 10 // 9600x total penalty
base = base / 5 // 72,000x total penalty
}
return base
}
......
......@@ -66,6 +66,11 @@ func (hdb *HostDB) load() error {
if err != nil {
hdb.log.Debugln("ERROR: could not insert host while loading:", host.NetAddress)
}
// Make sure that all hosts have gone through the initial scanning.
if len(host.ScanHistory) < 2 {
hdb.queueScan(host)
}
}
hdb.blockHeight = data.BlockHeight
hdb.lastChange = data.LastChange
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment