Commit 484631ed authored by Luke Champine's avatar Luke Champine Committed by GitHub

Merge pull request #1588 from NebulousLabs/scan-fix

Hostdb scanning bugfix
parents df9c8fad 251a2e7a
......@@ -10,7 +10,15 @@ import (
"testing"
"time"
"github.com/NebulousLabs/Sia/crypto"
"github.com/NebulousLabs/Sia/modules"
"github.com/NebulousLabs/Sia/modules/consensus"
"github.com/NebulousLabs/Sia/modules/gateway"
"github.com/NebulousLabs/Sia/modules/host"
"github.com/NebulousLabs/Sia/modules/miner"
"github.com/NebulousLabs/Sia/modules/renter"
"github.com/NebulousLabs/Sia/modules/transactionpool"
"github.com/NebulousLabs/Sia/modules/wallet"
)
// TestHostDBHostsActiveHandler checks the behavior of the call to
......@@ -227,6 +235,183 @@ func TestHostDBHostsHandler(t *testing.T) {
}
}
// assembleHostHostname is assembleServerTester but you can specify which
// hostname the host should use.
func assembleHostPort(key crypto.TwofishKey, hostHostname string, testdir string) (*serverTester, error) {
// assembleServerTester should not get called during short tests, as it
// takes a long time to run.
if testing.Short() {
panic("assembleServerTester called during short tests")
}
// Create the modules.
g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir))
if err != nil {
return nil, err
}
cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir))
if err != nil {
return nil, err
}
tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir))
if err != nil {
return nil, err
}
w, err := wallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir))
if err != nil {
return nil, err
}
if !w.Encrypted() {
_, err = w.Encrypt(key)
if err != nil {
return nil, err
}
}
err = w.Unlock(key)
if err != nil {
return nil, err
}
m, err := miner.New(cs, tp, w, filepath.Join(testdir, modules.MinerDir))
if err != nil {
return nil, err
}
h, err := host.New(cs, tp, w, hostHostname, filepath.Join(testdir, modules.HostDir))
if err != nil {
return nil, err
}
r, err := renter.New(g, cs, w, tp, filepath.Join(testdir, modules.RenterDir))
if err != nil {
return nil, err
}
srv, err := NewServer("localhost:0", "Sia-Agent", "", cs, nil, g, h, m, r, tp, w)
if err != nil {
return nil, err
}
// Assemble the serverTester.
st := &serverTester{
cs: cs,
gateway: g,
host: h,
miner: m,
renter: r,
tpool: tp,
wallet: w,
walletKey: key,
server: srv,
dir: testdir,
}
// TODO: A more reasonable way of listening for server errors.
go func() {
listenErr := srv.Serve()
if listenErr != nil {
panic(listenErr)
}
}()
return st, nil
}
// TestHostDBScanOnlineOffline checks that both online and offline hosts get
// scanned in the hostdb.
func TestHostDBScanOnlineOffline(t *testing.T) {
if testing.Short() {
t.SkipNow()
}
st, err := createServerTester("TestHostDBScanOnlineOffline")
if err != nil {
t.Fatal(err)
}
stHost, err := blankServerTester("TestHostDBScanOnlineAndOffline-Host")
if err != nil {
t.Fatal(err)
}
sts := []*serverTester{st, stHost}
err = fullyConnectNodes(sts)
if err != nil {
t.Fatal(err)
}
err = fundAllNodes(sts)
if err != nil {
t.Fatal(err)
}
// Announce the host.
err = stHost.acceptContracts()
if err != nil {
t.Fatal(err)
}
err = stHost.setHostStorage()
if err != nil {
t.Fatal(err)
}
err = stHost.announceHost()
if err != nil {
t.Fatal(err)
}
// Verify the host is visible.
var ah HostdbActiveGET
for i := 0; i < 50; i++ {
if err = st.getAPI("/hostdb/active", &ah); err != nil {
t.Fatal(err)
}
if len(ah.Hosts) == 1 {
break
}
time.Sleep(time.Millisecond * 100)
}
if len(ah.Hosts) != 1 {
t.Fatalf("expected 1 host, got %v", len(ah.Hosts))
}
hostAddr := ah.Hosts[0].NetAddress
// Close the host and wait for a scan to knock the host out of the hostdb.
err = stHost.server.Close()
if err != nil {
t.Fatal(err)
}
err = retry(60, time.Second, func() error {
if err := st.getAPI("/hostdb/active", &ah); err != nil {
return err
}
if len(ah.Hosts) == 0 {
return nil
}
return errors.New("host still in hostdb")
})
if err != nil {
t.Fatal(err)
}
// Reopen the host and wait for a scan to bring the host back into the
// hostdb.
stHost, err = assembleHostPort(stHost.walletKey, string(hostAddr), stHost.dir)
if err != nil {
t.Fatal(err)
}
sts[1] = stHost
err = fullyConnectNodes(sts)
if err != nil {
t.Fatal(err)
}
err = retry(60, time.Second, func() error {
// Get the hostdb internals.
if err = st.getAPI("/hostdb/active", &ah); err != nil {
return err
}
if len(ah.Hosts) != 1 {
return fmt.Errorf("expected 1 host, got %v", len(ah.Hosts))
}
return nil
})
if err != nil {
t.Fatal(err)
}
}
// TestHostDBAndRenterDownloadDynamicIPs checks that the hostdb and the renter are
// successfully able to follow a host that has changed IP addresses and then
// re-announced.
......
......@@ -7,26 +7,14 @@ import (
)
const (
// defaultScanSleep is the amount of time that the hostdb will sleep if it
// cannot successfully get a random number.
defaultScanSleep = 1*time.Hour + 37*time.Minute
// maxHostDowntime specifies the maximum amount of time that a host is
// allowed to be offline while still being in the hostdb.
maxHostDowntime = 30 * 24 * time.Hour
// maxScanSleep is the maximum amount of time that the hostdb will sleep
// between performing scans of the hosts.
maxScanSleep = 4 * time.Hour
// minScans specifies the number of scans that a host should have before the
// scans start getting compressed.
minScans = 20
// minScanSleep is the minimum amount of time that the hostdb will sleep
// between performing scans of the hosts.
minScanSleep = 1*time.Hour + 20*time.Minute
// maxSettingsLen indicates how long in bytes the host settings field is
// allowed to be before being ignored as a DoS attempt.
maxSettingsLen = 10e3
......@@ -60,3 +48,29 @@ var (
Testing: int(3),
}).(int)
)
var (
// defaultScanSleep is the amount of time that the hostdb will sleep if it
// cannot successfully get a random number.
defaultScanSleep = build.Select(build.Var{
Standard: time.Hour + time.Minute*37,
Dev: time.Minute * 5,
Testing: time.Second * 15,
}).(time.Duration)
// maxScanSleep is the maximum amount of time that the hostdb will sleep
// between performing scans of the hosts.
maxScanSleep = build.Select(build.Var{
Standard: time.Hour * 4,
Dev: time.Minute * 10,
Testing: time.Second * 15,
}).(time.Duration)
// minScanSleep is the minimum amount of time that the hostdb will sleep
// between performing scans of the hosts.
minScanSleep = build.Select(build.Var{
Standard: time.Hour + time.Minute*20,
Dev: time.Minute * 3,
Testing: time.Second * 14,
}).(time.Duration)
)
......@@ -5,8 +5,6 @@ package hostdb
// settings of the hosts.
import (
"crypto/rand"
"math/big"
"net"
"time"
......@@ -295,7 +293,7 @@ func (hdb *HostDB) threadedScan() {
if online && len(onlineHosts) < hostCheckupQuantity {
onlineHosts = append(onlineHosts, host)
} else if !online && len(offlineHosts) < hostCheckupQuantity {
offlineHosts = append(onlineHosts, host)
offlineHosts = append(offlineHosts, host)
}
}
......@@ -314,21 +312,20 @@ func (hdb *HostDB) threadedScan() {
// scanning. The minimums and maximums keep the scan time reasonable,
// while the randomness prevents the scanning from always happening at
// the same time of day or week.
maxBig := big.NewInt(int64(maxScanSleep))
minBig := big.NewInt(int64(minScanSleep))
randSleep, err := rand.Int(rand.Reader, maxBig.Sub(maxBig, minBig))
sleepTime := defaultScanSleep
sleepRange := int(maxScanSleep - minScanSleep)
sleepRand, err := crypto.RandIntn(sleepRange)
if err != nil {
build.Critical(err)
// If there's an error, sleep for the default amount of time.
defaultBig := big.NewInt(int64(defaultScanSleep))
randSleep = defaultBig.Sub(defaultBig, minBig)
} else {
sleepTime = minScanSleep + time.Duration(sleepRand)
}
// Sleep until it's time for the next scan cycle.
select {
case <-hdb.tg.StopChan():
return
case <-time.After(time.Duration(randSleep.Int64()) + minScanSleep):
case <-time.After(sleepTime):
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment