Commit 484631ed authored by Luke Champine's avatar Luke Champine Committed by GitHub

Merge pull request #1588 from NebulousLabs/scan-fix

Hostdb scanning bugfix
parents df9c8fad 251a2e7a
...@@ -10,7 +10,15 @@ import ( ...@@ -10,7 +10,15 @@ import (
"testing" "testing"
"time" "time"
"github.com/NebulousLabs/Sia/crypto"
"github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules"
"github.com/NebulousLabs/Sia/modules/consensus"
"github.com/NebulousLabs/Sia/modules/gateway"
"github.com/NebulousLabs/Sia/modules/host"
"github.com/NebulousLabs/Sia/modules/miner"
"github.com/NebulousLabs/Sia/modules/renter"
"github.com/NebulousLabs/Sia/modules/transactionpool"
"github.com/NebulousLabs/Sia/modules/wallet"
) )
// TestHostDBHostsActiveHandler checks the behavior of the call to // TestHostDBHostsActiveHandler checks the behavior of the call to
...@@ -227,6 +235,183 @@ func TestHostDBHostsHandler(t *testing.T) { ...@@ -227,6 +235,183 @@ func TestHostDBHostsHandler(t *testing.T) {
} }
} }
// assembleHostHostname is assembleServerTester but you can specify which
// hostname the host should use.
func assembleHostPort(key crypto.TwofishKey, hostHostname string, testdir string) (*serverTester, error) {
// assembleServerTester should not get called during short tests, as it
// takes a long time to run.
if testing.Short() {
panic("assembleServerTester called during short tests")
}
// Create the modules.
g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir))
if err != nil {
return nil, err
}
cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir))
if err != nil {
return nil, err
}
tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir))
if err != nil {
return nil, err
}
w, err := wallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir))
if err != nil {
return nil, err
}
if !w.Encrypted() {
_, err = w.Encrypt(key)
if err != nil {
return nil, err
}
}
err = w.Unlock(key)
if err != nil {
return nil, err
}
m, err := miner.New(cs, tp, w, filepath.Join(testdir, modules.MinerDir))
if err != nil {
return nil, err
}
h, err := host.New(cs, tp, w, hostHostname, filepath.Join(testdir, modules.HostDir))
if err != nil {
return nil, err
}
r, err := renter.New(g, cs, w, tp, filepath.Join(testdir, modules.RenterDir))
if err != nil {
return nil, err
}
srv, err := NewServer("localhost:0", "Sia-Agent", "", cs, nil, g, h, m, r, tp, w)
if err != nil {
return nil, err
}
// Assemble the serverTester.
st := &serverTester{
cs: cs,
gateway: g,
host: h,
miner: m,
renter: r,
tpool: tp,
wallet: w,
walletKey: key,
server: srv,
dir: testdir,
}
// TODO: A more reasonable way of listening for server errors.
go func() {
listenErr := srv.Serve()
if listenErr != nil {
panic(listenErr)
}
}()
return st, nil
}
// TestHostDBScanOnlineOffline checks that both online and offline hosts get
// scanned in the hostdb.
func TestHostDBScanOnlineOffline(t *testing.T) {
if testing.Short() {
t.SkipNow()
}
st, err := createServerTester("TestHostDBScanOnlineOffline")
if err != nil {
t.Fatal(err)
}
stHost, err := blankServerTester("TestHostDBScanOnlineAndOffline-Host")
if err != nil {
t.Fatal(err)
}
sts := []*serverTester{st, stHost}
err = fullyConnectNodes(sts)
if err != nil {
t.Fatal(err)
}
err = fundAllNodes(sts)
if err != nil {
t.Fatal(err)
}
// Announce the host.
err = stHost.acceptContracts()
if err != nil {
t.Fatal(err)
}
err = stHost.setHostStorage()
if err != nil {
t.Fatal(err)
}
err = stHost.announceHost()
if err != nil {
t.Fatal(err)
}
// Verify the host is visible.
var ah HostdbActiveGET
for i := 0; i < 50; i++ {
if err = st.getAPI("/hostdb/active", &ah); err != nil {
t.Fatal(err)
}
if len(ah.Hosts) == 1 {
break
}
time.Sleep(time.Millisecond * 100)
}
if len(ah.Hosts) != 1 {
t.Fatalf("expected 1 host, got %v", len(ah.Hosts))
}
hostAddr := ah.Hosts[0].NetAddress
// Close the host and wait for a scan to knock the host out of the hostdb.
err = stHost.server.Close()
if err != nil {
t.Fatal(err)
}
err = retry(60, time.Second, func() error {
if err := st.getAPI("/hostdb/active", &ah); err != nil {
return err
}
if len(ah.Hosts) == 0 {
return nil
}
return errors.New("host still in hostdb")
})
if err != nil {
t.Fatal(err)
}
// Reopen the host and wait for a scan to bring the host back into the
// hostdb.
stHost, err = assembleHostPort(stHost.walletKey, string(hostAddr), stHost.dir)
if err != nil {
t.Fatal(err)
}
sts[1] = stHost
err = fullyConnectNodes(sts)
if err != nil {
t.Fatal(err)
}
err = retry(60, time.Second, func() error {
// Get the hostdb internals.
if err = st.getAPI("/hostdb/active", &ah); err != nil {
return err
}
if len(ah.Hosts) != 1 {
return fmt.Errorf("expected 1 host, got %v", len(ah.Hosts))
}
return nil
})
if err != nil {
t.Fatal(err)
}
}
// TestHostDBAndRenterDownloadDynamicIPs checks that the hostdb and the renter are // TestHostDBAndRenterDownloadDynamicIPs checks that the hostdb and the renter are
// successfully able to follow a host that has changed IP addresses and then // successfully able to follow a host that has changed IP addresses and then
// re-announced. // re-announced.
......
...@@ -7,26 +7,14 @@ import ( ...@@ -7,26 +7,14 @@ import (
) )
const ( const (
// defaultScanSleep is the amount of time that the hostdb will sleep if it
// cannot successfully get a random number.
defaultScanSleep = 1*time.Hour + 37*time.Minute
// maxHostDowntime specifies the maximum amount of time that a host is // maxHostDowntime specifies the maximum amount of time that a host is
// allowed to be offline while still being in the hostdb. // allowed to be offline while still being in the hostdb.
maxHostDowntime = 30 * 24 * time.Hour maxHostDowntime = 30 * 24 * time.Hour
// maxScanSleep is the maximum amount of time that the hostdb will sleep
// between performing scans of the hosts.
maxScanSleep = 4 * time.Hour
// minScans specifies the number of scans that a host should have before the // minScans specifies the number of scans that a host should have before the
// scans start getting compressed. // scans start getting compressed.
minScans = 20 minScans = 20
// minScanSleep is the minimum amount of time that the hostdb will sleep
// between performing scans of the hosts.
minScanSleep = 1*time.Hour + 20*time.Minute
// maxSettingsLen indicates how long in bytes the host settings field is // maxSettingsLen indicates how long in bytes the host settings field is
// allowed to be before being ignored as a DoS attempt. // allowed to be before being ignored as a DoS attempt.
maxSettingsLen = 10e3 maxSettingsLen = 10e3
...@@ -60,3 +48,29 @@ var ( ...@@ -60,3 +48,29 @@ var (
Testing: int(3), Testing: int(3),
}).(int) }).(int)
) )
var (
// defaultScanSleep is the amount of time that the hostdb will sleep if it
// cannot successfully get a random number.
defaultScanSleep = build.Select(build.Var{
Standard: time.Hour + time.Minute*37,
Dev: time.Minute * 5,
Testing: time.Second * 15,
}).(time.Duration)
// maxScanSleep is the maximum amount of time that the hostdb will sleep
// between performing scans of the hosts.
maxScanSleep = build.Select(build.Var{
Standard: time.Hour * 4,
Dev: time.Minute * 10,
Testing: time.Second * 15,
}).(time.Duration)
// minScanSleep is the minimum amount of time that the hostdb will sleep
// between performing scans of the hosts.
minScanSleep = build.Select(build.Var{
Standard: time.Hour + time.Minute*20,
Dev: time.Minute * 3,
Testing: time.Second * 14,
}).(time.Duration)
)
...@@ -5,8 +5,6 @@ package hostdb ...@@ -5,8 +5,6 @@ package hostdb
// settings of the hosts. // settings of the hosts.
import ( import (
"crypto/rand"
"math/big"
"net" "net"
"time" "time"
...@@ -295,7 +293,7 @@ func (hdb *HostDB) threadedScan() { ...@@ -295,7 +293,7 @@ func (hdb *HostDB) threadedScan() {
if online && len(onlineHosts) < hostCheckupQuantity { if online && len(onlineHosts) < hostCheckupQuantity {
onlineHosts = append(onlineHosts, host) onlineHosts = append(onlineHosts, host)
} else if !online && len(offlineHosts) < hostCheckupQuantity { } else if !online && len(offlineHosts) < hostCheckupQuantity {
offlineHosts = append(onlineHosts, host) offlineHosts = append(offlineHosts, host)
} }
} }
...@@ -314,21 +312,20 @@ func (hdb *HostDB) threadedScan() { ...@@ -314,21 +312,20 @@ func (hdb *HostDB) threadedScan() {
// scanning. The minimums and maximums keep the scan time reasonable, // scanning. The minimums and maximums keep the scan time reasonable,
// while the randomness prevents the scanning from always happening at // while the randomness prevents the scanning from always happening at
// the same time of day or week. // the same time of day or week.
maxBig := big.NewInt(int64(maxScanSleep)) sleepTime := defaultScanSleep
minBig := big.NewInt(int64(minScanSleep)) sleepRange := int(maxScanSleep - minScanSleep)
randSleep, err := rand.Int(rand.Reader, maxBig.Sub(maxBig, minBig)) sleepRand, err := crypto.RandIntn(sleepRange)
if err != nil { if err != nil {
build.Critical(err) build.Critical(err)
// If there's an error, sleep for the default amount of time. } else {
defaultBig := big.NewInt(int64(defaultScanSleep)) sleepTime = minScanSleep + time.Duration(sleepRand)
randSleep = defaultBig.Sub(defaultBig, minBig)
} }
// Sleep until it's time for the next scan cycle. // Sleep until it's time for the next scan cycle.
select { select {
case <-hdb.tg.StopChan(): case <-hdb.tg.StopChan():
return return
case <-time.After(time.Duration(randSleep.Int64()) + minScanSleep): case <-time.After(sleepTime):
} }
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment