Unverified Commit 1982a962 authored by David Vorick's avatar David Vorick

Merge branch 'master' into host2

Conflicts:
	Makefile
	modules/host/storageobligations.go
parents 9ba9dede db145039
......@@ -4,7 +4,7 @@ os:
- linux
go:
- 1.6
- 1.8
install:
- make dependencies
......
Version History
---------------
February 2017:
v1.1.1 (patch release)
- Renter now performs much better at scale
- Myriad HostDB improvements
- Add siac command to support storage leaderboard
January 2017:
v1.1.0 (minor release)
- Greatly improved upload/download speeds
- Wallet now regularly "defragments"
- Better contract metrics
December 2016:
v1.0.4 (LTS release)
October 2016:
v1.0.3 (patch release)
- Greatly improved renter stability
- Smarter HostDB
- Numerous minor bug fixes
July 2016:
v1.0.1 (patch release)
- Restricted API address to localhost
- Fixed renter/host desynchronization
- Fixed host silently refusing new contracts
June 2016:
v1.0.0 (major release)
- Finalized API routes
- Add optional API authentication
- Improve automatic contract management
May 2016:
v0.6.0 (minor release)
- Switched to long-form renter contracts
- Added support for multiple hosting folders
- Hosts are now identified by their public key
January 2016:
v0.5.2 (patch release)
- Faster initial blockchain download
- Introduced headers-only broadcasting
v0.5.1 (patch release)
- Fixed bug severely impacting performance
- Restored (but deprecated) some siac commands
- Added modules flag, allowing modules to be disabled
v0.5.0 (minor release)
- Major API changes to most modules
- Automatic contract renewal
- Data on inactive hosts is reuploaded
- Support for folder structure
- Smarter host
October 2015:
v0.4.8 (patch release)
- Restored compatibility with v0.4.6
v0.4.7 (patch release)
- Dropped support for v0.3.3.x
v0.4.6 (patch release)
- Removed over-aggressive consistency check
v0.4.5 (patch release)
- Fixed last prominent bug in block database
- Closed some dangling resource handles
v0.4.4 (patch release)
- Uploading is much more reliable
- Price estimations are more accurate
- Bumped filesize limit to 20 GB
v0.4.3 (patch release)
- Block database is now faster and more stable
- Wallet no longer freezes when unlocked during IBD
- Optimized block encoding/decoding
September 2015:
v0.4.2 (patch release)
- HostDB is now smarter
- Tweaked renter contract creation
v0.4.1 (patch release)
- Added support for loading v0.3.3.x wallets
- Better pruning of dead nodes
- Improve database consistency
August 2015:
v0.4.0: Second stable currency release.
- Wallets are encrypted and generated from seed phrases
- Files are erasure-coded and transferred in parallel
- The blockchain is now fully on-disk
- Added UPnP support
June 2015:
v0.3.3.3 (patch release)
- Host announcements can be "forced"
- Wallets can be merged
- Unresponsive addresses are pruned from the node list
v0.3.3.2 (patch release)
- Siafunds can be loaded and sent
- Added block explorer
- Patched two critical security vulnerabilities
v0.3.3.1 (hotfix)
- Mining API sends headers instead of entire blocks
- Slashed default hosting price
v0.3.3: First stable currency release.
- Set release target
- Added progress bars to uploads
- Rigorous testing of consensus code
May 2015:
v0.3.2: Fourth open beta release.
- Switched encryption from block cipher to stream cipher
- Updates are now signed
- Added API calls to support external miners
v0.3.1: Third open beta release.
- Blocks are now stored on-disk in a database
- Files can be shared via .sia files or ASCII-encoded data
- RPCs are now multiplexed over one physical connection
March 2015:
v0.3.0: Second open beta release.
Jan 2015:
v0.2.0: First open beta release.
Dec 2014:
v0.1.0: Closed beta release.
......@@ -31,8 +31,9 @@ dependencies:
run = Test
pkgs = ./api ./build ./compatibility ./crypto ./encoding ./modules ./modules/consensus \
./modules/explorer ./modules/gateway ./modules/host ./modules/host/contractmanager \
./modules/renter ./modules/renter/contractor ./modules/renter/hostdb ./modules/renter/proto \
./modules/miner ./modules/wallet ./modules/transactionpool ./persist ./siac ./siad ./sync ./types
./modules/renter ./modules/renter/contractor ./modules/renter/hostdb ./modules/renter/hostdb/hosttree \
./modules/renter/proto ./modules/miner ./modules/wallet ./modules/transactionpool ./persist ./siac \
./siad ./sync ./types
# fmt calls go fmt on all packages.
fmt:
......
This diff is collapsed.
......@@ -5,6 +5,7 @@ import (
"net/http"
"strings"
"github.com/NebulousLabs/Sia/build"
"github.com/NebulousLabs/Sia/modules"
"github.com/julienschmidt/httprouter"
......@@ -155,10 +156,12 @@ func New(requiredUserAgent string, requiredPassword string, cs modules.Consensus
// Register API handlers
router := httprouter.New()
router.NotFound = http.HandlerFunc(UnrecognizedCallHandler)
router.RedirectTrailingSlash = false
// Consensus API Calls
if api.cs != nil {
router.GET("/consensus", api.consensusHandler)
router.POST("/consensus/validate/transactionset", api.consensusValidateTransactionsetHandler)
}
// Explorer API Calls
......@@ -206,6 +209,7 @@ func New(requiredUserAgent string, requiredPassword string, cs modules.Consensus
router.GET("/renter/contracts", api.renterContractsHandler)
router.GET("/renter/downloads", api.renterDownloadsHandler)
router.GET("/renter/files", api.renterFilesHandler)
router.GET("/renter/prices", api.renterPricesHandler)
// TODO: re-enable these routes once the new .sia format has been
// standardized and implemented.
......@@ -216,12 +220,14 @@ func New(requiredUserAgent string, requiredPassword string, cs modules.Consensus
router.POST("/renter/delete/*siapath", RequirePassword(api.renterDeleteHandler, requiredPassword))
router.GET("/renter/download/*siapath", RequirePassword(api.renterDownloadHandler, requiredPassword))
router.GET("/renter/downloadasync/*siapath", RequirePassword(api.renterDownloadAsyncHandler, requiredPassword))
router.POST("/renter/rename/*siapath", RequirePassword(api.renterRenameHandler, requiredPassword))
router.POST("/renter/upload/*siapath", RequirePassword(api.renterUploadHandler, requiredPassword))
// HostDB endpoints.
router.GET("/hostdb/active", api.renterHostsActiveHandler)
router.GET("/hostdb/all", api.renterHostsAllHandler)
router.GET("/hostdb/active", api.hostdbActiveHandler)
router.GET("/hostdb/all", api.hostdbAllHandler)
router.GET("/hostdb/hosts/:pubkey", api.hostdbHostsHandler)
}
// TransactionPool API Calls
......@@ -264,8 +270,11 @@ func UnrecognizedCallHandler(w http.ResponseWriter, req *http.Request) {
func WriteError(w http.ResponseWriter, err Error, code int) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(code)
if json.NewEncoder(w).Encode(err) != nil {
http.Error(w, "Failed to encode error response", http.StatusInternalServerError)
encodingErr := json.NewEncoder(w).Encode(err)
if _, isJsonErr := encodingErr.(*json.SyntaxError); isJsonErr {
// Marshalling should only fail in the event of a developer error.
// Specifically, only non-marshallable types should cause an error here.
build.Critical("failed to encode API error response:", encodingErr)
}
}
......@@ -274,8 +283,11 @@ func WriteError(w http.ResponseWriter, err Error, code int) {
// accordingly.
func WriteJSON(w http.ResponseWriter, obj interface{}) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
if json.NewEncoder(w).Encode(obj) != nil {
http.Error(w, "Failed to encode response", http.StatusInternalServerError)
err := json.NewEncoder(w).Encode(obj)
if _, isJsonErr := err.(*json.SyntaxError); isJsonErr {
// Marshalling should only fail in the event of a developer error.
// Specifically, only non-marshallable types should cause an error here.
build.Critical("failed to encode API response:", err)
}
}
......
......@@ -6,11 +6,12 @@ import (
// TestApiClient tests that the API client connects to the server tester and
// can call and decode routes correctly.
func TestIntegrationApiClient(t *testing.T) {
func TestApiClient(t *testing.T) {
if testing.Short() {
t.SkipNow()
}
st, err := createServerTester("TestApiClient")
t.Parallel()
st, err := createServerTester(t.Name())
if err != nil {
t.Fatal(err)
}
......@@ -27,12 +28,13 @@ func TestIntegrationApiClient(t *testing.T) {
// TestAuthenticatedApiClient tests that the API client connects to an
// authenticated server tester and can call and decode routes correctly, using
// the correct password.
func TestIntegrationAuthenticatedApiClient(t *testing.T) {
func TestAuthenticatedApiClient(t *testing.T) {
if testing.Short() {
t.SkipNow()
}
t.Parallel()
testpass := "testPassword"
st, err := createAuthenticatedServerTester("TestAuthenticatedApiClient", testpass)
st, err := createAuthenticatedServerTester(t.Name(), testpass)
if err != nil {
t.Fatal(err)
}
......
package api
import (
"encoding/json"
"net/http"
"github.com/NebulousLabs/Sia/types"
......@@ -28,3 +29,20 @@ func (api *API) consensusHandler(w http.ResponseWriter, req *http.Request, _ htt
Target: currentTarget,
})
}
// consensusValidateTransactionsetHandler handles the API calls to
// /consensus/validate/transactionset.
func (api *API) consensusValidateTransactionsetHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) {
var txnset []types.Transaction
err := json.NewDecoder(req.Body).Decode(&txnset)
if err != nil {
WriteError(w, Error{"could not decode transaction set: " + err.Error()}, http.StatusBadRequest)
return
}
_, err = api.cs.TryTransactionSet(txnset)
if err != nil {
WriteError(w, Error{"transaction set validation failed: " + err.Error()}, http.StatusBadRequest)
return
}
WriteSuccess(w)
}
package api
import (
"encoding/json"
"net/url"
"testing"
"github.com/NebulousLabs/Sia/types"
)
// TestIntegrationConsensusGet probes the GET call to /consensus.
// TestConsensusGet probes the GET call to /consensus.
func TestIntegrationConsensusGET(t *testing.T) {
if testing.Short() {
t.SkipNow()
}
st, err := createServerTester("TestIntegrationConsensusGET")
t.Parallel()
st, err := createServerTester(t.Name())
if err != nil {
t.Fatal(err)
}
defer st.server.Close()
var cg ConsensusGET
err = st.getAPI("/consensus", &cg)
if err != nil {
......@@ -33,3 +36,76 @@ func TestIntegrationConsensusGET(t *testing.T) {
t.Error("wrong target returned in consensus GET call")
}
}
// TestConsensusValidateTransactionSet probes the POST call to
// /consensus/validate/transactionset.
func TestConsensusValidateTransactionSet(t *testing.T) {
if testing.Short() {
t.SkipNow()
}
t.Parallel()
st, err := createServerTester(t.Name())
if err != nil {
t.Fatal(err)
}
defer st.server.Close()
// Anounce the host and start accepting contracts.
if err := st.announceHost(); err != nil {
t.Fatal(err)
}
if err = st.acceptContracts(); err != nil {
t.Fatal(err)
}
if err = st.setHostStorage(); err != nil {
t.Fatal(err)
}
// Set an allowance for the renter, allowing a contract to be formed.
allowanceValues := url.Values{}
allowanceValues.Set("funds", testFunds)
allowanceValues.Set("period", testPeriod)
if err = st.stdPostAPI("/renter", allowanceValues); err != nil {
t.Fatal(err)
}
st.miner.AddBlock()
// Get the contract
var cs RenterContracts
if err = st.getAPI("/renter/contracts", &cs); err != nil {
t.Fatal(err)
}
if len(cs.Contracts) != 1 {
t.Fatalf("expected renter to have 1 contracts; got %v", len(cs.Contracts))
}
contract := cs.Contracts[0]
// Validate the contract
jsonTxns, err := json.Marshal([]types.Transaction{contract.LastTransaction})
if err != nil {
t.Fatal(err)
}
resp, err := HttpPOST("http://"+st.server.listener.Addr().String()+"/consensus/validate/transactionset", string(jsonTxns))
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if non2xx(resp.StatusCode) {
t.Fatal(decodeError(resp))
}
// Try again with an invalid contract
contract.LastTransaction.FileContractRevisions[0].NewFileSize++
jsonTxns, err = json.Marshal([]types.Transaction{contract.LastTransaction})
if err != nil {
t.Fatal(err)
}
resp, err = HttpPOST("http://"+st.server.listener.Addr().String()+"/consensus/validate/transactionset", string(jsonTxns))
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if !non2xx(resp.StatusCode) {
t.Fatal("expected validation error")
}
}
......@@ -3,15 +3,195 @@ package api
// ecosystem_test.go provides tooling and tests for whole-ecosystem testing,
// consisting of multiple full, non-state-sharing nodes connected in various
// arrangements and performing various full-ecosystem tasks.
//
// To the absolute greatest extent possible, nodes are queried and updated
// exclusively through the API.
import (
"errors"
"net/url"
"testing"
"time"
"github.com/NebulousLabs/Sia/types"
)
// addStorageToAllHosts adds a storage folder with a bunch of storage to each
// host.
func addStorageToAllHosts(sts []*serverTester) error {
for _, st := range sts {
values := url.Values{}
values.Set("path", st.dir)
values.Set("size", "1048576")
err := st.stdPostAPI("/host/storage/folders/add", values)
if err != nil {
return err
}
}
return nil
}
// announceAllHosts will announce every host in the tester set to the
// blockchain.
func announceAllHosts(sts []*serverTester) error {
// Check that all announcements will be on the same chain.
_, err := synchronizationCheck(sts)
if err != nil {
return err
}
// Announce each host.
for _, st := range sts {
// Set the host to be accepting contracts.
acceptingContractsValues := url.Values{}
acceptingContractsValues.Set("acceptingcontracts", "true")
err = st.stdPostAPI("/host", acceptingContractsValues)
if err != nil {
return err
}
// Fetch the host net address.
var hg HostGET
err = st.getAPI("/host", &hg)
if err != nil {
return err
}
// Make the announcement.
announceValues := url.Values{}
announceValues.Set("address", string(hg.ExternalSettings.NetAddress))
err = st.stdPostAPI("/host/announce", announceValues)
if err != nil {
return err
}
}
// Wait until all of the transactions have propagated to all of the nodes.
//
// TODO: Replace this direct transaction pool call with a call to the
// /transactionpool endpoint.
//
// TODO: At some point the number of transactions needed to make an
// announcement may change. Currently its 2.
for i := 0; i < 50; i++ {
if len(sts[0].tpool.TransactionList()) == len(sts)*2 {
break
}
time.Sleep(time.Millisecond * 100)
}
if len(sts[0].tpool.TransactionList()) != len(sts)*2 {
return errors.New("Host announcements do not seem to have propagated to the leader's tpool")
}
// Mine a block and then wait for all of the nodes to syncrhonize to it.
_, err = sts[0].miner.AddBlock()
if err != nil {
return err
}
_, err = synchronizationCheck(sts)
if err != nil {
return err
}
// Block until every node has completed the scan of every other node, so
// that each node has a full hostdb.
for _, st := range sts {
var ah HostdbActiveGET
for i := 0; i < 50; i++ {
err = st.getAPI("/hostdb/active", &ah)
if err != nil {
return err
}
if len(ah.Hosts) >= len(sts) {
break
}
time.Sleep(time.Millisecond * 100)
}
if len(ah.Hosts) < len(sts) {
return errors.New("one of the nodes hostdbs was unable to find at least one host announcement")
}
}
return nil
}
// fullyConnectNodes takes a bunch of tester nodes and connects each to the
// other, creating a fully connected graph so that everyone is on the same
// chain.
//
// After connecting the nodes, it verifies that all the nodes have
// synchronized.
func fullyConnectNodes(sts []*serverTester) error {
for i, sta := range sts {
var gg GatewayGET
err := sta.getAPI("/gateway", &gg)
if err != nil {
return err
}
// Connect this node to every other node.
for _, stb := range sts[i+1:] {
// NOTE: this check depends on string-matching an error in the
// gateway. If that error changes at all, this string will need to
// be updated.
err := stb.stdPostAPI("/gateway/connect/"+string(gg.NetAddress), nil)
if err != nil && err.Error() != "already connected to this peer" {
return err
}
}
}
// Perform a synchronization check.
_, err := synchronizationCheck(sts)
return err
}
// fundAllNodes will make sure that each node has mined a block in the longest
// chain, then will mine enough blocks that the miner payouts manifest in the
// wallets of each node.
func fundAllNodes(sts []*serverTester) error {
// Check that all of the nodes are synchronized.
chainTip, err := synchronizationCheck(sts)
if err != nil {
return err
}
// Mine a block for each node to fund their wallet.
for i := range sts {
err := waitForBlock(chainTip, sts[i])
if err != nil {
return err
}
// Mine a block. The next iteration of this loop will ensure that the
// block propagates and does not get orphaned.
block, err := sts[i].miner.AddBlock()
if err != nil {
return err
}
chainTip = block.ID()
}
// Wait until the chain tip has propagated to the first node.
err = waitForBlock(chainTip, sts[0])
if err != nil {
return err
}
// Mine types.MaturityDelay more blocks from the final node to mine a
// block, to guarantee that all nodes have had their payouts mature, such
// that their wallets can begin spending immediately.
for i := types.BlockHeight(0); i <= types.MaturityDelay; i++ {
_, err := sts[0].miner.AddBlock()
if err != nil {
return err
}
}
// Block until every node has the full chain.
_, err = synchronizationCheck(sts)
return err
}
// synchronizationCheck takes a bunch of server testers as input and checks
// that they all have the same current block as the first server tester. The
// first server tester needs to have the most recent block in order for the
......@@ -52,6 +232,28 @@ func synchronizationCheck(sts []*serverTester) (types.BlockID, error) {
return leaderBlockID, nil
}
// waitForBlock will block until the provided chain tip is the most recent
// block in the provided testing node.
func waitForBlock(chainTip types.BlockID, st *serverTester) error {
var cg ConsensusGET
success := false
for j := 0; j < 100; j++ {
err := st.getAPI("/consensus", &cg)
if err != nil {
return err
}
if cg.CurrentBlock == chainTip {
success = true
break
}
time.Sleep(time.Millisecond * 100)
}
if !success {
return errors.New("node never reached the correct chain tip")
}
return nil
}
// TestHostPoorConnectivity creates several full server testers and links them
// together in a way that might mimic a full host ecosystem with a renter, and
// then isolates one of the hosts from the network, denying the host proper
......@@ -66,31 +268,31 @@ func TestHostPoorConnectivity(t *testing.T) {
// Create the various nodes that will be forming the simulated ecosystem of
// this test.
stLeader, err := createServerTester("TestHostPoorConnectivity - Leader")
stLeader, err := createServerTester(t.Name())
if err != nil {
t.Fatal(err)
}
stHost1, err := blankServerTester("TestHostPoorConnectivity - Host 1")
stHost1, err := blankServerTester(t.Name() + " - Host 1")
if err != nil {
t.Fatal(err)
}
stHost2, err := blankServerTester("TestHostPoorConnectivity - Host 2")
stHost2, err := blankServerTester(t.Name() + " - Host 2")
if err != nil