Commit 5d6427bb authored by Luke Champine's avatar Luke Champine

replace current persist with journal

parent 75ba81a7
package contractor
import (
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"testing"
"time"
......@@ -73,13 +70,6 @@ func TestNew(t *testing.T) {
if !os.IsNotExist(err) {
t.Fatalf("expected invalid directory, got %v", err)
}
// Corrupted persist file.
ioutil.WriteFile(filepath.Join(dir, "contractor.json"), []byte{1, 2, 3}, 0666)
_, err = New(stub, stub, stub, stub, dir)
if _, ok := err.(*json.SyntaxError); !ok {
t.Fatalf("expected invalid json, got %v", err)
}
}
// TestContract tests the Contract method.
......
......@@ -4,7 +4,6 @@ import (
"path/filepath"
"github.com/NebulousLabs/Sia/modules"
"github.com/NebulousLabs/Sia/persist"
"github.com/NebulousLabs/Sia/types"
)
......@@ -54,7 +53,7 @@ type (
persister interface {
save(contractorPersist) error
saveSync(contractorPersist) error
update(...interface{}) error
load(*contractorPersist) error
}
)
......@@ -68,32 +67,43 @@ type walletBridge struct {
func (ws *walletBridge) NextAddress() (types.UnlockConditions, error) { return ws.w.NextAddress() }
func (ws *walletBridge) StartTransaction() transactionBuilder { return ws.w.StartTransaction() }
// stdPersist implements the persister interface via persist.SaveFile and
// persist.LoadFile. The metadata and filename required by these functions is
// internal to stdPersist.
// stdPersist implements the persister interface via jj.OpenJournal and
// jj.CheckPoint. The filename required by these functions is internal to
// stdPersist.
type stdPersist struct {
meta persist.Metadata
journal *journal
filename string
}
func (p *stdPersist) save(data contractorPersist) error {
return persist.SaveFile(p.meta, data, p.filename)
func (p stdPersist) save(data contractorPersist) error {
if p.journal == nil {
var err error
p.journal, err = openJournal(p.filename, data)
return err
}
return p.journal.checkpoint(data)
}
func (p *stdPersist) saveSync(data contractorPersist) error {
return persist.SaveFileSync(p.meta, data, p.filename)
func (p *stdPersist) update(us ...interface{}) error {
var updates []journalUpdate
for i := 0; i < len(us); i += 2 {
updates = append(updates, newJournalUpdate(us[i].(string), us[i+1].(interface{})))
}
return p.journal.update(updates)
}
func (p *stdPersist) load(data *contractorPersist) error {
return persist.LoadFile(p.meta, data, p.filename)
var err error
p.journal, err = openJournal(p.filename, data)
if err != nil {
// try loading old persist
return loadv110persist(filepath.Dir(p.filename), data)
}
return err
}
func newPersist(dir string) *stdPersist {
return &stdPersist{
meta: persist.Metadata{
Header: "Contractor Persistence",
Version: "0.5.2",
},
filename: filepath.Join(dir, "contractor.json"),
filename: filepath.Join(dir, "contractor.journal"),
}
}
......@@ -2,6 +2,7 @@ package contractor
import (
"errors"
"fmt"
"sync"
"github.com/NebulousLabs/Sia/build"
......@@ -79,7 +80,18 @@ func (hd *hostDownloader) Sector(root crypto.Hash) ([]byte, error) {
hd.contractor.mu.Lock()
hd.contractor.contracts[contract.ID] = contract
hd.contractor.saveSync()
cpath := fmt.Sprintf("contracts.%s", contract.ID.String())
hd.contractor.persist.update(
cpath+".lastrevision.newrevisionnumber", contract.LastRevision.NewRevisionNumber,
cpath+".lastrevision.newvalidproofoutputs", contract.LastRevision.NewValidProofOutputs,
cpath+".lastrevision.newmissedproofoutputs", contract.LastRevision.NewMissedProofOutputs,
cpath+".lastrevisiontxn.filecontractrevisions.0.newrevisionnumber", contract.LastRevisionTxn.FileContractRevisions[0].NewRevisionNumber,
cpath+".lastrevisiontxn.filecontractrevisions.0.newvalidproofoutputs", contract.LastRevisionTxn.FileContractRevisions[0].NewValidProofOutputs,
cpath+".lastrevisiontxn.filecontractrevisions.0.newmissedproofoutputs", contract.LastRevisionTxn.FileContractRevisions[0].NewMissedProofOutputs,
cpath+".lastrevisiontxn.transactionsignatures.0.signature", contract.LastRevisionTxn.TransactionSignatures[0].Signature,
cpath+".lastrevisiontxn.transactionsignatures.1.signature", contract.LastRevisionTxn.TransactionSignatures[1].Signature,
cpath+".downloadspending", contract.DownloadSpending,
)
hd.contractor.mu.Unlock()
return sector, nil
......
......@@ -2,6 +2,7 @@ package contractor
import (
"errors"
"fmt"
"sync"
"github.com/NebulousLabs/Sia/build"
......@@ -113,7 +114,24 @@ func (he *hostEditor) Upload(data []byte) (crypto.Hash, error) {
}
he.contractor.mu.Lock()
he.contractor.contracts[contract.ID] = contract
he.contractor.saveSync()
cpath := fmt.Sprintf("contracts.%s", contract.ID.String())
he.contractor.persist.update(
cpath+".lastrevision.newrevisionnumber", contract.LastRevision.NewRevisionNumber,
cpath+".lastrevision.newfilesize", contract.LastRevision.NewFileSize,
cpath+".lastrevision.newfilemerkleroot", contract.LastRevision.NewFileMerkleRoot,
cpath+".lastrevision.newvalidproofoutputs", contract.LastRevision.NewValidProofOutputs,
cpath+".lastrevision.newmissedproofoutputs", contract.LastRevision.NewMissedProofOutputs,
cpath+".lastrevisiontxn.filecontractrevisions.0.newrevisionnumber", contract.LastRevisionTxn.FileContractRevisions[0].NewRevisionNumber,
cpath+".lastrevisiontxn.filecontractrevisions.0.newfilesize", contract.LastRevision.NewFileSize,
cpath+".lastrevisiontxn.filecontractrevisions.0.newfilemerkleroot", contract.LastRevision.NewFileMerkleRoot,
cpath+".lastrevisiontxn.filecontractrevisions.0.newvalidproofoutputs", contract.LastRevisionTxn.FileContractRevisions[0].NewValidProofOutputs,
cpath+".lastrevisiontxn.filecontractrevisions.0.newmissedproofoutputs", contract.LastRevisionTxn.FileContractRevisions[0].NewMissedProofOutputs,
cpath+".lastrevisiontxn.transactionsignatures.0.signature", contract.LastRevisionTxn.TransactionSignatures[0].Signature,
cpath+".lastrevisiontxn.transactionsignatures.1.signature", contract.LastRevisionTxn.TransactionSignatures[1].Signature,
cpath+".merkleroots."+fmt.Sprint(len(contract.MerkleRoots)-1), sectorRoot,
cpath+".uploadspending", contract.UploadSpending,
cpath+".storagespending", contract.StorageSpending,
)
he.contractor.mu.Unlock()
he.contract = contract
......
package contractor
import (
"path/filepath"
"github.com/NebulousLabs/Sia/crypto"
"github.com/NebulousLabs/Sia/modules"
"github.com/NebulousLabs/Sia/persist"
"github.com/NebulousLabs/Sia/types"
)
// contractorPersist defines what Contractor data persists across sessions.
type contractorPersist struct {
Allowance modules.Allowance
BlockHeight types.BlockHeight
CachedRevisions []cachedRevision
Contracts []modules.RenterContract
CurrentPeriod types.BlockHeight
LastChange modules.ConsensusChangeID
OldContracts []modules.RenterContract
RenewedIDs map[string]string
// COMPATv1.0.4-lts
FinancialMetrics struct {
ContractSpending types.Currency `json:"contractspending"`
DownloadSpending types.Currency `json:"downloadspending"`
StorageSpending types.Currency `json:"storagespending"`
UploadSpending types.Currency `json:"uploadspending"`
} `json:",omitempty"`
Allowance modules.Allowance `json:"allowance"`
BlockHeight types.BlockHeight `json:"blockheight"`
CachedRevisions map[string]cachedRevision `json:"cachedrevisions"`
Contracts map[string]modules.RenterContract `json:"contracts"`
CurrentPeriod types.BlockHeight `json:"currentperiod"`
LastChange modules.ConsensusChangeID `json:"lastchange"`
OldContracts []modules.RenterContract `json:"oldcontracts"`
RenewedIDs map[string]string `json:"renewedids"`
}
// persistData returns the data in the Contractor that will be saved to disk.
func (c *Contractor) persistData() contractorPersist {
data := contractorPersist{
Allowance: c.allowance,
BlockHeight: c.blockHeight,
CurrentPeriod: c.currentPeriod,
LastChange: c.lastChange,
RenewedIDs: make(map[string]string),
Allowance: c.allowance,
BlockHeight: c.blockHeight,
CachedRevisions: make(map[string]cachedRevision),
Contracts: make(map[string]modules.RenterContract),
CurrentPeriod: c.currentPeriod,
LastChange: c.lastChange,
RenewedIDs: make(map[string]string),
}
for _, rev := range c.cachedRevisions {
data.CachedRevisions = append(data.CachedRevisions, rev)
data.CachedRevisions[rev.Revision.ParentID.String()] = rev
}
for _, contract := range c.contracts {
data.Contracts = append(data.Contracts, contract)
data.Contracts[contract.ID.String()] = contract
}
for _, contract := range c.oldContracts {
data.OldContracts = append(data.OldContracts, contract)
......@@ -110,31 +107,6 @@ func (c *Contractor) load() error {
c.renewedIDs[types.FileContractID(oldHash)] = types.FileContractID(newHash)
}
// COMPATv1.0.4-lts
//
// If loading old persist, only aggregate metrics are known. Store these
// in a special contract under a special identifier.
if fm := data.FinancialMetrics; !fm.ContractSpending.Add(fm.DownloadSpending).Add(fm.StorageSpending).Add(fm.UploadSpending).IsZero() {
c.oldContracts[metricsContractID] = modules.RenterContract{
ID: metricsContractID,
TotalCost: fm.ContractSpending,
DownloadSpending: fm.DownloadSpending,
StorageSpending: fm.StorageSpending,
UploadSpending: fm.UploadSpending,
// Give the contract a fake startheight so that it will included
// with the other contracts in the current period. Note that in
// update.go, the special contract is specifically deleted when a
// new period begins.
StartHeight: c.currentPeriod + 1,
// We also need to add a ValidProofOutput so that the RenterFunds
// method will not panic. The value should be 0, i.e. "all funds
// were spent."
LastRevision: types.FileContractRevision{
NewValidProofOutputs: make([]types.SiacoinOutput, 2),
},
}
}
return nil
}
......@@ -145,7 +117,7 @@ func (c *Contractor) save() error {
// saveSync saves the Contractor persistence data to disk and then syncs to disk.
func (c *Contractor) saveSync() error {
return c.persist.saveSync(c.persistData())
return c.persist.save(c.persistData())
}
// saveRevision returns a function that saves a revision. It is used by the
......@@ -155,20 +127,22 @@ func (c *Contractor) saveRevision(id types.FileContractID) func(types.FileContra
c.mu.Lock()
defer c.mu.Unlock()
c.cachedRevisions[id] = cachedRevision{rev, newRoots}
return c.saveSync()
revPath := "cachedRevisions." + id.String()
return c.persist.update(revPath, cachedRevision{rev, newRoots})
}
}
// addPubKeys rescans the blockchain to fill in the HostPublicKey of
// contracts, identified by their NetAddress.
func addPubKeys(cs consensusSet, contracts []modules.RenterContract) []modules.RenterContract {
func addPubKeys(cs consensusSet, contracts map[string]modules.RenterContract) map[string]modules.RenterContract {
pubkeys := make(pubkeyScanner)
for _, c := range contracts {
pubkeys[c.NetAddress] = types.SiaPublicKey{}
}
cs.ConsensusSetSubscribe(pubkeys, modules.ConsensusChangeBeginning)
for i, c := range contracts {
contracts[i].HostPublicKey = pubkeys[c.NetAddress]
for id, c := range contracts {
c.HostPublicKey = pubkeys[c.NetAddress]
contracts[id] = c
}
return contracts
}
......@@ -196,3 +170,75 @@ func (pubkeys pubkeyScanner) ProcessConsensusChange(cc modules.ConsensusChange)
}
}
}
// COMPATv1.1.0
func loadv110persist(dir string, data *contractorPersist) error {
var oldPersist struct {
Allowance modules.Allowance
BlockHeight types.BlockHeight
CachedRevisions []cachedRevision
Contracts []modules.RenterContract
CurrentPeriod types.BlockHeight
LastChange modules.ConsensusChangeID
OldContracts []modules.RenterContract
RenewedIDs map[string]string
FinancialMetrics struct {
ContractSpending types.Currency
DownloadSpending types.Currency
StorageSpending types.Currency
UploadSpending types.Currency
}
}
err := persist.LoadFile(persist.Metadata{
Header: "Contractor Persistence",
Version: "0.5.2",
}, &oldPersist, filepath.Join(dir, "contractor.json"))
if err != nil {
return err
}
cachedRevisions := make(map[string]cachedRevision)
for _, rev := range oldPersist.CachedRevisions {
cachedRevisions[rev.Revision.ParentID.String()] = rev
}
contracts := make(map[string]modules.RenterContract)
for _, c := range oldPersist.Contracts {
contracts[c.ID.String()] = c
}
// COMPATv1.0.4-lts
//
// If loading old persist, only aggregate metrics are known. Store these
// in a special contract under a special identifier.
if fm := oldPersist.FinancialMetrics; !fm.ContractSpending.Add(fm.DownloadSpending).Add(fm.StorageSpending).Add(fm.UploadSpending).IsZero() {
oldPersist.OldContracts = append(oldPersist.OldContracts, modules.RenterContract{
ID: metricsContractID,
TotalCost: fm.ContractSpending,
DownloadSpending: fm.DownloadSpending,
StorageSpending: fm.StorageSpending,
UploadSpending: fm.UploadSpending,
// Give the contract a fake startheight so that it will included
// with the other contracts in the current period. Note that in
// update.go, the special contract is specifically deleted when a
// new period begins.
StartHeight: oldPersist.CurrentPeriod + 1,
// We also need to add a ValidProofOutput so that the RenterFunds
// method will not panic. The value should be 0, i.e. "all funds
// were spent."
LastRevision: types.FileContractRevision{
NewValidProofOutputs: make([]types.SiacoinOutput, 2),
},
})
}
*data = contractorPersist{
Allowance: oldPersist.Allowance,
BlockHeight: oldPersist.BlockHeight,
CachedRevisions: cachedRevisions,
Contracts: contracts,
CurrentPeriod: oldPersist.CurrentPeriod,
LastChange: oldPersist.LastChange,
OldContracts: oldPersist.OldContracts,
RenewedIDs: oldPersist.RenewedIDs,
}
return nil
}
......@@ -15,9 +15,9 @@ import (
// memPersist implements the persister interface in-memory.
type memPersist contractorPersist
func (m *memPersist) save(data contractorPersist) error { *m = memPersist(data); return nil }
func (m *memPersist) saveSync(data contractorPersist) error { *m = memPersist(data); return nil }
func (m memPersist) load(data *contractorPersist) error { *data = contractorPersist(m); return nil }
func (m *memPersist) save(data contractorPersist) error { *m = memPersist(data); return nil }
func (m *memPersist) update(...interface{}) error { return nil }
func (m memPersist) load(data *contractorPersist) error { *data = contractorPersist(m); return nil }
// TestSaveLoad tests that the contractor can save and load itself.
func TestSaveLoad(t *testing.T) {
......
......@@ -148,7 +148,7 @@ func TestIsOffline(t *testing.T) {
// construct a contractor with a hostdb containing the scans
c := &Contractor{
contracts: map[types.FileContractID]modules.RenterContract{
types.FileContractID{1}: {HostPublicKey: types.SiaPublicKey{Key: []byte("foo")}},
{1}: {HostPublicKey: types.SiaPublicKey{Key: []byte("foo")}},
},
hdb: mapHostDB{
hosts: map[string]modules.HostDBEntry{
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment