drop the storage manager

parent c0f70994
......@@ -30,7 +30,7 @@ dependencies:
# tests are run during testing.
run = Test
pkgs = ./api ./build ./compatibility ./crypto ./encoding ./modules ./modules/consensus \
./modules/explorer ./modules/gateway ./modules/host ./modules/host/contractmanager ./modules/host/storagemanager \
./modules/explorer ./modules/gateway ./modules/host ./modules/host/contractmanager \
./modules/renter ./modules/renter/contractor ./modules/renter/hostdb ./modules/renter/proto \
./modules/miner ./modules/wallet ./modules/transactionpool ./persist ./siac ./siad ./sync ./types
......
......@@ -42,9 +42,9 @@ type (
// folderIndex determines the index of the storage folder with the provided
// path.
func folderIndex(folderPath string, storageFolders []modules.StorageFolderMetadata) (int, error) {
for i, sf := range storageFolders {
for _, sf := range storageFolders {
if sf.Path == folderPath {
return i, nil
return int(sf.Index), nil
}
}
return -1, errStorageFolderNotFound
......
......@@ -10,17 +10,17 @@ import (
"github.com/NebulousLabs/Sia/crypto"
"github.com/NebulousLabs/Sia/modules"
"github.com/NebulousLabs/Sia/modules/host/storagemanager"
"github.com/NebulousLabs/Sia/modules/host/contractmanager"
)
var (
// Various folder sizes for testing host storage folder resizing.
// Must be provided as strings to the API call.
minFolderSizeString = strconv.FormatUint(storagemanager.MinimumStorageFolderSize(), 10)
maxFolderSizeString = strconv.FormatUint(storagemanager.MaximumStorageFolderSize(), 10)
tooSmallFolderString = strconv.FormatUint(storagemanager.MinimumStorageFolderSize()-1, 10)
tooLargeFolderString = strconv.FormatUint(storagemanager.MaximumStorageFolderSize()+1, 10)
mediumSizeFolderString = strconv.FormatUint(3*storagemanager.MinimumStorageFolderSize(), 10)
minFolderSizeString = strconv.FormatUint(modules.SectorSize*contractmanager.MinimumSectorsPerStorageFolder, 10)
maxFolderSizeString = strconv.FormatUint(modules.SectorSize*contractmanager.MaximumSectorsPerStorageFolder, 10)
tooSmallFolderString = strconv.FormatUint(modules.SectorSize*(contractmanager.MinimumSectorsPerStorageFolder-1), 10)
tooLargeFolderString = strconv.FormatUint(modules.SectorSize*(contractmanager.MaximumSectorsPerStorageFolder+1), 10)
mediumSizeFolderString = strconv.FormatUint(modules.SectorSize*contractmanager.MinimumSectorsPerStorageFolder*3, 10)
// Test cases for resizing a host's storage folder.
// Running all the invalid cases before the valid ones simplifies some
......@@ -32,14 +32,14 @@ var (
}{
// invalid sizes
{"", 0, io.EOF},
{"0", 0, storagemanager.ErrSmallStorageFolder},
{tooSmallFolderString, storagemanager.MinimumStorageFolderSize() - 1, storagemanager.ErrSmallStorageFolder},
{tooLargeFolderString, storagemanager.MaximumStorageFolderSize() + 1, storagemanager.ErrLargeStorageFolder},
{"0", 0, contractmanager.ErrSmallStorageFolder},
{tooSmallFolderString, modules.SectorSize * (contractmanager.MinimumSectorsPerStorageFolder - 1), contractmanager.ErrSmallStorageFolder},
{tooLargeFolderString, modules.SectorSize * (contractmanager.MaximumSectorsPerStorageFolder + 1), contractmanager.ErrLargeStorageFolder},
// valid sizes
{minFolderSizeString, storagemanager.MinimumStorageFolderSize(), nil},
{maxFolderSizeString, storagemanager.MaximumStorageFolderSize(), nil},
{mediumSizeFolderString, 3 * storagemanager.MinimumStorageFolderSize(), nil},
{minFolderSizeString, contractmanager.MinimumSectorsPerStorageFolder * modules.SectorSize, nil},
{maxFolderSizeString, contractmanager.MaximumSectorsPerStorageFolder * modules.SectorSize, nil},
{mediumSizeFolderString, 3 * contractmanager.MinimumSectorsPerStorageFolder * modules.SectorSize, nil},
}
)
......@@ -129,15 +129,15 @@ func TestAddFolderNoPath(t *testing.T) {
addValues := url.Values{}
addValues.Set("size", mediumSizeFolderString)
err = st.stdPostAPI("/host/storage/folders/add", addValues)
if err == nil || err.Error() != storagemanager.ErrEmptyPath.Error() {
t.Fatalf("expected error to be %v; got %v", storagemanager.ErrEmptyPath, err)
if err == nil {
t.Fatal(err)
}
// Setting the path to an empty string should trigger the same error.
addValues.Set("path", "")
err = st.stdPostAPI("/host/storage/folders/add", addValues)
if err == nil || err.Error() != storagemanager.ErrEmptyPath.Error() {
t.Fatalf("expected error to be %v; got %v", storagemanager.ErrEmptyPath, err)
if err == nil {
t.Fatal(err)
}
}
......@@ -183,8 +183,8 @@ func TestAddSameFolderTwice(t *testing.T) {
t.Fatal(err)
}
err = st.stdPostAPI("/host/storage/folders/add", addValues)
if err == nil || err.Error() != storagemanager.ErrRepeatFolder.Error() {
t.Fatalf("expected err to be %v, got %v", err, storagemanager.ErrRepeatFolder)
if err == nil || err.Error() != contractmanager.ErrRepeatFolder.Error() {
t.Fatalf("expected err to be %v, got %v", err, contractmanager.ErrRepeatFolder)
}
}
......@@ -226,18 +226,18 @@ func TestResizeEmptyStorageFolder(t *testing.T) {
// Attempting to resize to the same size should return an error.
err = st.stdPostAPI("/host/storage/folders/resize", resizeValues)
if err == nil || err.Error() != storagemanager.ErrNoResize.Error() {
t.Fatalf("expected error %v, got %v", storagemanager.ErrNoResize, err)
if err == nil || err.Error() != contractmanager.ErrNoResize.Error() {
t.Fatalf("expected error %v, got %v", contractmanager.ErrNoResize, err)
}
// Try resizing to a bunch of sizes (invalid ones first, valid ones second).
// This ordering simplifies logic within the for loop.
for _, test := range resizeTests {
for i, test := range resizeTests {
// Attempt to resize the host's storage folder.
resizeValues.Set("newsize", test.sizeString)
err = st.stdPostAPI("/host/storage/folders/resize", resizeValues)
if (err == nil && test.err != nil) || (err != nil && err.Error() != test.err.Error()) {
t.Fatalf("expected error to be %v, got %v", test.err, err)
t.Fatalf("test %v: expected error to be %v, got %v", i, test.err, err)
}
// Find out if the resize call worked as expected.
......@@ -249,7 +249,7 @@ func TestResizeEmptyStorageFolder(t *testing.T) {
if test.err == nil {
// Check that the folder's total capacity has been updated.
if got := sg.Folders[0].Capacity; got != test.size {
t.Fatalf("expected folder to be resized to %v; got %v instead", test.size, got)
t.Fatalf("test %v: expected folder to be resized to %v; got %v instead", i, test.size, got)
}
// Check that the folder's remaining capacity has been updated.
if got := sg.Folders[0].CapacityRemaining; got != test.size {
......@@ -340,8 +340,8 @@ func TestResizeNonemptyStorageFolder(t *testing.T) {
// Attempting to resize to the same size should return an error.
err = st.stdPostAPI("/host/storage/folders/resize", resizeValues)
if err == nil || err.Error() != storagemanager.ErrNoResize.Error() {
t.Fatalf("expected error %v, got %v", storagemanager.ErrNoResize, err)
if err == nil || err.Error() != contractmanager.ErrNoResize.Error() {
t.Fatalf("expected error %v, got %v", contractmanager.ErrNoResize, err)
}
// Try resizing to a bunch of sizes (invalid ones first, valid ones second).
......@@ -547,8 +547,8 @@ func TestRemoveStorageFolderForced(t *testing.T) {
removeValues := url.Values{}
removeValues.Set("path", st.dir)
err = st.stdPostAPI("/host/storage/folders/remove", removeValues)
if err == nil || err.Error() != storagemanager.ErrIncompleteOffload.Error() {
t.Fatalf("expected err to be %v; got %v", storagemanager.ErrIncompleteOffload, err)
if err == nil || err.Error() != contractmanager.ErrPartialRelocation.Error() {
t.Fatalf("expected err to be %v; got %v", contractmanager.ErrPartialRelocation, err)
}
// Forced removal of the folder should succeed, though.
removeValues.Set("force", "true")
......@@ -643,8 +643,8 @@ func TestDeleteNonexistentSector(t *testing.T) {
// Right now, the calls fail for the first reason. This test will report if that behavior changes.
badHash := crypto.HashObject("fake object").String()
err = st.stdPostAPI("/host/storage/sectors/delete/"+badHash, url.Values{})
if err == nil || err.Error() != storagemanager.ErrSectorNotFound.Error() {
t.Fatalf("expected error to be %v; got %v", storagemanager.ErrSectorNotFound, err)
if err == nil || err.Error() != contractmanager.ErrSectorNotFound.Error() {
t.Fatalf("expected error to be %v; got %v", contractmanager.ErrSectorNotFound, err)
}
wrongSize := "wrong size string"
err = st.stdPostAPI("/host/storage/sectors/delete/"+wrongSize, url.Values{})
......
......@@ -87,11 +87,11 @@ var (
panic("unrecognized release constant in host - maximum storage folders")
}()
// maximumSectorsPerStorageFolder sets an upper bound on how large storage
// MaximumSectorsPerStorageFolder sets an upper bound on how large storage
// folders in the host are allowed to be. There is a hard limit at 4
// billion sectors because the sector location map only uses 4 bytes to
// indicate the location of a sector.
maximumSectorsPerStorageFolder = func() uint64 {
MaximumSectorsPerStorageFolder = func() uint64 {
if build.Release == "dev" {
return 1 << 20 // 4 TiB
}
......@@ -104,7 +104,7 @@ var (
panic("unrecognized release constant in host - maximum storage folder size")
}()
// minimumSectorsPerStorageFolder defines the minimum number of sectors
// MinimumSectorsPerStorageFolder defines the minimum number of sectors
// that a storage folder is allowed to have. The minimum has been set as a
// guide to assist with network health, and to help discourage spammy hosts
// with very little storage. Even if the spammy hosts were allowed, they
......@@ -114,9 +114,9 @@ var (
//
// There are plans to continue raising the minimum storage requirements as
// the network gains maturity.
minimumSectorsPerStorageFolder = func() uint64 {
MinimumSectorsPerStorageFolder = func() uint64 {
if build.Release == "dev" {
return 1 << 3 // 32 MiB
return 1 << 6 // 256 MiB
}
if build.Release == "standard" {
// We are at a stage of Sia where we have plenty of hosts. It's
......@@ -126,7 +126,7 @@ var (
return 1 << 15 // 256 GiB
}
if build.Release == "testing" {
return 1 << 3 // 32 KiB
return 1 << 6 // 256 KiB
}
panic("unrecognized release constant in host - minimum storage folder size")
}()
......
......@@ -31,8 +31,8 @@ var (
// the maximum number of virtual sectors for that sector id already exist.
errMaxVirtualSectors = errors.New("sector collides with a physical sector that already has the maximum allowed number of virtual sectors")
// errSectorNotFound is returned when a lookup for a sector fails.
errSectorNotFound = errors.New("could not find the desired sector")
// ErrSectorNotFound is returned when a lookup for a sector fails.
ErrSectorNotFound = errors.New("could not find the desired sector")
)
// sectorLocation indicates the location of a sector on disk.
......@@ -141,11 +141,11 @@ func (cm *ContractManager) ReadSector(root crypto.Hash) ([]byte, error) {
sf, exists2 := cm.storageFolders[sl.storageFolder]
cm.wal.mu.Unlock()
if !exists1 {
return nil, errSectorNotFound
return nil, ErrSectorNotFound
}
if !exists2 {
cm.log.Critical("Unable to load storage folder despite having sector metadata")
return nil, errSectorNotFound
return nil, ErrSectorNotFound
}
// Read the sector.
......
......@@ -223,8 +223,7 @@ func (wal *writeAheadLog) managedDeleteSector(id sectorID) error {
var exists bool
location, exists = wal.cm.sectorLocations[id]
if !exists {
wal.mu.Unlock()
return errSectorNotFound
return ErrSectorNotFound
}
sf, exists = wal.cm.storageFolders[location.storageFolder]
if !exists {
......@@ -281,7 +280,7 @@ func (wal *writeAheadLog) managedRemoveSector(id sectorID) error {
var exists bool
location, exists = wal.cm.sectorLocations[id]
if !exists {
return errSectorNotFound
return ErrSectorNotFound
}
sf, exists = wal.cm.storageFolders[location.storageFolder]
if !exists {
......
......@@ -31,9 +31,9 @@ var (
// reduced in size.
errInsufficientRemainingStorageForShrink = errors.New("not enough storage remaining to support shrinking of disk")
// errLargeStorageFolder is returned if a new storage folder or a resized
// ErrLargeStorageFolder is returned if a new storage folder or a resized
// storage folder would exceed the maximum allowed size.
errLargeStorageFolder = fmt.Errorf("maximum allowed size for a storage folder is %v bytes", maximumSectorsPerStorageFolder*modules.SectorSize)
ErrLargeStorageFolder = fmt.Errorf("maximum allowed size for a storage folder is %v bytes", MaximumSectorsPerStorageFolder*modules.SectorSize)
// errMaxStorageFolders indicates that the limit on the number of allowed
// storage folders has been reached.
......@@ -45,18 +45,18 @@ var (
// advance that there are no free sectors.
errNoFreeSectors = errors.New("could not find a free sector in the usage array")
// errNoResize is returned if a new size is provided for a storage folder
// ErrNoResize is returned if a new size is provided for a storage folder
// that is the same as the current size of the storage folder.
errNoResize = errors.New("storage folder selected for resize, but new size is same as current size")
ErrNoResize = errors.New("storage folder selected for resize, but new size is same as current size")
// errRepeatFolder is returned if a storage folder is added which links to
// ErrRepeatFolder is returned if a storage folder is added which links to
// a path that is already in use by another storage folder. Only exact path
// matches will trigger the error.
errRepeatFolder = errors.New("selected path is already in use as a storage folder, please use 'resize'")
ErrRepeatFolder = errors.New("selected path is already in use as a storage folder, please use 'resize'")
// errSmallStorageFolder is returned if a new storage folder is not large
// ErrSmallStorageFolder is returned if a new storage folder is not large
// enough to meet the requirements for the minimum storage folder size.
errSmallStorageFolder = fmt.Errorf("minimum allowed size for a storage folder is %v bytes", minimumSectorsPerStorageFolder*modules.SectorSize)
ErrSmallStorageFolder = fmt.Errorf("minimum allowed size for a storage folder is %v bytes", MinimumSectorsPerStorageFolder*modules.SectorSize)
// errStorageFolderGranularity is returned if a call to AddStorageFolder
// tries to use a storage folder size that does not evenly fit into a
......@@ -310,11 +310,18 @@ func (cm *ContractManager) ResizeStorageFolder(index uint16, newSize uint64) err
return errStorageFolderNotFound
}
oldSize := uint64(len(sf.usage)) * storageFolderGranularity
if newSize/modules.SectorSize < MinimumSectorsPerStorageFolder {
return ErrSmallStorageFolder
}
if newSize/modules.SectorSize > MaximumSectorsPerStorageFolder {
return ErrLargeStorageFolder
}
oldSize := uint64(len(sf.usage)) * storageFolderGranularity * modules.SectorSize
if oldSize == newSize {
return errNoResize
return ErrNoResize
}
newSectorCount := uint32(newSize / storageFolderGranularity)
newSectorCount := uint32(newSize / modules.SectorSize)
if oldSize > newSize {
return cm.wal.shrinkStorageFolder(index, newSectorCount, true)
}
......
......@@ -107,7 +107,7 @@ func (wal *writeAheadLog) managedAddStorageFolder(sf *storageFolder) error {
// removed, however we refuse to add a replacement storage folder
// until the existing one has been removed entirely.
if sf.path == csf.path {
return errRepeatFolder
return ErrRepeatFolder
}
}
......@@ -319,11 +319,11 @@ func (cm *ContractManager) AddStorageFolder(path string, size uint64) error {
// Check that the storage folder being added meets the size requirements.
sectors := size / modules.SectorSize
if sectors > maximumSectorsPerStorageFolder {
return errLargeStorageFolder
if sectors > MaximumSectorsPerStorageFolder {
return ErrLargeStorageFolder
}
if sectors < minimumSectorsPerStorageFolder {
return errSmallStorageFolder
if sectors < MinimumSectorsPerStorageFolder {
return ErrSmallStorageFolder
}
if (size/modules.SectorSize)%storageFolderGranularity != 0 {
return errStorageFolderGranularity
......
......@@ -467,7 +467,7 @@ func TestAddStorageFolderDoubleAdd(t *testing.T) {
t.Fatal(err)
}
err = cmt.cm.AddStorageFolder(storageFolderOne, sfSize*2)
if err != errRepeatFolder {
if err != ErrRepeatFolder {
t.Fatal(err)
}
......@@ -561,7 +561,7 @@ func TestAddStorageFolderDoubleAddNoCommit(t *testing.T) {
t.Fatal(err)
}
err = cmt.cm.AddStorageFolder(storageFolderOne, sfSize*2)
if err != errRepeatFolder {
if err != ErrRepeatFolder {
t.Fatal(err)
}
......
......@@ -21,10 +21,10 @@ type (
)
var (
// errPartialRelocation is returned during an operation attempting to clear
// ErrPartialRelocation is returned during an operation attempting to clear
// out the sectors in a storage folder if errors prevented one or more of
// the sectors from being properly migrated to a new storage folder.
errPartialRelocation = errors.New("unable to migrate all sectors")
ErrPartialRelocation = errors.New("unable to migrate all sectors")
)
// TODO: Sector operations must return an error if they are requested on a
......@@ -180,7 +180,9 @@ func (wal *writeAheadLog) managedMoveSector(id sectorID) error {
wal.mu.Unlock()
return nil
}()
if err != nil {
if err == errInsufficientStorageForSector {
return err
} else if err != nil {
// Try the next storage folder.
storageFolders = append(storageFolders[:storageFolderIndex], storageFolders[storageFolderIndex+1:]...)
continue
......@@ -261,7 +263,7 @@ func (wal *writeAheadLog) managedEmptyStorageFolder(sfIndex uint16, startingPoin
// Return errPartialRelocation if not every sector was migrated out
// successfully.
if errCount > 0 {
return errCount, errPartialRelocation
return errCount, ErrPartialRelocation
}
return 0, nil
}
......@@ -395,6 +397,7 @@ func (wal *writeAheadLog) shrinkStorageFolder(index uint16, newSectorCount uint3
}},
})
syncChan = wal.syncChan
sf.usage = sf.usage[:newSectorCount/storageFolderGranularity]
wal.mu.Unlock()
// Wait until the shrink action has been synchronized.
......
......@@ -130,9 +130,9 @@ func (wal *writeAheadLog) growStorageFolder(index uint16, newSectorCount uint32)
currentHousingSize := int64(len(sf.usage)) * int64(modules.SectorSize) * storageFolderGranularity
currentMetadataSize := int64(len(sf.usage)) * sectorMetadataDiskSize * storageFolderGranularity
newHousingSize := int64(newSectorCount) * int64(modules.SectorSize)
newMetadataSize := int64(newSectorCount) * storageFolderGranularity
newMetadataSize := int64(newSectorCount) * sectorMetadataDiskSize
if newHousingSize <= currentHousingSize || newMetadataSize <= currentMetadataSize {
wal.cm.log.Critical("growStorageFolder called when the storage folder is not increasing in size")
wal.cm.log.Critical("growStorageFolder called when the storage folder is not increasing in size", newHousingSize, currentHousingSize, newMetadataSize, currentMetadataSize)
return errors.New("unable to make the requested change, please notify the devs that there is a bug")
}
housingWriteSize := newHousingSize - currentHousingSize
......
......@@ -130,13 +130,20 @@ func (wal *writeAheadLog) commit() {
// Sync all open, non-WAL files on the host.
wal.syncResources()
// Perform any cleanup actions on the updates.
for _, sc := range wal.uncommittedChanges {
for _, sfe := range sc.StorageFolderExtensions {
wal.commitStorageFolderExtension(sfe)
}
// TODO: Virtual sector handling here.
}
// Extract any unfinished long-running jobs from the list of WAL items.
unfinishedAdditions := findUnfinishedStorageFolderAdditions(wal.uncommittedChanges)
unfinishedExtensions := findUnfinishedStorageFolderExtensions(wal.uncommittedChanges)
// Save the set of uncommitted changes to a new variable and then clear the
// WAL variable.
uncommittedChanges := wal.uncommittedChanges
// Clear the set of uncommitted changes.
wal.uncommittedChanges = nil
// Begin writing to the settings file.
......@@ -161,19 +168,6 @@ func (wal *writeAheadLog) commit() {
}
}()
// Perform any cleanup actions on the updates.
wg.Add(1)
go func() {
defer wg.Done()
for _, sc := range uncommittedChanges {
for _, sfe := range sc.StorageFolderExtensions {
wal.commitStorageFolderExtension(sfe)
}
// TODO: Virtual sector handling here.
}
}()
// Begin writing new changes to the WAL.
wg.Add(1)
go func() {
......
......@@ -196,7 +196,7 @@ func newMockHostTester(d dependencies, name string) (*hostTester, error) {
if err != nil {
return nil, err
}
err = ht.host.AddStorageFolder(storageFolderOne, modules.SectorSize*8)
err = ht.host.AddStorageFolder(storageFolderOne, modules.SectorSize*64)
if err != nil {
return nil, err
}
......@@ -205,7 +205,7 @@ func newMockHostTester(d dependencies, name string) (*hostTester, error) {
if err != nil {
return nil, err
}
err = ht.host.AddStorageFolder(storageFolderTwo, modules.SectorSize*8*2)
err = ht.host.AddStorageFolder(storageFolderTwo, modules.SectorSize*64*2)
if err != nil {
return nil, err
}
......
package storagemanager
import (
"errors"
)
// consistency.go contains a bunch of consistency checks for the host. Because
// a lot of the consistency is checking that different parts of the host match
// up, it was decided that all consistency checking should go into a single
// file. As an example, the state of the storage obligations sets the standard
// for what it means for the storage folders to be consistent. And the storage
// folders set the standard for what it means for the sectors to be consistent.
//
// Consistency checks should be run infrequently, and should take only a minute
// or two to complete, even for a host that has millions of sectors. This means
// that certain consistency checks are off-limits.
// TODO: Check that 'Path' and the symlinked path point to the same place.
// TODO: Consistency checks should be accompanied by repair tools. Perhaps that
// means instead of calling build.Critical... it just returns some sort of
// diagnostic?
var (
// errStorageFolderMaxSizeExceeded is returned when a storage folder is
// found to have exceeded the maximum set by the build constants.
errStorageFolderMaxSizeExceeded = errors.New("storage folder has exceeded the maximum allowed size")
// errStorageFolderMinSizeViolated is returned when a storage folder has a
// size which is smaller than the minimum set by the build constants.
errStorageFolderMinSizeViolated = errors.New("storage folder has less storage than the minimum allowed")
// errStorageFolderSizeRemainingDivergence is returned when a storage
// folder has an amount of storage remaining that indicates an impossible
// state, such as having more storage remaining than total storage
// available.
errStorageFolderSizeRemainingDivergence = errors.New("storage folder has an impossible size remaining value")
// errStorageFolderInvalidUIDLen is returned when a storage folder has a
// UID which has the wrong length.
errStorageFolderInvalidUIDLen = errors.New("storage folder has a UID with an invalid length, often indicating the wrong build of 'siad' is being used")
// errStorageFolderDuplicateUID is returned when a storage folder has a UID
// which is known to already be owned by a different storage folder.
errStorageFolderDuplicateUID = errors.New("storage folder has a UID which is already owned by another storage folder")
)
// storageFolderSizeConsistency checks that all of the storage folders have
// sane sizes.
func (sm *StorageManager) storageFolderSizeConsistency() error {
knownUIDs := make(map[string]int)
for i, sf := range sm.storageFolders {
// The size of a storage folder should be between the minimum and the
// maximum allowed size.
if sf.Size > maximumStorageFolderSize {
sm.log.Critical("storage folder", i, "exceeds the maximum allowed storage folder size")
return errStorageFolderMaxSizeExceeded
}
if sf.Size < minimumStorageFolderSize {
sm.log.Critical("storage folder", i, "has less than the minimum allowed storage folder size")
return errStorageFolderMinSizeViolated
}
// The amount of storage remaining should not be greater than the
// folder size.
if sf.SizeRemaining > sf.Size {
sm.log.Critical("storage folder", i, "has more storage remaining than it has storage total")
return errStorageFolderSizeRemainingDivergence
}
// The UID has a fixed size.
if len(sf.UID) != storageFolderUIDSize {
sm.log.Critical("storage folder", i, "has an ID which is not valid")
return errStorageFolderInvalidUIDLen
}
// Check that the storage folder UID is not conflicting with any other
// known storage folder UID.
conflict, exists := knownUIDs[sf.uidString()]
if exists {
sm.log.Critical("storage folder", i, "has a duplicate UID, conflicting with storage folder", conflict)
return errStorageFolderDuplicateUID
}
// Add this storage folder's UID to the set of known UIDs.
knownUIDs[sf.uidString()] = i
}
return nil
}
package storagemanager
import (
"github.com/NebulousLabs/Sia/build"
)
const (
// maximumStorageFolders indicates the maximum number of storage folders
// that the host allows. Some operations, such as creating a new storage
// folder, take longer if there are more storage folders. Static RAM usage
// also increases as the number of storage folders increase. For this
// reason, a limit on the maximum number of storage folders has been set.
maximumStorageFolders = 100
)
var (
// maximumStorageFolderSize sets an upper bound on how large storage
// folders in the host are allowed to be. It makes sure that inputs and
// constructions are sane. While it's conceivable that someone could create
// a rig with a single logical storage folder greater than 128 TiB in size
// in production, it's probably not a great idea, especially when you are
// allowed to use many storage folders. All told, a single host on today's
// constants can support up to ~10 PB of storage.
maximumStorageFolderSize = func() uint64 {
if build.Release == "dev" {
return 1 << 40 // 1 TiB
}
if build.Release == "standard" {
return 1 << 50 // 1 PiB
}
if build.Release == "testing" {
return 1 << 20 // 1 MiB
}
panic("unrecognized release constant in host - maximum storage folder size")
}()
// maximumVirtualSectors defines the maximum number of virtual sectors that
// can be tied to each physical sector.
maximumVirtualSectors = func() int {
if build.Release == "dev" {
// The testing value is at 35 to provide flexibility. The
// development value is at 5 because hitting the virtual sector
// limit in a sane development environment is more difficult than
// hitting the virtual sector limit in a controlled testing
// environment (dev environment doesn't have access to private
// methods such as 'addSector'.
return 5
}
if build.Release == "standard" {
// Each virtual sector adds about 8 bytes of load to the host
// persistence structures, and additionally adds 8 bytes of load
// when reading or modifying a sector. Though a few virtual sectors
// with 10e3 or even 100e3 virtual sectors would not be too
// detrimental to the host, tens of thousands of physical sectors
// that each have ten thousand virtual sectors could pose a problem
// for the host. In most situations, a renter will not need more 2
// or 3 virtual sectors when manipulating data, so 250 is generous
// as long as the renter is properly encrypting data. 250 is
// unlikely to cause the host problems, even if an attacker is
// creating hundreds of thousands of physical sectors (an expensive
// action!) each with 250 virtual sectors.
return 250
}
if build.Release == "testing" {
return 35
}
panic("unrecognized release constant in host - maximum virtual sector size")
}()
// minimumStorageFolderSize defines the smallest size that a storage folder
// is allowed to be. The new design of the storage folder structure means
// that this limit is not as relevant as it was originally, but hosts with
// little storage capacity are not very useful to the network, and can
// actually frustrate price planning. 32 GB has been chosen as a minimum
// for the early days of the network, to allow people to experiment in the
// beta, but in the future I think something like 256 GB would be much more
// appropriate.
minimumStorageFolderSize = func() uint64 {
if build.Release == "dev" {
return 1 << 25 // 32 MiB
}
if build.Release == "standard" {
return 1 << 35 // 32 GiB
}
if build.Release == "testing" {
return 1 << 15 // 32 KiB
}
panic("unrecognized release constant in host - minimum storage folder size")
}()
// storageFolderUIDSize determines the number of bytes used to determine
// the storage folder UID. Production and development environments use 4
// bytes to minimize the possibility of accidental collisions, and testing
// environments use 1 byte so that collisions can be forced while using the
// live code.
storageFolderUIDSize = func() int {
if build.Release == "dev" {
return 2
}
if build.Release == "standard" {
return 4
}
if build.Release == "testing" {
return 1
}
panic("unrecognized release constant in host - storageFolderUIDSize")
}()
// bucketSectorUsage maps sector IDs to the number of times they are used
// in file contracts. If all data is correctly encrypted using a unique
// seed, each sector will be in use exactly one time. The host however
// cannot control this, and a user may upload unencrypted data or
// intentionally upload colliding sectors as a means of attack. The host
// can only delete a sector when it is in use zero times. The number of
// times a sector is in use is encoded as a big endian uint64.
bucketSectorUsage = []byte("BucketSectorUsage")
)
// MaximumStorageFolderSize provides the maximumStorageFolderSize value to
// other modules for testing purposes.
func MaximumStorageFolderSize() uint64 {
return maximumStorageFolderSize
}
// MinimumStorageFolderSize provides the minimumStorageFolderSize value to
// other modules for testing purposes.
func MinimumStorageFolderSize() uint64 {
return minimumStorageFolderSize
}
package storagemanager
import (
"crypto/rand"
"errors"
"io/ioutil"
"os"
"strings"
"github.com/NebulousLabs/Sia/persist"
)
// Fake errors that get returned when a simulated failure of a dependency is
// desired for testing.
var (
mockErrListen = errors.New("simulated Listen failure")
mockErrLoadFile = errors.New("simulated LoadFile failure")
mockErrMkdirAll = errors.New("simulated MkdirAll failure")
mockErrNewLogger = errors.New("simulated NewLogger failure")