Commit 50f857d4 authored by Christopher Schinnerl's avatar Christopher Schinnerl

optimize file endpoint

parent a5e070ec
package renter
import (
"math"
"os"
"path/filepath"
"strings"
"sync"
"gitlab.com/NebulousLabs/Sia/modules/renter/siadir"
......@@ -85,40 +82,7 @@ func (r *Renter) FileList() ([]modules.FileInfo, error) {
}
defer r.tg.Done()
offlineMap, goodForRenewMap, contractsMap := r.managedContractUtilityMaps()
fileList := []modules.FileInfo{}
err := filepath.Walk(r.staticFilesDir, func(path string, info os.FileInfo, err error) error {
// This error is non-nil if filepath.Walk couldn't stat a file or
// folder. We simply ignore missing files.
if os.IsNotExist(err) {
return nil
}
if err != nil {
return err
}
// Skip folders and non-sia files.
if info.IsDir() || filepath.Ext(path) != modules.SiaFileExtension {
return nil
}
// Load the Siafile.
str := strings.TrimSuffix(strings.TrimPrefix(path, r.staticFilesDir), modules.SiaFileExtension)
siaPath, err := modules.NewSiaPath(str)
if err != nil {
return err
}
file, err := r.fileInfo(siaPath, offlineMap, goodForRenewMap, contractsMap)
if os.IsNotExist(err) || err == siafile.ErrUnknownPath {
return nil
}
if err != nil {
return err
}
fileList = append(fileList, file)
return nil
})
return fileList, err
return r.staticFileSet.FileList(offlineMap, goodForRenewMap, contractsMap)
}
// File returns file from siaPath queried by user.
......@@ -129,7 +93,7 @@ func (r *Renter) File(siaPath modules.SiaPath) (modules.FileInfo, error) {
}
defer r.tg.Done()
offline, goodForRenew, contracts := r.managedContractUtilityMaps()
return r.fileInfo(siaPath, offline, goodForRenew, contracts)
return r.staticFileSet.FileInfo(siaPath, offline, goodForRenew, contracts)
}
// RenameFile takes an existing file and changes the nickname. The original
......@@ -185,51 +149,6 @@ func (r *Renter) SetFileStuck(siaPath modules.SiaPath, stuck bool) error {
return entry.SetAllStuck(stuck)
}
// fileInfo returns information on a siafile. As a performance optimization, the
// fileInfo takes the maps returned by renter.managedContractUtilityMaps as
// many files at once.
func (r *Renter) fileInfo(siaPath modules.SiaPath, offline map[string]bool, goodForRenew map[string]bool, contracts map[string]modules.RenterContract) (modules.FileInfo, error) {
// Get the file's metadata and its contracts
md, err := r.staticFileSet.Metadata(siaPath)
if err != nil {
return modules.FileInfo{}, err
}
// Build the FileInfo
var onDisk bool
localPath := md.LocalPath
if localPath != "" {
_, err = os.Stat(localPath)
onDisk = err == nil
}
fileInfo := modules.FileInfo{
AccessTime: md.AccessTime,
Available: md.CachedRedundancy >= 1,
ChangeTime: md.ChangeTime,
CipherType: md.StaticMasterKeyType.String(),
CreateTime: md.CreateTime,
Expiration: md.CachedExpiration,
Filesize: uint64(md.FileSize),
Health: md.CachedHealth,
LocalPath: localPath,
MaxHealth: math.Max(md.CachedHealth, md.CachedStuckHealth),
MaxHealthPercent: md.HealthPercentage(),
ModTime: md.ModTime,
NumStuckChunks: md.NumStuckChunks,
OnDisk: onDisk,
Recoverable: onDisk || md.CachedRedundancy >= 1,
Redundancy: md.CachedRedundancy,
Renewing: true,
SiaPath: siaPath,
Stuck: md.NumStuckChunks > 0,
StuckHealth: md.CachedStuckHealth,
UploadedBytes: md.CachedUploadedBytes,
UploadProgress: md.CachedUploadProgress,
}
return fileInfo, nil
}
// fileToSiaFile converts a legacy file to a SiaFile. Fields that can't be
// populated using the legacy file remain blank.
func (r *Renter) fileToSiaFile(f *file, repairPath string, oldContracts []modules.RenterContract) (*siafile.SiaFileSetEntry, error) {
......
......@@ -168,7 +168,7 @@ func New(siaPath modules.SiaPath, siaFilePath, source string, wal *writeaheadlog
file.chunks[i].Pieces = make([][]piece, erasureCode.NumPieces())
}
// Init cached fields for 0-Byte files.
if file.staticMetadata.StaticFileSize == 0 {
if file.staticMetadata.FileSize == 0 {
file.staticMetadata.CachedHealth = 0
file.staticMetadata.CachedStuckHealth = 0
file.staticMetadata.CachedRedundancy = float64(erasureCode.NumPieces()) / float64(erasureCode.MinPieces())
......@@ -903,7 +903,7 @@ func (sf *SiaFile) updateUploadProgressAndBytes() {
sf.staticMetadata.CachedUploadProgress = 100
return
}
desired := uint64(len(sf.staticChunks)) * modules.SectorSize * uint64(sf.staticMetadata.staticErasureCode.NumPieces())
desired := uint64(len(sf.chunks)) * modules.SectorSize * uint64(sf.staticMetadata.staticErasureCode.NumPieces())
// Update cache.
sf.staticMetadata.CachedUploadProgress = math.Min(100*(float64(uploaded)/float64(desired)), 100)
}
......
......@@ -4,10 +4,13 @@ import (
"fmt"
"math"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
"time"
"github.com/karrick/godirwalk"
"gitlab.com/NebulousLabs/Sia/build"
"gitlab.com/NebulousLabs/Sia/crypto"
"gitlab.com/NebulousLabs/Sia/modules"
......@@ -202,6 +205,51 @@ func (sfs *SiaFileSet) exists(siaPath modules.SiaPath) bool {
return !os.IsNotExist(err)
}
// readLockFileInfo returns information on a siafile. As a performance optimization, the
// fileInfo takes the maps returned by renter.managedContractUtilityMaps as
// many files at once.
func (sfs *SiaFileSet) readLockFileInfo(siaPath modules.SiaPath, offline map[string]bool, goodForRenew map[string]bool, contracts map[string]modules.RenterContract) (modules.FileInfo, error) {
// Get the file's metadata and its contracts
md, err := sfs.readLockMetadata(siaPath)
if err != nil {
return modules.FileInfo{}, err
}
// Build the FileInfo
var onDisk bool
localPath := md.LocalPath
if localPath != "" {
_, err = os.Stat(localPath)
onDisk = err == nil
}
fileInfo := modules.FileInfo{
AccessTime: md.AccessTime,
Available: md.CachedRedundancy >= 1,
ChangeTime: md.ChangeTime,
CipherType: md.StaticMasterKeyType.String(),
CreateTime: md.CreateTime,
Expiration: md.CachedExpiration,
Filesize: uint64(md.FileSize),
Health: md.CachedHealth,
LocalPath: localPath,
MaxHealth: math.Max(md.CachedHealth, md.CachedStuckHealth),
MaxHealthPercent: md.HealthPercentage(),
ModTime: md.ModTime,
NumStuckChunks: md.NumStuckChunks,
OnDisk: onDisk,
Recoverable: onDisk || md.CachedRedundancy >= 1,
Redundancy: md.CachedRedundancy,
Renewing: true,
SiaPath: siaPath,
Stuck: md.NumStuckChunks > 0,
StuckHealth: md.CachedStuckHealth,
UploadedBytes: md.CachedUploadedBytes,
UploadProgress: md.CachedUploadProgress,
}
return fileInfo, nil
}
// newSiaFileSetEntry initializes and returns a siaFileSetEntry
func (sfs *SiaFileSet) newSiaFileSetEntry(sf *SiaFile) (*siaFileSetEntry, error) {
threads := make(map[uint64]threadInfo)
......@@ -260,8 +308,12 @@ func (sfs *SiaFileSet) open(siaPath modules.SiaPath) (*SiaFileSetEntry, error) {
}, nil
}
// metadata returns the metadata of the SiaFile at siaPath.
func (sfs *SiaFileSet) metadata(siaPath modules.SiaPath) (Metadata, error) {
// readLockMetadata returns the metadata of the SiaFile at siaPath. NOTE: The
// 'readLock' prefix in this case is used to indicate that it's save to call
// this method with other 'readLock' methods without locking since is doesn't
// write to any fields. This guarantee can be made by locking sfs.mu and then
// spawning multiple threads which call 'readLock' methods in parallel.
func (sfs *SiaFileSet) readLockMetadata(siaPath modules.SiaPath) (Metadata, error) {
var entry *siaFileSetEntry
entry, _, exists := sfs.siaPathToEntryAndUID(siaPath)
if exists {
......@@ -310,6 +362,69 @@ func (sfs *SiaFileSet) Exists(siaPath modules.SiaPath) bool {
return sfs.exists(siaPath)
}
// FileInfo returns information on a siafile. As a performance optimization, the
// fileInfo takes the maps returned by renter.managedContractUtilityMaps as
// many files at once.
func (sfs *SiaFileSet) FileInfo(siaPath modules.SiaPath, offline map[string]bool, goodForRenew map[string]bool, contracts map[string]modules.RenterContract) (modules.FileInfo, error) {
sfs.mu.Lock()
defer sfs.mu.Unlock()
return sfs.readLockFileInfo(siaPath, offline, goodForRenew, contracts)
}
// FileList returns all of the files that the renter has.
func (sfs *SiaFileSet) FileList(offlineMap map[string]bool, goodForRenewMap map[string]bool, contractsMap map[string]modules.RenterContract) ([]modules.FileInfo, error) {
// Guarantee that no other thread is writing to sfs.
sfs.mu.Lock()
defer sfs.mu.Unlock()
// Declare a worker method to spawn workers which keep loading files from disk
// until the loadChan is closed.
fileList := []modules.FileInfo{}
var fileListMu sync.Mutex
loadChan := make(chan string)
var wg sync.WaitGroup
worker := func() {
for path := range loadChan {
// Load the Siafile.
str := strings.TrimSuffix(strings.TrimPrefix(path, sfs.siaFileDir), modules.SiaFileExtension)
siaPath, err := modules.NewSiaPath(str)
if err != nil {
continue
}
file, err := sfs.readLockFileInfo(siaPath, offlineMap, goodForRenewMap, contractsMap)
if os.IsNotExist(err) || err == ErrUnknownPath {
continue
}
if err != nil {
continue
}
fileListMu.Lock()
fileList = append(fileList, file)
fileListMu.Unlock()
}
wg.Done()
}
// spin up 20 threads
for i := 0; i < 20; i++ {
wg.Add(1)
go worker()
}
// Walk over the whole tree.
err := godirwalk.Walk(sfs.siaFileDir, &godirwalk.Options{
Unsorted: true,
Callback: func(path string, info *godirwalk.Dirent) error {
// Skip folders and non-sia files.
if info.IsDir() || filepath.Ext(path) != modules.SiaFileExtension {
return nil
}
loadChan <- path
return nil
},
})
close(loadChan)
wg.Wait()
return fileList, err
}
// NewSiaFile create a new SiaFile, adds it to the SiaFileSet, adds the thread
// to the threadMap, and returns the SiaFileSetEntry. Since this method returns
// the SiaFileSetEntry, wherever NewSiaFile is called there should be a Close
......@@ -354,7 +469,7 @@ func (sfs *SiaFileSet) Open(siaPath modules.SiaPath) (*SiaFileSetEntry, error) {
func (sfs *SiaFileSet) Metadata(siaPath modules.SiaPath) (Metadata, error) {
sfs.mu.Lock()
defer sfs.mu.Unlock()
return sfs.metadata(siaPath)
return sfs.readLockMetadata(siaPath)
}
// Rename will move a siafile from one path to a new path. Existing entries that
......
......@@ -409,14 +409,14 @@ func (c *Client) RenterUploadDefaultPost(path string, siaPath modules.SiaPath) (
// RenterUploadStreamPost uploads data using a stream. It will return a
// io.WriteCloser that can be used to write the data to the request body. The
// writer needs to be closed when done for the whole file to be uploaded.
func (c *Client) RenterUploadStreamPost(r io.Reader, siaPath string, dataPieces, parityPieces uint64, force bool) error {
siaPath = escapeSiaPath(trimSiaPath(siaPath))
func (c *Client) RenterUploadStreamPost(r io.Reader, siaPath modules.SiaPath, dataPieces, parityPieces uint64, force bool) error {
sp := escapeSiaPath(siaPath)
values := url.Values{}
values.Set("datapieces", strconv.FormatUint(dataPieces, 10))
values.Set("paritypieces", strconv.FormatUint(parityPieces, 10))
values.Set("force", strconv.FormatBool(force))
values.Set("stream", strconv.FormatBool(true))
_, err := c.postRawResponse(fmt.Sprintf("/renter/uploadstream/%s?%s", siaPath, values.Encode()), r)
_, err := c.postRawResponse(fmt.Sprintf("/renter/uploadstream/%s?%s", sp, values.Encode()), r)
return err
}
......
This diff is collapsed.
......@@ -311,9 +311,12 @@ func testUploadStreaming(t *testing.T, tg *siatest.TestGroup) {
d := bytes.NewReader(data)
// Upload the data.
siaPath := "/foo"
siaPath, err := modules.NewSiaPath("/foo")
if err != nil {
t.Fatal(err)
}
r := tg.Renters()[0]
err := r.RenterUploadStreamPost(d, siaPath, 1, uint64(len(tg.Hosts())-1), false)
err = r.RenterUploadStreamPost(d, siaPath, 1, uint64(len(tg.Hosts())-1), false)
if err != nil {
t.Fatal(err)
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment