Commit 317c6916 authored by Christopher Schinnerl's avatar Christopher Schinnerl

remove the siapath field from the siafile

parent 9b2859d9
Pipeline #52440938 failed with stages
in 80 minutes and 44 seconds
......@@ -347,7 +347,7 @@ func (r *Renter) managedDownload(p modules.RenterDownloadParameters) (*download,
destination: dw,
destinationType: destinationType,
destinationString: p.Destination,
file: entry.SiaFile.Snapshot(),
file: entry.Snapshot(),
latencyTarget: 25e3 * time.Millisecond, // TODO: high default until full latency support is added.
length: p.Length,
......
......@@ -592,7 +592,7 @@ func TestRenterDeleteFile(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = entry2.Rename("1", filepath.Join(rt.renter.staticFilesDir, "1"+siafile.ShareExtension)) // set name to "1"
err = entry2.Rename(filepath.Join(rt.renter.staticFilesDir, "1"+siafile.ShareExtension)) // set name to "1"
if err != nil {
t.Fatal(err)
}
......@@ -708,7 +708,7 @@ func TestRenterRenameFile(t *testing.T) {
// Rename a file that does exist.
entry, _ := rt.renter.newRenterTestFile()
err = entry.Rename("1", filepath.Join(rt.renter.staticFilesDir, "1"+siafile.ShareExtension))
err = entry.Rename(filepath.Join(rt.renter.staticFilesDir, "1"+siafile.ShareExtension))
if err != nil {
t.Fatal(err)
}
......@@ -742,7 +742,7 @@ func TestRenterRenameFile(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = entry2.Rename("1", filepath.Join(rt.renter.staticFilesDir, "1"+siafile.ShareExtension))
err = entry2.Rename(filepath.Join(rt.renter.staticFilesDir, "1"+siafile.ShareExtension))
if err != nil {
t.Fatal(err)
}
......
......@@ -6,6 +6,7 @@ import (
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"gitlab.com/NebulousLabs/Sia/modules"
......@@ -36,8 +37,8 @@ func equalFiles(f1, f2 *siafile.SiaFile) error {
if f1 == nil || f2 == nil {
return fmt.Errorf("one or both files are nil")
}
if f1.SiaPath() != f2.SiaPath() {
return fmt.Errorf("names do not match: %v %v", f1.SiaPath(), f2.SiaPath())
if f1.SiaFilePath() != f2.SiaFilePath() {
return fmt.Errorf("names do not match: %v %v", f1.SiaFilePath(), f2.SiaFilePath())
}
if f1.Size() != f2.Size() {
return fmt.Errorf("sizes do not match: %v %v", f1.Size(), f2.Size())
......@@ -166,17 +167,17 @@ func TestRenterPaths(t *testing.T) {
if err != nil {
t.Fatal(err)
}
f1.Rename(siaPath1, filepath.Join(rt.renter.staticFilesDir, siaPath1+siafile.ShareExtension))
f1.Rename(filepath.Join(rt.renter.staticFilesDir, siaPath1+siafile.ShareExtension))
f2, err := newTestingFile()
if err != nil {
t.Fatal(err)
}
f2.Rename(siaPath2, filepath.Join(rt.renter.staticFilesDir, siaPath2+siafile.ShareExtension))
f2.Rename(filepath.Join(rt.renter.staticFilesDir, siaPath2+siafile.ShareExtension))
f3, err := newTestingFile()
if err != nil {
t.Fatal(err)
}
f3.Rename(siaPath3, filepath.Join(rt.renter.staticFilesDir, siaPath3+siafile.ShareExtension))
f3.Rename(filepath.Join(rt.renter.staticFilesDir, siaPath3+siafile.ShareExtension))
// Restart the renter to re-do the init cycle.
err = rt.renter.Close()
......@@ -225,7 +226,16 @@ func TestRenterPaths(t *testing.T) {
return nil
})
// walk will descend into foo/bar/, reading baz, bar, and finally foo
expWalkStr := (f3.SiaPath() + ".sia") + (f2.SiaPath() + ".sia") + (f1.SiaPath() + ".sia")
sp3 := strings.TrimPrefix(f3.SiaFilePath(), rt.renter.staticFilesDir)
sp2 := strings.TrimPrefix(f2.SiaFilePath(), rt.renter.staticFilesDir)
sp1 := strings.TrimPrefix(f1.SiaFilePath(), rt.renter.staticFilesDir)
sp3 = strings.TrimSuffix(sp3, siafile.ShareExtension)
sp2 = strings.TrimSuffix(sp2, siafile.ShareExtension)
sp1 = strings.TrimSuffix(sp1, siafile.ShareExtension)
sp3 = strings.TrimPrefix(sp3, "/")
sp2 = strings.TrimPrefix(sp2, "/")
sp1 = strings.TrimPrefix(sp1, "/")
expWalkStr := (sp3 + ".sia") + (sp2 + ".sia") + (sp1 + ".sia")
if filepath.ToSlash(walkStr) != expWalkStr {
t.Fatalf("Bad walk string: expected %v, got %v", expWalkStr, walkStr)
}
......
......@@ -75,7 +75,7 @@ OUTER:
if fastrand.Intn(100) < 80 {
spk := hostkeys[fastrand.Intn(len(hostkeys))]
offset := uint64(fastrand.Intn(int(sf.staticMetadata.StaticFileSize)))
chunkIndex, _ := sf.Snapshot().ChunkIndexByOffset(offset)
chunkIndex, _ := sf.ChunkIndexByOffset(offset)
pieceIndex := uint64(fastrand.Intn(sf.staticMetadata.staticErasureCode.NumPieces()))
if err := sf.AddPiece(spk, chunkIndex, pieceIndex, crypto.Hash{}); err != nil {
if errors.Contains(err, errDiskFault) {
......
......@@ -19,7 +19,6 @@ type (
StaticFileSize int64 `json:"filesize"` // total size of the file
StaticPieceSize uint64 `json:"piecesize"` // size of a single piece of the file
LocalPath string `json:"localpath"` // file to the local copy of the file used for repairing
SiaPath string `json:"siapath"` // the path of the file on the Sia network
// fields for encryption
StaticMasterKey []byte `json:"masterkey"` // masterkey used to encrypt pieces
......@@ -146,17 +145,6 @@ func (sf *SiaFile) ChunkSize() uint64 {
return sf.staticChunkSize()
}
// DirSiaPath returns the SiaPath of the directory that the SiaFile is in
func (sf *SiaFile) DirSiaPath() string {
sf.mu.Lock()
defer sf.mu.Unlock()
dirSiaPath := filepath.Dir(sf.staticMetadata.SiaPath)
if dirSiaPath == "." {
dirSiaPath = ""
}
return dirSiaPath
}
// LastHealthCheckTime returns the LastHealthCheckTime timestamp of the file
func (sf *SiaFile) LastHealthCheckTime() time.Time {
sf.mu.Lock()
......@@ -218,7 +206,7 @@ func (sf *SiaFile) RecentRepairTime() time.Time {
}
// Rename changes the name of the file to a new one.
func (sf *SiaFile) Rename(newSiaPath, newSiaFilePath string) error {
func (sf *SiaFile) Rename(newSiaFilePath string) error {
sf.mu.Lock()
defer sf.mu.Unlock()
// Create path to renamed location.
......@@ -233,7 +221,6 @@ func (sf *SiaFile) Rename(newSiaPath, newSiaFilePath string) error {
updates := []writeaheadlog.Update{sf.createDeleteUpdate()}
// Rename file in memory.
sf.siaFilePath = newSiaFilePath
sf.staticMetadata.SiaPath = newSiaPath
// Update the ChangeTime because the metadata changed.
sf.staticMetadata.ChangeTime = time.Now()
// Write the header to the new location.
......@@ -282,13 +269,6 @@ func (sf *SiaFile) SetLocalPath(path string) error {
return sf.createAndApplyTransaction(updates...)
}
// SiaPath returns the file's sia path.
func (sf *SiaFile) SiaPath() string {
sf.mu.RLock()
defer sf.mu.RUnlock()
return sf.staticMetadata.SiaPath
}
// Size returns the file's size.
func (sf *SiaFile) Size() uint64 {
return uint64(sf.staticMetadata.StaticFileSize)
......
......@@ -67,11 +67,10 @@ func (sfs *SiaFileSet) NewFromLegacyData(fd FileData) (*SiaFileSetEntry, error)
StaticErasureCodeParams: ecParams,
StaticPagesPerChunk: numChunkPagesRequired(fd.ErasureCode.NumPieces()),
StaticPieceSize: fd.PieceSize,
SiaPath: fd.Name,
},
deleted: fd.Deleted,
deps: modules.ProdDependencies,
siaFilePath: filepath.Join(sfs.siaFileDir, fd.Name+ShareExtension),
siaFilePath: filepath.Join(sfs.staticSiaFileDir, fd.Name+ShareExtension),
staticUniqueID: fd.UID,
wal: sfs.wal,
}
......
......@@ -8,6 +8,7 @@ import (
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
......@@ -106,6 +107,25 @@ func (sf *SiaFile) addRandomHostKeys(n int) {
}
}
// addRandomPiecesToFile is a testing function that adds a random number of
// pieces to a siafile.
func addRandomPiecesToFile(sf *SiaFile) {
// Add pieces to each chunk.
for chunkIndex := range sf.staticChunks {
for pieceIndex := 0; pieceIndex < sf.ErasureCode().NumPieces(); pieceIndex++ {
numPieces := fastrand.Intn(3) // up to 2 hosts for each piece
for i := 0; i < numPieces; i++ {
pk := types.SiaPublicKey{Key: fastrand.Bytes(crypto.EntropySize)}
mr := crypto.Hash{}
fastrand.Read(mr[:])
if err := sf.AddPiece(pk, uint64(chunkIndex), uint64(pieceIndex), mr); err != nil {
panic(err)
}
}
}
}
}
// newBlankTestFileAndWAL creates an empty SiaFile for testing and also returns
// the WAL used in the creation and the path of the WAL.
func newBlankTestFileAndWAL() (*SiaFile, *writeaheadlog.WAL, string) {
......@@ -141,20 +161,7 @@ func newBlankTestFile() *SiaFile {
// number of pieces.
func newTestFile() *SiaFile {
sf := newBlankTestFile()
// Add pieces to each chunk.
for chunkIndex := range sf.staticChunks {
for pieceIndex := 0; pieceIndex < sf.ErasureCode().NumPieces(); pieceIndex++ {
numPieces := fastrand.Intn(3) // up to 2 hosts for each piece
for i := 0; i < numPieces; i++ {
pk := types.SiaPublicKey{Key: fastrand.Bytes(crypto.EntropySize)}
mr := crypto.Hash{}
fastrand.Read(mr[:])
if err := sf.AddPiece(pk, uint64(chunkIndex), uint64(pieceIndex), mr); err != nil {
panic(err)
}
}
}
}
addRandomPiecesToFile(sf)
return sf
}
......@@ -368,12 +375,12 @@ func TestRename(t *testing.T) {
entry, _, _ := newTestSiaFileSetWithFile()
// Create new paths for the file.
newSiaPath := entry.staticMetadata.SiaPath + "1"
newSiaFilePath := entry.siaFilePath + "1"
newSiaPath := entry.SiaPath() + "1"
oldSiaFilePath := entry.siaFilePath
newSiaFilePath := strings.TrimSuffix(oldSiaFilePath, ShareExtension) + "1" + ShareExtension
// Rename file
if err := entry.Rename(newSiaPath, newSiaFilePath); err != nil {
if err := entry.Rename(newSiaFilePath); err != nil {
t.Fatal("Failed to rename file", err)
}
......@@ -392,8 +399,8 @@ func TestRename(t *testing.T) {
if entry.siaFilePath != newSiaFilePath {
t.Fatal("SiaFilePath wasn't updated correctly")
}
if entry.staticMetadata.SiaPath != newSiaPath {
t.Fatal("SiaPath wasn't updated correctly")
if entry.SiaPath() != newSiaPath {
t.Fatal("SiaPath wasn't updated correctly", entry.SiaPath(), newSiaPath)
}
}
......
......@@ -112,6 +112,13 @@ func (hpk HostPublicKey) MarshalSia(w io.Writer) error {
return e.Err()
}
// SiaFilePath returns the siafile's path on disk.
func (sf *SiaFile) SiaFilePath() string {
sf.mu.RLock()
defer sf.mu.RUnlock()
return sf.siaFilePath
}
// UnmarshalSia implements the encoding.SiaUnmarshaler interface.
func (hpk *HostPublicKey) UnmarshalSia(r io.Reader) error {
d := encoding.NewDecoder(r, encoding.DefaultAllocLimit)
......@@ -151,7 +158,6 @@ func New(siaFilePath, siaPath, source string, wal *writeaheadlog.WAL, erasureCod
StaticErasureCodeParams: ecParams,
StaticPagesPerChunk: numChunkPagesRequired(erasureCode.NumPieces()),
StaticPieceSize: modules.SectorSize - masterKey.Type().Overhead(),
SiaPath: siaPath,
},
deps: modules.ProdDependencies,
siaFilePath: siaFilePath,
......
......@@ -11,6 +11,7 @@ import (
"gitlab.com/NebulousLabs/Sia/crypto"
"gitlab.com/NebulousLabs/Sia/modules"
"gitlab.com/NebulousLabs/errors"
"gitlab.com/NebulousLabs/fastrand"
"gitlab.com/NebulousLabs/writeaheadlog"
......@@ -24,8 +25,8 @@ type (
// SiaFileSet is a helper struct responsible for managing the renter's
// siafiles in memory
SiaFileSet struct {
siaFileDir string
siaFileMap map[string]*siaFileSetEntry
staticSiaFileDir string
siaFileMap map[string]*siaFileSetEntry
// utilities
mu sync.Mutex
......@@ -61,9 +62,9 @@ type (
// NewSiaFileSet initializes and returns a SiaFileSet
func NewSiaFileSet(filesDir string, wal *writeaheadlog.WAL) *SiaFileSet {
return &SiaFileSet{
siaFileDir: filesDir,
siaFileMap: make(map[string]*siaFileSetEntry),
wal: wal,
staticSiaFileDir: filesDir,
siaFileMap: make(map[string]*siaFileSetEntry),
wal: wal,
}
}
......@@ -137,7 +138,7 @@ func (sfs *SiaFileSet) closeEntry(entry *SiaFileSetEntry) {
// and then a new/different file was uploaded with the same siapath.
//
// If they are not the same entry, there is nothing more to do.
currentEntry := sfs.siaFileMap[entry.staticMetadata.SiaPath]
currentEntry := sfs.siaFileMap[entry.SiaPath()]
if currentEntry != entry.siaFileSetEntry {
return
}
......@@ -145,10 +146,27 @@ func (sfs *SiaFileSet) closeEntry(entry *SiaFileSetEntry) {
// If there are no more threads that have the current entry open, delete
// this entry from the set cache.
if len(currentEntry.threadMap) == 0 {
delete(sfs.siaFileMap, entry.staticMetadata.SiaPath)
delete(sfs.siaFileMap, entry.SiaPath())
}
}
// SiaPath returns the siapath of an entry.
func (entry *SiaFileSetEntry) SiaPath() string {
s := strings.TrimPrefix(entry.SiaFilePath(), entry.siaFileSet.staticSiaFileDir)
s = strings.TrimSuffix(s, ShareExtension)
return strings.TrimPrefix(s, "/")
}
// DirSiaPath returns the SiaPath of the directory that the SiaFile is in
func (entry *SiaFileSetEntry) DirSiaPath() string {
siapath := entry.SiaPath()
dirSiaPath := filepath.Dir(siapath)
if dirSiaPath == "." {
dirSiaPath = ""
}
return dirSiaPath
}
// exists checks to see if a file with the provided siaPath already exists in
// the renter
func (sfs *SiaFileSet) exists(siaPath string) bool {
......@@ -160,7 +178,7 @@ func (sfs *SiaFileSet) exists(siaPath string) bool {
return exists
}
// Check for file on disk
_, err := os.Stat(filepath.Join(sfs.siaFileDir, siaPath+ShareExtension))
_, err := os.Stat(filepath.Join(sfs.staticSiaFileDir, siaPath+ShareExtension))
return !os.IsNotExist(err)
}
......@@ -182,7 +200,7 @@ func (sfs *SiaFileSet) open(siaPath string) (*SiaFileSetEntry, error) {
entry, exists := sfs.siaFileMap[siaPath]
if !exists {
// Try and Load File from disk
sf, err := LoadSiaFile(filepath.Join(sfs.siaFileDir, siaPath+ShareExtension), sfs.wal)
sf, err := LoadSiaFile(filepath.Join(sfs.staticSiaFileDir, siaPath+ShareExtension), sfs.wal)
if os.IsNotExist(err) {
return nil, ErrUnknownPath
}
......@@ -222,7 +240,7 @@ func (sfs *SiaFileSet) Delete(siaPath string) error {
}
// Remove the siafile from the set map so that other threads can't find it.
delete(sfs.siaFileMap, entry.staticMetadata.SiaPath)
delete(sfs.siaFileMap, entry.SiaPath())
return nil
}
......@@ -234,6 +252,82 @@ func (sfs *SiaFileSet) Exists(siaPath string) bool {
return sfs.exists(siaPath)
}
// NewFromFileData creates a new SiaFile from a FileData object that was
// previously created from a legacy file.
func (sfs *SiaFileSet) NewFromFileData(fd FileData) (*SiaFileSetEntry, error) {
sfs.mu.Lock()
defer sfs.mu.Unlock()
// Make sure there are no leading slashes
fd.Name = strings.TrimPrefix(fd.Name, "/")
// legacy masterKeys are always twofish keys
mk, err := crypto.NewSiaKey(crypto.TypeTwofish, fd.MasterKey[:])
if err != nil {
return nil, errors.AddContext(err, "failed to restore master key")
}
currentTime := time.Now()
ecType, ecParams := marshalErasureCoder(fd.ErasureCode)
file := &SiaFile{
staticMetadata: metadata{
AccessTime: currentTime,
ChunkOffset: defaultReservedMDPages * pageSize,
ChangeTime: currentTime,
CreateTime: currentTime,
StaticFileSize: int64(fd.FileSize),
LocalPath: fd.RepairPath,
StaticMasterKey: mk.Key(),
StaticMasterKeyType: mk.Type(),
Mode: fd.Mode,
ModTime: currentTime,
staticErasureCode: fd.ErasureCode,
StaticErasureCodeType: ecType,
StaticErasureCodeParams: ecParams,
StaticPagesPerChunk: numChunkPagesRequired(fd.ErasureCode.NumPieces()),
StaticPieceSize: fd.PieceSize,
},
deleted: fd.Deleted,
deps: modules.ProdDependencies,
siaFilePath: filepath.Join(sfs.staticSiaFileDir, fd.Name+ShareExtension),
staticUniqueID: fd.UID,
wal: sfs.wal,
}
file.staticChunks = make([]chunk, len(fd.Chunks))
for i := range file.staticChunks {
file.staticChunks[i].Pieces = make([][]piece, file.staticMetadata.staticErasureCode.NumPieces())
}
// Populate the pubKeyTable of the file and add the pieces.
pubKeyMap := make(map[string]uint32)
for chunkIndex, chunk := range fd.Chunks {
for pieceIndex, pieceSet := range chunk.Pieces {
for _, p := range pieceSet {
// Check if we already added that public key.
tableOffset, exists := pubKeyMap[string(p.HostPubKey.Key)]
if !exists {
tableOffset = uint32(len(file.pubKeyTable))
pubKeyMap[string(p.HostPubKey.Key)] = tableOffset
file.pubKeyTable = append(file.pubKeyTable, HostPublicKey{
PublicKey: p.HostPubKey,
Used: true,
})
}
// Add the piece to the SiaFile.
file.staticChunks[chunkIndex].Pieces[pieceIndex] = append(file.staticChunks[chunkIndex].Pieces[pieceIndex], piece{
HostTableOffset: tableOffset,
MerkleRoot: p.MerkleRoot,
})
}
}
}
entry := sfs.newSiaFileSetEntry(file)
threadUID := randomThreadUID()
entry.threadMap[threadUID] = newThreadInfo()
sfs.siaFileMap[fd.Name] = entry
return &SiaFileSetEntry{
siaFileSetEntry: entry,
threadUID: threadUID,
}, file.saveFile()
}
// NewSiaFile create a new SiaFile, adds it to the SiaFileSet, adds the thread
// to the threadMap, and returns the SiaFileSetEntry. Since this method returns
// the SiaFileSetEntry, wherever NewSiaFile is called there should be a Close
......@@ -249,7 +343,7 @@ func (sfs *SiaFileSet) NewSiaFile(up modules.FileUploadParams, masterKey crypto.
return nil, ErrPathOverload
}
// Make sure there are no leading slashes
siaFilePath := filepath.Join(sfs.siaFileDir, siaPath+ShareExtension)
siaFilePath := filepath.Join(sfs.staticSiaFileDir, siaPath+ShareExtension)
sf, err := New(siaFilePath, siaPath, up.Source, sfs.wal, up.ErasureCode, masterKey, fileSize, fileMode)
if err != nil {
return nil, err
......@@ -297,7 +391,6 @@ func (sfs *SiaFileSet) Rename(siaPath, newSiaPath string) error {
// Update SiaFileSet map to hold the entry in the new siapath.
sfs.siaFileMap[newSiaPath] = entry.siaFileSetEntry
delete(sfs.siaFileMap, siaPath)
// Update the siafile to have a new name.
return entry.Rename(newSiaPath, filepath.Join(sfs.siaFileDir, newSiaPath+ShareExtension))
return entry.Rename(filepath.Join(sfs.staticSiaFileDir, newSiaPath+ShareExtension))
}
......@@ -128,7 +128,9 @@ func (s *Snapshot) Size() uint64 {
}
// Snapshot creates a snapshot of the SiaFile.
func (sf *SiaFile) Snapshot() *Snapshot {
func (entry *SiaFileSetEntry) Snapshot() *Snapshot {
siaPath := entry.SiaPath()
sf := entry.SiaFile
mk := sf.MasterKey()
sf.mu.RLock()
defer sf.mu.RUnlock()
......@@ -178,6 +180,6 @@ func (sf *SiaFile) Snapshot() *Snapshot {
staticMasterKey: mk,
staticMode: sf.staticMetadata.Mode,
staticPubKeyTable: pkt,
staticSiaPath: sf.staticMetadata.SiaPath,
staticSiaPath: siaPath,
}
}
......@@ -20,7 +20,11 @@ func TestSnapshot(t *testing.T) {
t.Parallel()
// Create a random file for testing and create a snapshot from it.
sf := newTestFile()
sf, _, err := newTestSiaFileSetWithFile()
if err != nil {
t.Fatal(err)
}
addRandomPiecesToFile(sf.SiaFile)
snap := sf.Snapshot()
// Make sure the snapshot has the same fields as the SiaFile.
......@@ -55,8 +59,8 @@ func TestSnapshot(t *testing.T) {
if !reflect.DeepEqual(sf.pubKeyTable, snap.staticPubKeyTable) {
t.Error("pubkeytables don't match")
}
if sf.staticMetadata.SiaPath != snap.staticSiaPath {
t.Error("siapaths don't match")
if sf.SiaPath() != snap.staticSiaPath {
t.Error("siapaths don't match", sf.SiaPath(), snap.staticSiaPath)
}
// Compare the pieces.
for i := range sf.staticChunks {
......@@ -113,7 +117,14 @@ func benchmarkSnapshot(b *testing.B, fileSize uint64) {
numChunks++
}
wal, _ := newTestWAL()
sf, err := New(siaFilePath, siaPath, source, wal, rc, sk, fileSize, fileMode)
siaFileDir := filepath.Join(os.TempDir(), "siafiles")
sfs := NewSiaFileSet(siaFileDir, wal)
up := modules.FileUploadParams{
Source: source,
SiaPath: siaPath,
ErasureCode: rc,
}
sf, err := sfs.NewSiaFile(up, sk, fileSize, fileMode)
if err != nil {
b.Fatal(err)
}
......
......@@ -165,7 +165,7 @@ func (r *Renter) managedDownloadLogicalChunkData(chunk *unfinishedUploadChunk) e
d, err := r.managedNewDownload(downloadParams{
destination: buf,
destinationType: "buffer",
file: chunk.fileEntry.SiaFile.Snapshot(),
file: chunk.fileEntry.Snapshot(),
latencyTarget: 200e3, // No need to rush latency on repair downloads.
length: downloadLength,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment