Commit ea6f3ab9 authored by Mads's avatar Mads

Refactor: Last nights work

parent b4b514a9
Pipeline #19638692 failed with stages
in 58 seconds
// Package config provides way to store and access configuration in an application.
// It provides a hardcoded default configuraiton that can be augmented using
// a local json file and/or environment variables.
//
// The path of the local json file can be specified with the -c switch. If
// omitted, the default path is used. The default path is ./config.json.
package config
import (
......@@ -25,9 +28,9 @@ func init() {
configLocation = flag.String("c", "./config.json", "Location of the JSON config file")
}
// New runs on import and takes care of bootstrapping the default config
// New takes care of bootstrapping the default config
// and populating the config from the config file and from the environment.
// The envinronment are searched for variables prepended with VEE_ and
// The environment are searched for variables prepended with VEE_ and
// uppercased. Thus config.Port can be overwritten with VEE_PORT.
func New() (*Obj, error) {
......
......@@ -9,27 +9,41 @@ import (
//FileService represents an implementation of vee.FileService
type FileService struct {
DB vee.MetadataService
DB vee.MetadataService
Store vee.ObjectService
}
// File queries the metadata database and tries to retrieve a file
func (s *FileService) File(name uuid.UUID) (*vee.File, error) {
var f vee.File
_, err := s.DB.File(name)
// NewFile returns a pointer to a new file with only it's name set
func (s *FileService) NewFile() *vee.File {
return &vee.File{Name: uuid.New()}
}
// GetFile queries the metadata database and tries to retrieve a file
func (s *FileService) GetFile(f *vee.File) error {
err := s.DB.GetFile(f)
if err != nil {
return err
}
if f.Invalidat.Before(time.Now()) {
return vee.ErrExpiredObject
}
err = s.Store.GetFile(f)
if err != nil {
return nil, err
return err
}
// Fetch the file metadata from the database
// Check that the file is still valid as per it's invalidat field
// Return error if invalid
// Fetch actual file from the object store
// Return file
return &f, nil
return nil
}
// CreateFile creates a file in the object store and creates the metadata.
func (s *FileService) CreateFile(name uuid.UUID, nonce [32]byte, createdat time.Time, invalidat time.Time) error {
// Upload file to object store
// Create file metadata in the database
// SaveFile saves a file in the object store and creates the metadata.
func (s *FileService) SaveFile(f *vee.File) error {
err := s.Store.CreateFile(f)
if err != nil {
return err
}
err = s.DB.CreateFile(f)
if err != nil {
err = s.Store.DeleteFile(f)
return err
}
return nil
}
......@@ -4,6 +4,7 @@ import (
"errors"
"reflect"
"testing"
"time"
"github.com/google/uuid"
vee "gitlab.com/MadsRC/vee/app"
......@@ -11,45 +12,155 @@ import (
"gitlab.com/MadsRC/vee/app/mock"
)
func resetMocks(mbs *mock.MetadataService, obs *mock.ObjectService) {
//Mock MetadataService.File() call.
mbs.GetFileFn = func(f *vee.File) error {
return nil
}
//Mock ObjectService.GetFile() call.
obs.GetFileFn = func(f *vee.File) error {
return nil
}
//Mock ObjectService.DeleteFile() call.
obs.DeleteFileFn = func(f *vee.File) error {
return nil
}
//Mock MetadataService.CreateFile() call.
mbs.CreateFileFn = func(f *vee.File) error {
return nil
}
//Mock ObjectService.CreateFile() call.
obs.CreateFileFn = func(f *vee.File) error {
return nil
}
mbs.DeleteFileFn = func(f *vee.File) error {
return nil
}
}
func TestFileService(t *testing.T) {
t.Run("File() calls", func(t *testing.T) {
// Inject our mock
var mbs mock.MetadataService
var fs fileservice.FileService
fs.DB = &mbs
//Mock MetadataService.File() call.
mbs.FileFn = func(name uuid.UUID) (*vee.File, error) {
return &vee.File{Name: name}, nil
}
_, err := fs.File(uuid.New())
// Inject our mock
var mbs mock.MetadataService
var obs mock.ObjectService
var fs fileservice.FileService
fs.DB = &mbs
fs.Store = &obs
resetMocks(&mbs, &obs)
t.Run("GetFile() calls", func(t *testing.T) {
err := fs.GetFile(&vee.File{
Name: uuid.New(),
Invalidat: time.Now().AddDate(0, 0, +1),
})
if err != nil {
t.Fatalf("File() returned err: %v", err)
t.Fatalf("GetFile() returned err: %v", err)
}
t.Run("MetadataService.File invocation", func(t *testing.T) {
t.Run("MetadataService.GetFile invocation", func(t *testing.T) {
//Mock MetadataService.File() call.
mbs.FileFn = func(name uuid.UUID) (*vee.File, error) {
return &vee.File{Name: name}, nil
}
_, _ = fs.File(uuid.New())
resetMocks(&mbs, &obs)
_ = fs.GetFile(&vee.File{Name: uuid.New()})
// Validate the mock
if !mbs.FileInvoked {
t.Fatal("expected File() to be invoked")
if !mbs.GetFileInvoked {
t.Fatal("expected GetFile() to be invoked")
}
})
t.Run("MetadataService.File err returned back", func(t *testing.T) {
resetMocks(&mbs, &obs)
t.Run("MetadataService.GetFile err returned back", func(t *testing.T) {
theError := errors.New("DB Error")
//Mock MetadataService.File() call.
mbs.GetFileFn = func(f *vee.File) error {
return theError
}
err := fs.GetFile(&vee.File{Name: uuid.New()})
if !reflect.DeepEqual(err, theError) {
t.Fatalf("Error was not as expected: %v", err)
}
})
resetMocks(&mbs, &obs)
t.Run("Test expired file", func(t *testing.T) {
mbs.GetFileFn = func(f *vee.File) error {
f.Invalidat = time.Now().AddDate(0, 0, -1)
return nil
}
err = fs.GetFile(&vee.File{Name: uuid.New()})
if !reflect.DeepEqual(err, vee.ErrExpiredObject) {
t.Fatalf("Expected error for file expiration, got: %s\n", err)
}
})
resetMocks(&mbs, &obs)
t.Run("Test ObjectService failure", func(t *testing.T) {
theError := errors.New("DB Error")
mbs.FileFn = func(name uuid.UUID) (*vee.File, error) {
return nil, theError
//Mock MetadataService.File() call.
obs.GetFileFn = func(f *vee.File) error {
return theError
}
_, err := fs.File(uuid.New())
err := fs.GetFile(&vee.File{
Name: uuid.New(),
Invalidat: time.Now().AddDate(0, 0, +1),
})
if !reflect.DeepEqual(err, theError) {
t.Fatalf("Error was not as expected: %v", err)
}
})
})
resetMocks(&mbs, &obs)
t.Run("SaveFile()", func(t *testing.T) {
file := fs.NewFile()
err := fs.SaveFile(file)
if err != nil {
t.Fatalf("SaveFile() returned err: %v", err)
}
if !obs.CreateFileInvoked {
t.Fatal("expected ObjectService.CreateFile() to be invoked")
}
if !mbs.CreateFileInvoked {
t.Fatal("expected MetadataService.CreateFile() to be invoked")
}
if obs.DeleteFileInvoked {
t.Fatal("Did not expect ObjectService.DeleteFile() to be invoked")
}
//Mock ObjectService.CreateFile() call.
obs.CreateFileFn = func(f *vee.File) error {
return errors.New("I like cake")
}
err = fs.SaveFile(file)
if err == errors.New("I like cake") {
t.Fatalf("SaveFile() returned unexpected err: %v", err)
}
obs.CreateFileFn = func(f *vee.File) error {
return nil
}
mbs.CreateFileFn = func(f *vee.File) error {
return errors.New("fisk")
}
err = fs.SaveFile(file)
if err != nil {
t.Fatalf("SaveFile() returned err: %v", err)
}
if !obs.CreateFileInvoked {
t.Fatal("expected ObjectService.CreateFile() to be invoked")
}
if !mbs.CreateFileInvoked {
t.Fatal("expected MetadataService.CreateFile() to be invoked")
}
if !obs.DeleteFileInvoked {
t.Fatal("expected ObjectService.DeleteFile() to be invoked")
}
})
t.Run("NewFile()", func(t *testing.T) {
var f1 = fs.NewFile()
var f2 = fs.NewFile()
if f1.Name.String() == f2.Name.String() {
t.Fatal("NewFile() did not return objects with random UUID")
}
})
}
package http
import (
"fmt"
"io"
"mime/multipart"
"net/http"
"strconv"
"strings"
"github.com/google/uuid"
vee "gitlab.com/MadsRC/vee/app"
)
type Handler struct {
func internalServerError(w http.ResponseWriter, r *http.Request) {
http.Error(w, "Internal server Error", http.StatusInternalServerError)
return
}
func badRequestError(w http.ResponseWriter, r *http.Request, msg string) {
http.Error(w, msg, http.StatusBadRequest)
return
}
// ReceiveFileHandler handles the multipart/form-data request received from a
// client.
//
// Do note that the Content-Type should be checked before passing control of
// the request to ReceiveFileHandler. If ReceiveFileHandler is given a
// request which isn't a multipart/form-data, a HTTP status code of 500 is
// returned
type ReceiveFileHandler struct {
FileService vee.FileService
Config vee.Config
Log vee.LogService
}
// extractSize takes a string, as returned by Header.Get("content-disposition")
// and tries to extract the size parameter. If returned, non-error, value is -1
// no size parameter has been found.
func extractSize(s string) (int64, error) {
s = strings.TrimSpace(s)
sSlice := strings.Split(s, ";")
for i := range sSlice {
if strings.Contains(sSlice[i], "size=") {
sSlice[i] = strings.Replace(sSlice[i], "\"", "", -1)
possibleSize := strings.Split(sSlice[i], "=")[1]
sizeInt, err := strconv.ParseInt(possibleSize, 10, 32)
if err != nil {
return 0, err
}
return sizeInt, nil
}
}
return -1, nil
}
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
name := uuid.New()
_, err := h.FileService.File(name)
// validateFilePart takes a part of a multipart request and
// ensures that it has a content-disposition and that
// said content-disposition includes a size parameter.
func validateFilePart(part *multipart.Part) error {
cd := part.Header.Get("content-disposition")
if cd == "" {
return fmt.Errorf("Missing content-disposition")
}
partSize, err := extractSize(cd)
if err != nil {
if strings.Contains(err.Error(), "invalid syntax") {
err = fmt.Errorf("Part %s size parameter has bad syntax\"", part.FormName())
}
return err
}
if partSize == -1 {
return fmt.Errorf("Missing size for part %s", part.FormName())
}
return nil
}
func (h *ReceiveFileHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
reader, err := r.MultipartReader()
if err != nil {
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
h.Log.Print(err)
internalServerError(w, r)
return
}
w.Write([]byte("I like cake"))
return
for {
part, err := reader.NextPart()
if err == io.EOF {
break
}
if err != nil {
h.Log.Print(err)
internalServerError(w, r)
return
}
err = validateFilePart(part)
if err != nil {
h.Log.Debug(err)
badRequestError(w, r, err.Error())
return
}
}
}
package http_test
import (
"errors"
"bytes"
"crypto/rand"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"net/textproto"
"testing"
"github.com/google/uuid"
vee "gitlab.com/MadsRC/vee/app"
veeHTTP "gitlab.com/MadsRC/vee/app/http"
"gitlab.com/MadsRC/vee/app/mock"
)
func TestHandler(t *testing.T) {
// Inject our mock
var fs mock.FileService
var h veeHTTP.Handler
h.FileService = &fs
func createMultipartBody(size int64, cd string) (*bytes.Buffer, string, error) {
data := make([]byte, size)
rand.Read(data)
dataReader := bytes.NewReader(data)
//Mock File() call.
fs.FileFn = func(name uuid.UUID) (*vee.File, error) {
return &vee.File{Name: name}, nil
}
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
//Invoke the handler
w := httptest.NewRecorder()
r, _ := http.NewRequest("GET", "/files/5c46eccf-cb2f-4c7f-a7cd-2a21eb263b58", nil)
h.ServeHTTP(w, r)
mimeheader := make(textproto.MIMEHeader)
if cd != "" {
mimeheader.Set("Content-Disposition", cd)
}
// Validate the mock
if !fs.FileInvoked {
t.Fatal("expected File() to be invoked")
part, err := writer.CreatePart(mimeheader)
if err != nil {
return nil, "", err
}
if w.Code != http.StatusOK {
t.Fatalf("Expected response code to be %d, was %d\n", http.StatusOK, w.Code)
_, err = io.Copy(part, dataReader)
if err != nil {
return nil, "", err
}
err = writer.Close()
if err != nil {
return nil, "", err
}
return body, writer.FormDataContentType(), nil
}
func TestHandlerError(t *testing.T) {
func TestHandler(t *testing.T) {
// Inject our mock
var fs mock.FileService
var h veeHTTP.Handler
var logService mock.LogService
var h veeHTTP.ReceiveFileHandler
h.FileService = &fs
h.Log = &logService
//Mock File() call.
fs.FileFn = func(name uuid.UUID) (*vee.File, error) {
return nil, errors.New("This is an error")
//Mock LogService.Print() call.
logService.PrintFn = func(v ...interface{}) {
return
}
//Mock LogService.Debug() call.
logService.DebugFn = func(v ...interface{}) {
return
}
//Invoke the handler
w := httptest.NewRecorder()
r, _ := http.NewRequest("GET", "/files/5c46eccf-cb2f-4c7f-a7cd-2a21eb263b58", nil)
h.ServeHTTP(w, r)
t.Run("Content-TypeNotMultipartFormData", func(t *testing.T) {
//Invoke the handler
w := httptest.NewRecorder()
r, _ := http.NewRequest("POST", "/file", nil)
h.ServeHTTP(w, r)
// Validate the mock
if !fs.FileInvoked {
t.Fatal("expected File() to be invoked")
}
if w.Code != http.StatusInternalServerError {
t.Fatalf("Expected response code to be %d, was %d\n", http.StatusInternalServerError, w.Code)
}
})
t.Run("Content-Disposition and size present", func(t *testing.T) {
//Invoke the handler
w := httptest.NewRecorder()
if w.Code != http.StatusInternalServerError {
t.Fatalf("Expected response code to be %d, was %d\n", http.StatusInternalServerError, w.Code)
}
tables := []struct {
x string
y int
}{
{"form-data; filename=\"randomdata.bin\"; name=\"file\"; size=\"10\"", http.StatusOK},
{"form-data; filename=\"randomdata.bin\"; name=\"file\"; size=10", http.StatusOK},
{"form-data; filename=\"randomdata.bin\"; name=\"file\"; size=kat", http.StatusBadRequest},
{"form-data; filename=\"randomdata.bin\"; name=\"file\"; size=\"kat\"", http.StatusBadRequest},
{"form-data; filename=\"randomdata.bin\"; name=\"file\"", http.StatusBadRequest},
{"", http.StatusBadRequest},
}
for _, table := range tables {
body, ct, err := createMultipartBody(10, table.x)
if err != nil {
t.Fatalf("Failed creating multipart: %v\n", err)
}
r, err := http.NewRequest("POST", "/file", body)
if err != nil {
t.Fatalf("Failed creating multipart: %v\n", err)
}
r.Header.Set("Content-Type", ct)
if err != nil {
t.Fatalf("Failed creating multipart: %v\n", err)
}
h.ServeHTTP(w, r)
if w.Code != table.y {
t.Fatalf("Expected response code to be %d, was %d\n", table.y, w.Code)
}
}
})
}
......@@ -3,7 +3,6 @@ package minio
import (
"time"
"github.com/google/uuid"
"github.com/minio/minio-go"
vee "gitlab.com/MadsRC/vee/app"
)
......@@ -15,17 +14,20 @@ type ObjectService struct {
Log vee.LogService
}
//File returns a file for a given name.
func (s *ObjectService) File(name uuid.UUID) (*vee.File, int64, error) {
var f vee.File
//GetFile returns a file for a given name.
func (s *ObjectService) GetFile(f *vee.File) error {
var err error
obj, err := s.Store.GetObject(s.Config.BucketName.String(), name.String(), minio.GetObjectOptions{})
obj, err := s.Store.GetObject(s.Config.BucketName.String(), f.Name.String(), minio.GetObjectOptions{})
if err != nil {
return nil, 0, err
return err
}
f.Content = obj
stat, err := obj.Stat()
return &f, stat.Size, err
if err != nil {
return err
}
f.Size = stat.Size
return err
}
//CreateFile creates a file with the given values
......@@ -43,6 +45,9 @@ func (s *ObjectService) File(name uuid.UUID) (*vee.File, int64, error) {
// lead to OOM.
func (s *ObjectService) CreateFile(f *vee.File) error {
var err error
if f.Name == vee.NilUUID {
return vee.ErrEmptyName
}
err = s.Store.MakeBucket(s.Config.BucketName.String(), s.Config.Location)
if err != nil {
exists, err := s.Store.BucketExists(s.Config.BucketName.String())
......@@ -66,3 +71,12 @@ func (s *ObjectService) CreateFile(f *vee.File) error {
return err
}
// DeleteFile deletes the object in the object store with the given file's name
func (s *ObjectService) DeleteFile(f *vee.File) error {
if f.Name == vee.NilUUID {
return vee.ErrEmptyName
}
err := s.Store.RemoveObject(s.Config.BucketName.String(), f.Name.String())
return err
}
......@@ -10,6 +10,7 @@ import (
"testing"
dc "github.com/fsouza/go-dockerclient"
"github.com/google/uuid"
"github.com/minio/minio-go"
"github.com/ory/dockertest"
vee "gitlab.com/MadsRC/vee/app"
......@@ -66,7 +67,7 @@ func TestMain(m *testing.M) {
}
func TestCreateFile(t *testing.T) {
func TestFile(t *testing.T) {
var logService mock.LogService
logService.DebugfFn = func(format string, v ...interface{}) {
return
......@@ -78,6 +79,7 @@ func TestCreateFile(t *testing.T) {
}
fileData := bytes.NewBufferString("R29waGVycyBydWxlIQ==")
file := vee.File{
Name: uuid.New(),
Content: fileData,
Size: int64(fileData.Len()),
}
......@@ -86,5 +88,22 @@ func TestCreateFile(t *testing.T) {
if err != nil {
t.Fatalf("Unexpected error: %s\n", err)
}
err = obs.GetFile(&file)
if err != nil {
t.Fatalf("Unexpected error: %s\n", err)
}
err = obs.DeleteFile(&file)
if err != nil {
t.Fatalf("Unexpected error: %s\n", err)
}
err = obs.CreateFile(&vee.File{})
if err != vee.ErrEmptyName {
t.Fatalf("Expected CreateFile to return ErrEmptyName, got: %s\n", err)
}
err = obs.DeleteFile(&vee.File{})
if err != vee.ErrEmptyName {
t.Fatalf("Expected DeleteFile to return ErrEmptyName, got: %s\n", err)
}
}
......@@ -3,50 +3,93 @@ package mock
import (
"io"
"github.com/google/uuid"
vee "gitlab.com/MadsRC/vee/app"
)
//FileService represents a mock implementation of vee.FileService
type FileService struct {
FileFn func(name uuid.UUID) (*vee.File, error)
FileInvoked bool
NewFileFn func() *vee.File
NewFileInvoked bool
GetFileFn func(f *vee.File) error
GetFileInvoked bool
SaveFileFn func(f *vee.File) error
SaveFileInvoked bool
}
// NewFile invokes the mock implementation and marks the function as invoked
func (s *FileService) NewFile() *vee.File {
s.NewFileInvoked = true
return s.NewFile()
}
// GetFile invokes the mock implementation and marks the function as invoked
func (s *FileService) GetFile(f *vee.File) error {
s.GetFileInvoked = true
return s.GetFileFn(f)
}
// SaveFile invokes the mock implementation aand marks the function as invoked
func (s *FileService) SaveFile(f *vee.File) error {
s.SaveFileInvoked = true
return s.SaveFileFn(f)
}
//MetadataService represents a mock implementation of vee.MetadataService
type MetadataService struct {
GetFileFn func(f *vee.File) error
GetFileInvoked bool
CreateFileFn func(f *vee.File) error
CreateFileInvoked bool
DeleteFileFn func(f *vee.File) error
DeleteFileInvoked bool
}
// File invokes the mock implementation and marks the function as invoked
func (s *FileService) File(name uuid.UUID) (*vee.File, error) {
s.FileInvoked = true
return s.FileFn(name)
// GetFile invokes the mock implementation and marks the function as invoked
func (s *MetadataService) GetFile(f *vee.File) error {
s.GetFileInvoked = true
return s.GetFileFn(f)
}
// CreateFile invokes the mock implementation aand marks the function as invoked
func (s *FileService) CreateFile(f *vee.File) error {
func (s *MetadataService) CreateFile(f *vee.File) error {
s.CreateFileInvoked = true
return s.CreateFileFn(f)