From b0e17d6a69a6e81cd1ca693cba47840de1f230da Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Thu, 12 Nov 2020 17:54:46 +1100 Subject: [PATCH 01/14] Add zip config flags Adds a config package with specific zip configuration structure that can be shared between packages. --- internal/config/config.go | 42 +++++------- internal/config/headers.go | 32 +++++++++ .../{config_test.go => headers_test.go} | 0 internal/serving/disk/zip/serving.go | 7 +- internal/vfs/zip/archive_test.go | 20 ++++-- internal/vfs/zip/lru_cache.go | 6 +- internal/vfs/zip/vfs.go | 68 +++---------------- internal/vfs/zip/vfs_test.go | 18 +++-- main.go | 30 ++++++++ 9 files changed, 123 insertions(+), 100 deletions(-) create mode 100644 internal/config/headers.go rename internal/config/{config_test.go => headers_test.go} (100%) diff --git a/internal/config/config.go b/internal/config/config.go index d415b21d..ef2af686 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,32 +1,22 @@ package config -import ( - "errors" - "net/http" - "strings" -) +import "time" -var errInvalidHeaderParameter = errors.New("invalid syntax specified as header parameter") +// TODO: refactor config flags in main.go and find a better way to handle all settings -// AddCustomHeaders adds a map of Headers to a Response -func AddCustomHeaders(w http.ResponseWriter, headers http.Header) error { - for k, v := range headers { - for _, value := range v { - w.Header().Add(k, value) - } - } - return nil -} +// ZipVFSConfig struct that can be accessed by different packages to share +// configuration parameters +var ZipVFSConfig *ZipServing -// ParseHeaderString parses a string of key values into a map -func ParseHeaderString(customHeaders []string) (http.Header, error) { - headers := http.Header{} - for _, keyValueString := range customHeaders { - keyValue := strings.SplitN(keyValueString, ":", 2) - if len(keyValue) != 2 { - return nil, errInvalidHeaderParameter - } - headers[strings.TrimSpace(keyValue[0])] = append(headers[strings.TrimSpace(keyValue[0])], strings.TrimSpace(keyValue[1])) - } - return headers, nil +// ZipServing stores all configuration values to be used by the zip VFS opening and +// caching +type ZipServing struct { + ExpirationInterval time.Duration + CleanupInterval time.Duration + RefreshInterval time.Duration + OpenTimeout time.Duration + DataOffsetItems int64 + DataOffsetExpirationInterval time.Duration + ReadlinkItems int64 + ReadlinkExpirationInterval time.Duration } diff --git a/internal/config/headers.go b/internal/config/headers.go new file mode 100644 index 00000000..d415b21d --- /dev/null +++ b/internal/config/headers.go @@ -0,0 +1,32 @@ +package config + +import ( + "errors" + "net/http" + "strings" +) + +var errInvalidHeaderParameter = errors.New("invalid syntax specified as header parameter") + +// AddCustomHeaders adds a map of Headers to a Response +func AddCustomHeaders(w http.ResponseWriter, headers http.Header) error { + for k, v := range headers { + for _, value := range v { + w.Header().Add(k, value) + } + } + return nil +} + +// ParseHeaderString parses a string of key values into a map +func ParseHeaderString(customHeaders []string) (http.Header, error) { + headers := http.Header{} + for _, keyValueString := range customHeaders { + keyValue := strings.SplitN(keyValueString, ":", 2) + if len(keyValue) != 2 { + return nil, errInvalidHeaderParameter + } + headers[strings.TrimSpace(keyValue[0])] = append(headers[strings.TrimSpace(keyValue[0])], strings.TrimSpace(keyValue[1])) + } + return headers, nil +} diff --git a/internal/config/config_test.go b/internal/config/headers_test.go similarity index 100% rename from internal/config/config_test.go rename to internal/config/headers_test.go diff --git a/internal/serving/disk/zip/serving.go b/internal/serving/disk/zip/serving.go index 95894fc9..9b3a7aa7 100644 --- a/internal/serving/disk/zip/serving.go +++ b/internal/serving/disk/zip/serving.go @@ -1,16 +1,21 @@ package zip import ( + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/serving" "gitlab.com/gitlab-org/gitlab-pages/internal/serving/disk" "gitlab.com/gitlab-org/gitlab-pages/internal/vfs" "gitlab.com/gitlab-org/gitlab-pages/internal/vfs/zip" ) -var instance = disk.New(vfs.Instrumented(zip.New())) +var instance serving.Serving // Instance returns a serving instance that is capable of reading files // from a zip archives opened from a URL, most likely stored in object storage func Instance() serving.Serving { + if instance == nil { + instance = disk.New(vfs.Instrumented(zip.New(config.ZipVFSConfig))) + } + return instance } diff --git a/internal/vfs/zip/archive_test.go b/internal/vfs/zip/archive_test.go index e1b0d116..f4fd6b5a 100644 --- a/internal/vfs/zip/archive_test.go +++ b/internal/vfs/zip/archive_test.go @@ -12,12 +12,24 @@ import ( "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/httprange" "gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers" ) var chdirSet = false +var zipCfg = &config.ZipServing{ + ExpirationInterval: time.Minute, + CleanupInterval: time.Minute / 2, + RefreshInterval: time.Minute / 2, + OpenTimeout: time.Minute / 2, + DataOffsetItems: 1000, + DataOffsetExpirationInterval: time.Hour, + ReadlinkItems: 500, + ReadlinkExpirationInterval: time.Hour, +} + func TestOpen(t *testing.T) { zip, cleanup := openZipArchive(t, nil) defer cleanup() @@ -75,7 +87,7 @@ func TestOpenCached(t *testing.T) { testServerURL, cleanup := newZipFileServerURL(t, "group/zip.gitlab.io/public-without-dirs.zip", &requests) defer cleanup() - fs := New() + fs := New(zipCfg) // We use array instead of map to ensure // predictable ordering of test execution @@ -322,7 +334,7 @@ func TestArchiveCanBeReadAfterOpenCtxCanceled(t *testing.T) { testServerURL, cleanup := newZipFileServerURL(t, "group/zip.gitlab.io/public.zip", nil) defer cleanup() - fs := New().(*zipVFS) + fs := New(zipCfg).(*zipVFS) zip := newArchive(fs, time.Second) ctx, cancel := context.WithCancel(context.Background()) cancel() @@ -345,7 +357,7 @@ func TestReadArchiveFails(t *testing.T) { testServerURL, cleanup := newZipFileServerURL(t, "group/zip.gitlab.io/public.zip", nil) defer cleanup() - fs := New().(*zipVFS) + fs := New(zipCfg).(*zipVFS) zip := newArchive(fs, time.Second) err := zip.openArchive(context.Background(), testServerURL+"/unkown.html") @@ -365,7 +377,7 @@ func openZipArchive(t *testing.T, requests *int64) (*zipArchive, func()) { testServerURL, cleanup := newZipFileServerURL(t, "group/zip.gitlab.io/public-without-dirs.zip", requests) - fs := New().(*zipVFS) + fs := New(zipCfg).(*zipVFS) zip := newArchive(fs, time.Second) err := zip.openArchive(context.Background(), testServerURL+"/public.zip") diff --git a/internal/vfs/zip/lru_cache.go b/internal/vfs/zip/lru_cache.go index 36817e66..9810e245 100644 --- a/internal/vfs/zip/lru_cache.go +++ b/internal/vfs/zip/lru_cache.go @@ -24,10 +24,10 @@ type lruCache struct { cache *ccache.Cache } -func newLruCache(op string, maxEntries uint32, duration time.Duration) *lruCache { +func newLruCache(op string, maxEntries int64, duration time.Duration) *lruCache { configuration := ccache.Configure() - configuration.MaxSize(int64(maxEntries)) - configuration.ItemsToPrune(maxEntries / lruCacheItemsToPruneDiv) + configuration.MaxSize(maxEntries) + configuration.ItemsToPrune(uint32(maxEntries) / lruCacheItemsToPruneDiv) configuration.GetsPerPromote(lruCacheGetsPerPromote) // if item gets requested frequently promote it configuration.OnDelete(func(*ccache.Item) { metrics.ZipCachedEntries.WithLabelValues(op).Dec() diff --git a/internal/vfs/zip/vfs.go b/internal/vfs/zip/vfs.go index 692a4a69..feeab974 100644 --- a/internal/vfs/zip/vfs.go +++ b/internal/vfs/zip/vfs.go @@ -9,29 +9,12 @@ import ( "github.com/patrickmn/go-cache" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/httprange" "gitlab.com/gitlab-org/gitlab-pages/internal/vfs" "gitlab.com/gitlab-org/gitlab-pages/metrics" ) -const ( - // TODO: make these configurable https://gitlab.com/gitlab-org/gitlab-pages/-/issues/464 - defaultCacheExpirationInterval = time.Minute - defaultCacheCleanupInterval = time.Minute / 2 - defaultCacheRefreshInterval = time.Minute / 2 - defaultOpenTimeout = time.Minute / 2 - - // we assume that each item costs around 100 bytes - // this gives around 5MB of raw memory needed without acceleration structures - defaultDataOffsetItems = 50000 - defaultDataOffsetExpirationInterval = time.Hour - - // we assume that each item costs around 200 bytes - // this gives around 2MB of raw memory needed without acceleration structures - defaultReadlinkItems = 10000 - defaultReadlinkExpirationInterval = time.Hour -) - var ( errAlreadyCached = errors.New("archive already cached") ) @@ -52,48 +35,13 @@ type zipVFS struct { archiveCount int64 } -// Option function allows to override default values -type Option func(*zipVFS) - -// WithCacheRefreshInterval when used it can override defaultCacheRefreshInterval -func WithCacheRefreshInterval(interval time.Duration) Option { - return func(vfs *zipVFS) { - vfs.cacheRefreshInterval = interval - } -} - -// WithCacheExpirationInterval when used it can override defaultCacheExpirationInterval -func WithCacheExpirationInterval(interval time.Duration) Option { - return func(vfs *zipVFS) { - vfs.cacheExpirationInterval = interval - } -} - -// WithCacheCleanupInterval when used it can override defaultCacheCleanupInterval -func WithCacheCleanupInterval(interval time.Duration) Option { - return func(vfs *zipVFS) { - vfs.cacheCleanupInterval = interval - } -} - -// WithOpenTimeout when used it can override openTimeout -func WithOpenTimeout(interval time.Duration) Option { - return func(vfs *zipVFS) { - vfs.openTimeout = interval - } -} - // New creates a zipVFS instance that can be used by a serving request -func New(options ...Option) vfs.VFS { +func New(cfg *config.ZipServing) vfs.VFS { zipVFS := &zipVFS{ - cacheExpirationInterval: defaultCacheExpirationInterval, - cacheRefreshInterval: defaultCacheRefreshInterval, - cacheCleanupInterval: defaultCacheCleanupInterval, - openTimeout: defaultOpenTimeout, - } - - for _, option := range options { - option(zipVFS) + cacheExpirationInterval: cfg.ExpirationInterval, + cacheRefreshInterval: cfg.RefreshInterval, + cacheCleanupInterval: cfg.CleanupInterval, + openTimeout: cfg.OpenTimeout, } zipVFS.cache = cache.New(zipVFS.cacheExpirationInterval, zipVFS.cacheCleanupInterval) @@ -104,8 +52,8 @@ func New(options ...Option) vfs.VFS { }) // TODO: To be removed with https://gitlab.com/gitlab-org/gitlab-pages/-/issues/480 - zipVFS.dataOffsetCache = newLruCache("data-offset", defaultDataOffsetItems, defaultDataOffsetExpirationInterval) - zipVFS.readlinkCache = newLruCache("readlink", defaultReadlinkItems, defaultReadlinkExpirationInterval) + zipVFS.dataOffsetCache = newLruCache("data-offset", cfg.DataOffsetItems, cfg.DataOffsetExpirationInterval) + zipVFS.readlinkCache = newLruCache("readlink", cfg.ReadlinkItems, cfg.ReadlinkExpirationInterval) return zipVFS } diff --git a/internal/vfs/zip/vfs_test.go b/internal/vfs/zip/vfs_test.go index dff2ff43..aa1ed109 100644 --- a/internal/vfs/zip/vfs_test.go +++ b/internal/vfs/zip/vfs_test.go @@ -9,6 +9,7 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/httprange" "gitlab.com/gitlab-org/gitlab-pages/internal/vfs" "gitlab.com/gitlab-org/gitlab-pages/metrics" @@ -35,7 +36,7 @@ func TestVFSRoot(t *testing.T) { }, } - vfs := New() + vfs := New(zipCfg) for name, tt := range tests { t.Run(name, func(t *testing.T) { @@ -73,7 +74,11 @@ func TestVFSFindOrOpenArchiveConcurrentAccess(t *testing.T) { path := testServerURL + "/public.zip" - vfs := New().(*zipVFS) + vfs := New(config.ZipServing{ + OpenTimeout: time.Second, + DataOffsetItems: 20, + ReadlinkItems: 20, + }).(*zipVFS) root, err := vfs.Root(context.Background(), path) require.NoError(t, err) @@ -158,10 +163,11 @@ func TestVFSFindOrOpenArchiveRefresh(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { withExpectedArchiveCount(t, 1, func(t *testing.T) { - vfs := New( - WithCacheExpirationInterval(test.expirationInterval), - WithCacheRefreshInterval(test.refreshInterval), - ).(*zipVFS) + cfg := *zipCfg + cfg.ExpirationInterval = test.expirationInterval + cfg.RefreshInterval = test.refreshInterval + + vfs := New(&cfg).(*zipVFS) path := testServerURL + test.path diff --git a/main.go b/main.go index 7defd281..b5056fa9 100644 --- a/main.go +++ b/main.go @@ -13,8 +13,10 @@ import ( "github.com/namsral/flag" log "github.com/sirupsen/logrus" + "gitlab.com/gitlab-org/labkit/errortracking" + cfg "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/logging" "gitlab.com/gitlab-org/gitlab-pages/internal/request" "gitlab.com/gitlab-org/gitlab-pages/internal/tlsconfig" @@ -78,6 +80,22 @@ var ( disableCrossOriginRequests = flag.Bool("disable-cross-origin-requests", false, "Disable cross-origin requests") + // zip serving settings + zipCacheExpiration = flag.Duration("zip-cache-expiration", 60*time.Second, "Zip serving archive cache expiration interval (default: 60s)") + zipCacheCleanup = flag.Duration("zip-cache-cleanup", 30*time.Second, "Zip serving archive cache cleanup interval (default: 30s)") + zipCacheRefresh = flag.Duration("zip-cache-refresh", 30*time.Second, "Zip serving archive cache refresh interval (default: 30s)") + + // we assume that each item costs around 100 bytes + // this gives around 5MB of raw memory needed without acceleration structures + zipCacheDataOffsetItems = flag.Int64("zip-cache-dataoffset-items", 50000, "Zip serving number of files to cache per archive (default: 50,000)") + zipCachetDataOffsetExpiration = flag.Duration("zip-cache-dataoffset-expiration", time.Hour, "Zip serving cached files expiration interval (default: 1h)") + // we assume that each item costs around 200 bytes + // this gives around 2MB of raw memory needed without acceleration structures + zipCacheReadlinkItems = flag.Int64("zip-cache-readlink-items", 10000, "Zip serving number of symbolic links to cache per archive (default: 10,000)") + zipCacheReadlinkExpirationInterval = flag.Duration("zip-cache-readlink-expiration", time.Hour, "Zip serving cached symbolic links expiration interval (default: 1h)") + + zipOpenTimeout = flag.Duration("zip-open-timeout", 30*time.Second, "Zip archive open timeout (default: 30s)") + // See init() listenHTTP MultiStringFlag listenHTTPS MultiStringFlag @@ -212,6 +230,18 @@ func configFromFlags() appConfig { checkAuthenticationConfig(config) + cfg.ZipVFSConfig = &cfg.ZipServing{ + ExpirationInterval: *zipCacheExpiration, + CleanupInterval: *zipCacheCleanup, + RefreshInterval: *zipCacheRefresh, + OpenTimeout: *zipOpenTimeout, + DataOffsetItems: *zipCacheDataOffsetItems, + DataOffsetExpirationInterval: *zipCachetDataOffsetExpiration, + ReadlinkItems: *zipCacheReadlinkItems, + ReadlinkExpirationInterval: *zipCacheReadlinkExpirationInterval, + } + + fmt.Printf("THE CONFIG FIRST:\n%+v\n", cfg.ZipVFSConfig) return config } -- GitLab From 188a44eba522c23110897ac6fb1091a68680db4e Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Fri, 13 Nov 2020 15:37:38 +1100 Subject: [PATCH 02/14] Add default config Use DefaultConfig as global --- internal/config/config.go | 41 ++++++++++++++++++++-------- internal/serving/disk/zip/serving.go | 2 +- internal/vfs/zip/archive_test.go | 12 +++----- internal/vfs/zip/vfs.go | 16 +++++++++-- internal/vfs/zip/vfs_test.go | 7 +---- main.go | 29 +++++--------------- 6 files changed, 57 insertions(+), 50 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index ef2af686..3e0f1965 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,22 +1,41 @@ package config -import "time" +import ( + "time" +) // TODO: refactor config flags in main.go and find a better way to handle all settings -// ZipVFSConfig struct that can be accessed by different packages to share +type Config struct { + zip *ZipServing +} + +// DefaultConfig struct that can be accessed by different packages to share // configuration parameters -var ZipVFSConfig *ZipServing +var DefaultConfig = &Config{ + // TODO: think of a way to not repeat these here and in main.go + zip: &ZipServing{ + ExpirationInterval: time.Minute, + CleanupInterval: time.Minute / 2, + RefreshInterval: time.Minute / 2, + OpenTimeout: time.Minute / 2, + }, +} // ZipServing stores all configuration values to be used by the zip VFS opening and // caching type ZipServing struct { - ExpirationInterval time.Duration - CleanupInterval time.Duration - RefreshInterval time.Duration - OpenTimeout time.Duration - DataOffsetItems int64 - DataOffsetExpirationInterval time.Duration - ReadlinkItems int64 - ReadlinkExpirationInterval time.Duration + ExpirationInterval time.Duration + CleanupInterval time.Duration + RefreshInterval time.Duration + OpenTimeout time.Duration +} + +// SetZip config to the global config +func (c *Config) SetZip(zip *ZipServing) { + c.zip = zip +} + +func (c *Config) GetZip() *ZipServing { + return c.zip } diff --git a/internal/serving/disk/zip/serving.go b/internal/serving/disk/zip/serving.go index 9b3a7aa7..53e66f13 100644 --- a/internal/serving/disk/zip/serving.go +++ b/internal/serving/disk/zip/serving.go @@ -14,7 +14,7 @@ var instance serving.Serving // from a zip archives opened from a URL, most likely stored in object storage func Instance() serving.Serving { if instance == nil { - instance = disk.New(vfs.Instrumented(zip.New(config.ZipVFSConfig))) + instance = disk.New(vfs.Instrumented(zip.New(config.DefaultConfig.GetZip()))) } return instance diff --git a/internal/vfs/zip/archive_test.go b/internal/vfs/zip/archive_test.go index f4fd6b5a..2bafece4 100644 --- a/internal/vfs/zip/archive_test.go +++ b/internal/vfs/zip/archive_test.go @@ -20,14 +20,10 @@ import ( var chdirSet = false var zipCfg = &config.ZipServing{ - ExpirationInterval: time.Minute, - CleanupInterval: time.Minute / 2, - RefreshInterval: time.Minute / 2, - OpenTimeout: time.Minute / 2, - DataOffsetItems: 1000, - DataOffsetExpirationInterval: time.Hour, - ReadlinkItems: 500, - ReadlinkExpirationInterval: time.Hour, + ExpirationInterval: time.Minute, + CleanupInterval: time.Minute / 2, + RefreshInterval: time.Minute / 2, + OpenTimeout: time.Minute / 2, } func TestOpen(t *testing.T) { diff --git a/internal/vfs/zip/vfs.go b/internal/vfs/zip/vfs.go index feeab974..d90d4796 100644 --- a/internal/vfs/zip/vfs.go +++ b/internal/vfs/zip/vfs.go @@ -15,6 +15,18 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/metrics" ) +const ( + // we assume that each item costs around 100 bytes + // this gives around 5MB of raw memory needed without acceleration structures + defaultDataOffsetItems = 50000 + defaultDataOffsetExpirationInterval = time.Hour + + // we assume that each item costs around 200 bytes + // this gives around 2MB of raw memory needed without acceleration structures + defaultReadlinkItems = 10000 + defaultReadlinkExpirationInterval = time.Hour +) + var ( errAlreadyCached = errors.New("archive already cached") ) @@ -52,8 +64,8 @@ func New(cfg *config.ZipServing) vfs.VFS { }) // TODO: To be removed with https://gitlab.com/gitlab-org/gitlab-pages/-/issues/480 - zipVFS.dataOffsetCache = newLruCache("data-offset", cfg.DataOffsetItems, cfg.DataOffsetExpirationInterval) - zipVFS.readlinkCache = newLruCache("readlink", cfg.ReadlinkItems, cfg.ReadlinkExpirationInterval) + zipVFS.dataOffsetCache = newLruCache("data-offset", defaultDataOffsetItems, defaultDataOffsetExpirationInterval) + zipVFS.readlinkCache = newLruCache("readlink", defaultReadlinkItems, defaultReadlinkExpirationInterval) return zipVFS } diff --git a/internal/vfs/zip/vfs_test.go b/internal/vfs/zip/vfs_test.go index aa1ed109..0a8a5e0b 100644 --- a/internal/vfs/zip/vfs_test.go +++ b/internal/vfs/zip/vfs_test.go @@ -9,7 +9,6 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/httprange" "gitlab.com/gitlab-org/gitlab-pages/internal/vfs" "gitlab.com/gitlab-org/gitlab-pages/metrics" @@ -74,11 +73,7 @@ func TestVFSFindOrOpenArchiveConcurrentAccess(t *testing.T) { path := testServerURL + "/public.zip" - vfs := New(config.ZipServing{ - OpenTimeout: time.Second, - DataOffsetItems: 20, - ReadlinkItems: 20, - }).(*zipVFS) + vfs := New(zipCfg).(*zipVFS) root, err := vfs.Root(context.Background(), path) require.NoError(t, err) diff --git a/main.go b/main.go index b5056fa9..1b5109fa 100644 --- a/main.go +++ b/main.go @@ -84,17 +84,7 @@ var ( zipCacheExpiration = flag.Duration("zip-cache-expiration", 60*time.Second, "Zip serving archive cache expiration interval (default: 60s)") zipCacheCleanup = flag.Duration("zip-cache-cleanup", 30*time.Second, "Zip serving archive cache cleanup interval (default: 30s)") zipCacheRefresh = flag.Duration("zip-cache-refresh", 30*time.Second, "Zip serving archive cache refresh interval (default: 30s)") - - // we assume that each item costs around 100 bytes - // this gives around 5MB of raw memory needed without acceleration structures - zipCacheDataOffsetItems = flag.Int64("zip-cache-dataoffset-items", 50000, "Zip serving number of files to cache per archive (default: 50,000)") - zipCachetDataOffsetExpiration = flag.Duration("zip-cache-dataoffset-expiration", time.Hour, "Zip serving cached files expiration interval (default: 1h)") - // we assume that each item costs around 200 bytes - // this gives around 2MB of raw memory needed without acceleration structures - zipCacheReadlinkItems = flag.Int64("zip-cache-readlink-items", 10000, "Zip serving number of symbolic links to cache per archive (default: 10,000)") - zipCacheReadlinkExpirationInterval = flag.Duration("zip-cache-readlink-expiration", time.Hour, "Zip serving cached symbolic links expiration interval (default: 1h)") - - zipOpenTimeout = flag.Duration("zip-open-timeout", 30*time.Second, "Zip archive open timeout (default: 30s)") + zipOpenTimeout = flag.Duration("zip-open-timeout", 30*time.Second, "Zip archive open timeout (default: 30s)") // See init() listenHTTP MultiStringFlag @@ -230,18 +220,13 @@ func configFromFlags() appConfig { checkAuthenticationConfig(config) - cfg.ZipVFSConfig = &cfg.ZipServing{ - ExpirationInterval: *zipCacheExpiration, - CleanupInterval: *zipCacheCleanup, - RefreshInterval: *zipCacheRefresh, - OpenTimeout: *zipOpenTimeout, - DataOffsetItems: *zipCacheDataOffsetItems, - DataOffsetExpirationInterval: *zipCachetDataOffsetExpiration, - ReadlinkItems: *zipCacheReadlinkItems, - ReadlinkExpirationInterval: *zipCacheReadlinkExpirationInterval, - } + cfg.DefaultConfig.SetZip(&cfg.ZipServing{ + ExpirationInterval: *zipCacheExpiration, + CleanupInterval: *zipCacheCleanup, + RefreshInterval: *zipCacheRefresh, + OpenTimeout: *zipOpenTimeout, + }) - fmt.Printf("THE CONFIG FIRST:\n%+v\n", cfg.ZipVFSConfig) return config } -- GitLab From e59ec12bcc06b10860c811bc3810a6320b9855d2 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Mon, 16 Nov 2020 13:56:22 +1100 Subject: [PATCH 03/14] Move headers to middleware --- app.go | 6 +++--- internal/{config => middleware}/headers.go | 5 ++--- internal/{config => middleware}/headers_test.go | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) rename internal/{config => middleware}/headers.go (95%) rename internal/{config => middleware}/headers_test.go (99%) diff --git a/app.go b/app.go index ca495073..a802f96d 100644 --- a/app.go +++ b/app.go @@ -21,11 +21,11 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/internal/acme" "gitlab.com/gitlab-org/gitlab-pages/internal/artifact" "gitlab.com/gitlab-org/gitlab-pages/internal/auth" - headerConfig "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/domain" "gitlab.com/gitlab-org/gitlab-pages/internal/handlers" "gitlab.com/gitlab-org/gitlab-pages/internal/httperrors" "gitlab.com/gitlab-org/gitlab-pages/internal/logging" + "gitlab.com/gitlab-org/gitlab-pages/internal/middleware" "gitlab.com/gitlab-org/gitlab-pages/internal/netutil" "gitlab.com/gitlab-org/gitlab-pages/internal/request" "gitlab.com/gitlab-org/gitlab-pages/internal/source" @@ -186,7 +186,7 @@ func (a *theApp) healthCheckMiddleware(handler http.Handler) (http.Handler, erro // customHeadersMiddleware will inject custom headers into the response func (a *theApp) customHeadersMiddleware(handler http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - headerConfig.AddCustomHeaders(w, a.CustomHeaders) + middleware.AddCustomHeaders(w, a.CustomHeaders) handler.ServeHTTP(w, r) }) @@ -493,7 +493,7 @@ func runApp(config appConfig) { } if len(config.CustomHeaders) != 0 { - customHeaders, err := headerConfig.ParseHeaderString(config.CustomHeaders) + customHeaders, err := middleware.ParseHeaderString(config.CustomHeaders) if err != nil { log.WithError(err).Fatal("Unable to parse header string") } diff --git a/internal/config/headers.go b/internal/middleware/headers.go similarity index 95% rename from internal/config/headers.go rename to internal/middleware/headers.go index d415b21d..77b008f3 100644 --- a/internal/config/headers.go +++ b/internal/middleware/headers.go @@ -1,4 +1,4 @@ -package config +package middleware import ( "errors" @@ -9,13 +9,12 @@ import ( var errInvalidHeaderParameter = errors.New("invalid syntax specified as header parameter") // AddCustomHeaders adds a map of Headers to a Response -func AddCustomHeaders(w http.ResponseWriter, headers http.Header) error { +func AddCustomHeaders(w http.ResponseWriter, headers http.Header) { for k, v := range headers { for _, value := range v { w.Header().Add(k, value) } } - return nil } // ParseHeaderString parses a string of key values into a map diff --git a/internal/config/headers_test.go b/internal/middleware/headers_test.go similarity index 99% rename from internal/config/headers_test.go rename to internal/middleware/headers_test.go index 44afd470..17d31b50 100644 --- a/internal/config/headers_test.go +++ b/internal/middleware/headers_test.go @@ -1,4 +1,4 @@ -package config +package middleware import ( "net/http/httptest" -- GitLab From a7db22b91f700f5f8ed17fd141a21b763c2af87e Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Mon, 16 Nov 2020 14:11:54 +1100 Subject: [PATCH 04/14] Use default config for tests --- internal/vfs/zip/archive_test.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/internal/vfs/zip/archive_test.go b/internal/vfs/zip/archive_test.go index 2bafece4..08aa22ff 100644 --- a/internal/vfs/zip/archive_test.go +++ b/internal/vfs/zip/archive_test.go @@ -19,12 +19,7 @@ import ( var chdirSet = false -var zipCfg = &config.ZipServing{ - ExpirationInterval: time.Minute, - CleanupInterval: time.Minute / 2, - RefreshInterval: time.Minute / 2, - OpenTimeout: time.Minute / 2, -} +var zipCfg = config.DefaultConfig.GetZip() func TestOpen(t *testing.T) { zip, cleanup := openZipArchive(t, nil) -- GitLab From 22ede0d8ea69bf26929825a75dad4172ef25cc26 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Thu, 19 Nov 2020 11:43:10 +1100 Subject: [PATCH 05/14] Move zip flags to config --- internal/config/config.go | 38 ++++++++++++---------------- internal/serving/disk/zip/serving.go | 2 +- internal/vfs/zip/archive_test.go | 7 ++--- main.go | 14 ---------- test/acceptance/zip_test.go | 25 ++++++++++++++++++ 5 files changed, 46 insertions(+), 40 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 3e0f1965..b5fd834b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2,24 +2,27 @@ package config import ( "time" + + "github.com/namsral/flag" ) -// TODO: refactor config flags in main.go and find a better way to handle all settings +// Default configuration that can be accessed by different packages +var Default *Config -type Config struct { - zip *ZipServing +// TODO: move all flags to this package, including flag.Parse() +func init() { + Default = &Config{ + Zip: &ZipServing{}, + } + + flag.DurationVar(&Default.Zip.ExpirationInterval, "zip-cache-expiration", 60*time.Second, "Zip serving archive cache expiration interval") + flag.DurationVar(&Default.Zip.CleanupInterval, "zip-cache-cleanup", 30*time.Second, "Zip serving archive cache cleanup interval") + flag.DurationVar(&Default.Zip.RefreshInterval, "zip-cache-refresh", 30*time.Second, "Zip serving archive cache refresh interval") + flag.DurationVar(&Default.Zip.OpenTimeout, "zip-open-timeout", 30*time.Second, "Zip archive open timeout") } -// DefaultConfig struct that can be accessed by different packages to share -// configuration parameters -var DefaultConfig = &Config{ - // TODO: think of a way to not repeat these here and in main.go - zip: &ZipServing{ - ExpirationInterval: time.Minute, - CleanupInterval: time.Minute / 2, - RefreshInterval: time.Minute / 2, - OpenTimeout: time.Minute / 2, - }, +type Config struct { + Zip *ZipServing } // ZipServing stores all configuration values to be used by the zip VFS opening and @@ -30,12 +33,3 @@ type ZipServing struct { RefreshInterval time.Duration OpenTimeout time.Duration } - -// SetZip config to the global config -func (c *Config) SetZip(zip *ZipServing) { - c.zip = zip -} - -func (c *Config) GetZip() *ZipServing { - return c.zip -} diff --git a/internal/serving/disk/zip/serving.go b/internal/serving/disk/zip/serving.go index 53e66f13..b552050c 100644 --- a/internal/serving/disk/zip/serving.go +++ b/internal/serving/disk/zip/serving.go @@ -14,7 +14,7 @@ var instance serving.Serving // from a zip archives opened from a URL, most likely stored in object storage func Instance() serving.Serving { if instance == nil { - instance = disk.New(vfs.Instrumented(zip.New(config.DefaultConfig.GetZip()))) + instance = disk.New(vfs.Instrumented(zip.New(config.Default.Zip))) } return instance diff --git a/internal/vfs/zip/archive_test.go b/internal/vfs/zip/archive_test.go index 08aa22ff..3273edad 100644 --- a/internal/vfs/zip/archive_test.go +++ b/internal/vfs/zip/archive_test.go @@ -17,9 +17,10 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers" ) -var chdirSet = false - -var zipCfg = config.DefaultConfig.GetZip() +var ( + chdirSet = false + zipCfg = config.Default.Zip +) func TestOpen(t *testing.T) { zip, cleanup := openZipArchive(t, nil) diff --git a/main.go b/main.go index 1b5109fa..95eae4ce 100644 --- a/main.go +++ b/main.go @@ -16,7 +16,6 @@ import ( "gitlab.com/gitlab-org/labkit/errortracking" - cfg "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/logging" "gitlab.com/gitlab-org/gitlab-pages/internal/request" "gitlab.com/gitlab-org/gitlab-pages/internal/tlsconfig" @@ -80,12 +79,6 @@ var ( disableCrossOriginRequests = flag.Bool("disable-cross-origin-requests", false, "Disable cross-origin requests") - // zip serving settings - zipCacheExpiration = flag.Duration("zip-cache-expiration", 60*time.Second, "Zip serving archive cache expiration interval (default: 60s)") - zipCacheCleanup = flag.Duration("zip-cache-cleanup", 30*time.Second, "Zip serving archive cache cleanup interval (default: 30s)") - zipCacheRefresh = flag.Duration("zip-cache-refresh", 30*time.Second, "Zip serving archive cache refresh interval (default: 30s)") - zipOpenTimeout = flag.Duration("zip-open-timeout", 30*time.Second, "Zip archive open timeout (default: 30s)") - // See init() listenHTTP MultiStringFlag listenHTTPS MultiStringFlag @@ -220,13 +213,6 @@ func configFromFlags() appConfig { checkAuthenticationConfig(config) - cfg.DefaultConfig.SetZip(&cfg.ZipServing{ - ExpirationInterval: *zipCacheExpiration, - CleanupInterval: *zipCacheCleanup, - RefreshInterval: *zipCacheRefresh, - OpenTimeout: *zipOpenTimeout, - }) - return config } diff --git a/test/acceptance/zip_test.go b/test/acceptance/zip_test.go index ea703ebe..3bfa21bf 100644 --- a/test/acceptance/zip_test.go +++ b/test/acceptance/zip_test.go @@ -104,6 +104,31 @@ func TestZipServing(t *testing.T) { } } +func TestZipServingConfigShortTimeout(t *testing.T) { + skipUnlessEnabled(t) + + var apiCalled bool + source := NewGitlabDomainsSourceStub(t, &apiCalled) + defer source.Close() + + gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t) + + pagesArgs := []string{"-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey, "-domain-config-source", "gitlab", + "-zip-open-timeout=1ns"} // <- test purpose + + teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{}, pagesArgs...) + defer teardown() + + _, cleanup := newZipFileServerURL(t, "../../shared/pages/group/zip.gitlab.io/public.zip") + defer cleanup() + + response, err := GetPageFromListener(t, httpListener, "zip.gitlab.io", "/") + require.NoError(t, err) + defer response.Body.Close() + + require.Equal(t, http.StatusInternalServerError, response.StatusCode, "should fail to serve") +} + func newZipFileServerURL(t *testing.T, zipFilePath string) (string, func()) { t.Helper() -- GitLab From d28903a32e2bfa8a80547e639e941eac4843034f Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Thu, 19 Nov 2020 11:53:30 +1100 Subject: [PATCH 06/14] Log zip config --- internal/config/config.go | 6 +++++- internal/serving/disk/zip/serving_test.go | 5 +++++ internal/vfs/zip/archive_test.go | 7 ++++++- internal/vfs/zip/vfs.go | 2 ++ main.go | 8 ++++++++ 5 files changed, 26 insertions(+), 2 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index b5fd834b..527b423b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "time" "github.com/namsral/flag" @@ -10,7 +11,7 @@ import ( var Default *Config // TODO: move all flags to this package, including flag.Parse() -func init() { +func Init() { Default = &Config{ Zip: &ZipServing{}, } @@ -19,6 +20,9 @@ func init() { flag.DurationVar(&Default.Zip.CleanupInterval, "zip-cache-cleanup", 30*time.Second, "Zip serving archive cache cleanup interval") flag.DurationVar(&Default.Zip.RefreshInterval, "zip-cache-refresh", 30*time.Second, "Zip serving archive cache refresh interval") flag.DurationVar(&Default.Zip.OpenTimeout, "zip-open-timeout", 30*time.Second, "Zip archive open timeout") + + // flag.Parse() + fmt.Printf("init: CONFIG: %+v\n", Default.Zip) } type Config struct { diff --git a/internal/serving/disk/zip/serving_test.go b/internal/serving/disk/zip/serving_test.go index bca2ae51..9a7618fe 100644 --- a/internal/serving/disk/zip/serving_test.go +++ b/internal/serving/disk/zip/serving_test.go @@ -8,10 +8,15 @@ import ( "github.com/stretchr/testify/require" + cfg "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/serving" "gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers" ) +func init() { + cfg.Init() +} + func TestZip_ServeFileHTTP(t *testing.T) { testServerURL, cleanup := newZipFileServerURL(t, "group/zip.gitlab.io/public-without-dirs.zip") defer cleanup() diff --git a/internal/vfs/zip/archive_test.go b/internal/vfs/zip/archive_test.go index 3273edad..37e81d7a 100644 --- a/internal/vfs/zip/archive_test.go +++ b/internal/vfs/zip/archive_test.go @@ -19,9 +19,14 @@ import ( var ( chdirSet = false - zipCfg = config.Default.Zip + zipCfg *config.ZipServing ) +func init() { + config.Init() + zipCfg = config.Default.Zip +} + func TestOpen(t *testing.T) { zip, cleanup := openZipArchive(t, nil) defer cleanup() diff --git a/internal/vfs/zip/vfs.go b/internal/vfs/zip/vfs.go index d90d4796..f853b0b9 100644 --- a/internal/vfs/zip/vfs.go +++ b/internal/vfs/zip/vfs.go @@ -3,6 +3,7 @@ package zip import ( "context" "errors" + "fmt" "net/url" "sync" "time" @@ -49,6 +50,7 @@ type zipVFS struct { // New creates a zipVFS instance that can be used by a serving request func New(cfg *config.ZipServing) vfs.VFS { + fmt.Printf("\n\n\n what the config?: %+v\n\n\n", cfg) zipVFS := &zipVFS{ cacheExpirationInterval: cfg.ExpirationInterval, cacheRefreshInterval: cfg.RefreshInterval, diff --git a/main.go b/main.go index 95eae4ce..53be4607 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,7 @@ import ( "gitlab.com/gitlab-org/labkit/errortracking" + cfg "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/logging" "gitlab.com/gitlab-org/gitlab-pages/internal/request" "gitlab.com/gitlab-org/gitlab-pages/internal/tlsconfig" @@ -295,6 +296,10 @@ func loadConfig() appConfig { "api-secret-key": *gitLabAPISecretKey, "domain-config-source": config.DomainConfigurationSource, "auth-redirect-uri": config.RedirectURI, + "zip-cache-expiration": cfg.Default.Zip.ExpirationInterval, + "zip-cache-cleanup": cfg.Default.Zip.CleanupInterval, + "zip-cache-refresh": cfg.Default.Zip.RefreshInterval, + "zip-open-timeout": cfg.Default.Zip.OpenTimeout, }).Debug("Start daemon with configuration") return config @@ -306,6 +311,8 @@ func appMain() { // read from -config=/path/to/gitlab-pages-config flag.String(flag.DefaultConfigFlagname, "", "path to config file") flag.Parse() + + fmt.Printf("appMain: CONFIG: %+v\n", cfg.Default.Zip) if err := tlsconfig.ValidateTLSVersions(*tlsMinVersion, *tlsMaxVersion); err != nil { fatal(err, "invalid TLS version") } @@ -440,6 +447,7 @@ func main() { metrics.MustRegister() + cfg.Init() daemonMain() appMain() } -- GitLab From 973ff1719c8559a12610ec3d3ffe070d2bb592a9 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Thu, 19 Nov 2020 20:57:02 +1100 Subject: [PATCH 07/14] Move flag definition to main Add zip config to appConfig --- app.go | 6 +++++ app_config.go | 5 +++++ internal/config/config.go | 27 ++++++++--------------- internal/serving/disk/zip/serving_test.go | 5 ----- internal/vfs/zip/archive_test.go | 1 - main.go | 22 +++++++++++++----- 6 files changed, 37 insertions(+), 29 deletions(-) diff --git a/app.go b/app.go index a802f96d..5468036d 100644 --- a/app.go +++ b/app.go @@ -21,6 +21,7 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/internal/acme" "gitlab.com/gitlab-org/gitlab-pages/internal/artifact" "gitlab.com/gitlab-org/gitlab-pages/internal/auth" + cfg "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/domain" "gitlab.com/gitlab-org/gitlab-pages/internal/handlers" "gitlab.com/gitlab-org/gitlab-pages/internal/httperrors" @@ -504,6 +505,11 @@ func runApp(config appConfig) { log.WithError(err).Warn("Loading extended MIME database failed") } + cfg.Default.Zip.ExpirationInterval = config.ZipCacheExpiry + cfg.Default.Zip.CleanupInterval = config.ZipCacheCleanup + cfg.Default.Zip.RefreshInterval = config.ZipCacheRefresh + cfg.Default.Zip.OpenTimeout = config.ZipeOpenTimeout + a.Run() } diff --git a/app_config.go b/app_config.go index bb4aa917..0dd192d5 100644 --- a/app_config.go +++ b/app_config.go @@ -41,6 +41,11 @@ type appConfig struct { SentryDSN string SentryEnvironment string CustomHeaders []string + + ZipCacheExpiry time.Duration + ZipCacheRefresh time.Duration + ZipCacheCleanup time.Duration + ZipeOpenTimeout time.Duration } // InternalGitLabServerURL returns URL to a GitLab instance. diff --git a/internal/config/config.go b/internal/config/config.go index 527b423b..f1d70f2c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,28 +1,19 @@ package config import ( - "fmt" "time" - - "github.com/namsral/flag" ) // Default configuration that can be accessed by different packages -var Default *Config - -// TODO: move all flags to this package, including flag.Parse() -func Init() { - Default = &Config{ - Zip: &ZipServing{}, - } - - flag.DurationVar(&Default.Zip.ExpirationInterval, "zip-cache-expiration", 60*time.Second, "Zip serving archive cache expiration interval") - flag.DurationVar(&Default.Zip.CleanupInterval, "zip-cache-cleanup", 30*time.Second, "Zip serving archive cache cleanup interval") - flag.DurationVar(&Default.Zip.RefreshInterval, "zip-cache-refresh", 30*time.Second, "Zip serving archive cache refresh interval") - flag.DurationVar(&Default.Zip.OpenTimeout, "zip-open-timeout", 30*time.Second, "Zip archive open timeout") - - // flag.Parse() - fmt.Printf("init: CONFIG: %+v\n", Default.Zip) +var Default = &Config{ + // TODO: remove duplication once all flags are defined in this package + // https://gitlab.com/gitlab-org/gitlab-pages/-/issues/507 + Zip: &ZipServing{ + ExpirationInterval: time.Minute, + CleanupInterval: time.Minute / 2, + RefreshInterval: time.Minute / 2, + OpenTimeout: time.Minute / 2, + }, } type Config struct { diff --git a/internal/serving/disk/zip/serving_test.go b/internal/serving/disk/zip/serving_test.go index 9a7618fe..bca2ae51 100644 --- a/internal/serving/disk/zip/serving_test.go +++ b/internal/serving/disk/zip/serving_test.go @@ -8,15 +8,10 @@ import ( "github.com/stretchr/testify/require" - cfg "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/serving" "gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers" ) -func init() { - cfg.Init() -} - func TestZip_ServeFileHTTP(t *testing.T) { testServerURL, cleanup := newZipFileServerURL(t, "group/zip.gitlab.io/public-without-dirs.zip") defer cleanup() diff --git a/internal/vfs/zip/archive_test.go b/internal/vfs/zip/archive_test.go index 37e81d7a..e5fcde54 100644 --- a/internal/vfs/zip/archive_test.go +++ b/internal/vfs/zip/archive_test.go @@ -23,7 +23,6 @@ var ( ) func init() { - config.Init() zipCfg = config.Default.Zip } diff --git a/main.go b/main.go index 53be4607..f8c1cfaf 100644 --- a/main.go +++ b/main.go @@ -31,6 +31,7 @@ var VERSION = "dev" var REVISION = "HEAD" func init() { + // TODO: move all flags to config pkg https://gitlab.com/gitlab-org/gitlab-pages/-/issues/507 flag.Var(&listenHTTP, "listen-http", "The address(es) to listen on for HTTP requests") flag.Var(&listenHTTPS, "listen-https", "The address(es) to listen on for HTTPS requests") flag.Var(&listenProxy, "listen-proxy", "The address(es) to listen on for proxy requests") @@ -39,6 +40,7 @@ func init() { } var ( + // TODO: move all flags to config pkg https://gitlab.com/gitlab-org/gitlab-pages/-/issues/507 pagesRootCert = flag.String("root-cert", "", "The default path to file certificate to serve static pages") pagesRootKey = flag.String("root-key", "", "The default path to file certificate to serve static pages") redirectHTTP = flag.Bool("redirect-http", false, "Redirect pages from HTTP to HTTPS") @@ -77,6 +79,11 @@ var ( insecureCiphers = flag.Bool("insecure-ciphers", false, "Use default list of cipher suites, may contain insecure ones like 3DES and RC4") tlsMinVersion = flag.String("tls-min-version", "tls1.2", tlsconfig.FlagUsage("min")) tlsMaxVersion = flag.String("tls-max-version", "", tlsconfig.FlagUsage("max")) + // TODO: move all flags to config pkg https://gitlab.com/gitlab-org/gitlab-pages/-/issues/507 + zipCacheExpiration = flag.Duration("zip-cache-expiration", 60*time.Second, "Zip serving archive cache expiration interval") + zipCacheCleanup = flag.Duration("zip-cache-cleanup", 30*time.Second, "Zip serving archive cache cleanup interval") + zipCacheRefresh = flag.Duration("zip-cache-refresh", 30*time.Second, "Zip serving archive cache refresh interval") + zipOpenTimeout = flag.Duration("zip-open-timeout", 30*time.Second, "Zip archive open timeout") disableCrossOriginRequests = flag.Bool("disable-cross-origin-requests", false, "Disable cross-origin requests") @@ -212,6 +219,11 @@ func configFromFlags() appConfig { config.SentryDSN = *sentryDSN config.SentryEnvironment = *sentryEnvironment + config.ZipCacheExpiry = *zipCacheExpiration + config.ZipCacheCleanup = *zipCacheCleanup + config.ZipCacheRefresh = *zipCacheRefresh + config.ZipeOpenTimeout = *zipOpenTimeout + checkAuthenticationConfig(config) return config @@ -296,10 +308,10 @@ func loadConfig() appConfig { "api-secret-key": *gitLabAPISecretKey, "domain-config-source": config.DomainConfigurationSource, "auth-redirect-uri": config.RedirectURI, - "zip-cache-expiration": cfg.Default.Zip.ExpirationInterval, - "zip-cache-cleanup": cfg.Default.Zip.CleanupInterval, - "zip-cache-refresh": cfg.Default.Zip.RefreshInterval, - "zip-open-timeout": cfg.Default.Zip.OpenTimeout, + "zip-cache-expiration": config.ZipCacheExpiry, + "zip-cache-cleanup": config.ZipCacheCleanup, + "zip-cache-refresh": config.ZipCacheRefresh, + "zip-open-timeout": config.ZipeOpenTimeout, }).Debug("Start daemon with configuration") return config @@ -310,6 +322,7 @@ func appMain() { // read from -config=/path/to/gitlab-pages-config flag.String(flag.DefaultConfigFlagname, "", "path to config file") + flag.Parse() fmt.Printf("appMain: CONFIG: %+v\n", cfg.Default.Zip) @@ -447,7 +460,6 @@ func main() { metrics.MustRegister() - cfg.Init() daemonMain() appMain() } -- GitLab From 64374fa4294cf1b143cc67e1e85dfbf726ed0572 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Sun, 22 Nov 2020 11:31:47 +1100 Subject: [PATCH 08/14] Increase time for flaky test Cleanup --- internal/config/config.go | 9 +-------- internal/serving/disk/zip/serving_test.go | 8 ++++++++ internal/vfs/zip/archive_test.go | 11 ++++++----- internal/vfs/zip/vfs.go | 2 -- internal/vfs/zip/vfs_test.go | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index f1d70f2c..66f1a2b9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -6,14 +6,7 @@ import ( // Default configuration that can be accessed by different packages var Default = &Config{ - // TODO: remove duplication once all flags are defined in this package - // https://gitlab.com/gitlab-org/gitlab-pages/-/issues/507 - Zip: &ZipServing{ - ExpirationInterval: time.Minute, - CleanupInterval: time.Minute / 2, - RefreshInterval: time.Minute / 2, - OpenTimeout: time.Minute / 2, - }, + Zip: &ZipServing{}, } type Config struct { diff --git a/internal/serving/disk/zip/serving_test.go b/internal/serving/disk/zip/serving_test.go index bca2ae51..5bf16d7b 100644 --- a/internal/serving/disk/zip/serving_test.go +++ b/internal/serving/disk/zip/serving_test.go @@ -5,9 +5,11 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/serving" "gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers" ) @@ -53,6 +55,12 @@ func TestZip_ServeFileHTTP(t *testing.T) { }, } + config.Default.Zip = &config.ZipServing{ + ExpirationInterval: 10 * time.Second, + CleanupInterval: 5 * time.Second, + RefreshInterval: 5 * time.Second, + OpenTimeout: 5 * time.Second, + } s := Instance() for name, test := range tests { diff --git a/internal/vfs/zip/archive_test.go b/internal/vfs/zip/archive_test.go index e5fcde54..da778e62 100644 --- a/internal/vfs/zip/archive_test.go +++ b/internal/vfs/zip/archive_test.go @@ -19,13 +19,14 @@ import ( var ( chdirSet = false - zipCfg *config.ZipServing + zipCfg = &config.ZipServing{ + ExpirationInterval: 10 * time.Second, + CleanupInterval: 5 * time.Second, + RefreshInterval: 5 * time.Second, + OpenTimeout: 5 * time.Second, + } ) -func init() { - zipCfg = config.Default.Zip -} - func TestOpen(t *testing.T) { zip, cleanup := openZipArchive(t, nil) defer cleanup() diff --git a/internal/vfs/zip/vfs.go b/internal/vfs/zip/vfs.go index f853b0b9..d90d4796 100644 --- a/internal/vfs/zip/vfs.go +++ b/internal/vfs/zip/vfs.go @@ -3,7 +3,6 @@ package zip import ( "context" "errors" - "fmt" "net/url" "sync" "time" @@ -50,7 +49,6 @@ type zipVFS struct { // New creates a zipVFS instance that can be used by a serving request func New(cfg *config.ZipServing) vfs.VFS { - fmt.Printf("\n\n\n what the config?: %+v\n\n\n", cfg) zipVFS := &zipVFS{ cacheExpirationInterval: cfg.ExpirationInterval, cacheRefreshInterval: cfg.RefreshInterval, diff --git a/internal/vfs/zip/vfs_test.go b/internal/vfs/zip/vfs_test.go index 0a8a5e0b..ffda1fb6 100644 --- a/internal/vfs/zip/vfs_test.go +++ b/internal/vfs/zip/vfs_test.go @@ -98,7 +98,7 @@ func TestVFSFindOrOpenArchiveConcurrentAccess(t *testing.T) { require.Eventually(t, func() bool { _, err := vfs.findOrOpenArchive(context.Background(), path, path) return err == errAlreadyCached - }, time.Second, time.Nanosecond) + }, 3*time.Second, time.Nanosecond) } func TestVFSFindOrOpenArchiveRefresh(t *testing.T) { -- GitLab From b0d592dc5cd8969893a542a10cc655bed069deaf Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Sun, 22 Nov 2020 01:34:51 +0000 Subject: [PATCH 09/14] Apply 1 suggestion(s) to 1 file(s) --- main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/main.go b/main.go index f8c1cfaf..99ae9460 100644 --- a/main.go +++ b/main.go @@ -325,7 +325,6 @@ func appMain() { flag.Parse() - fmt.Printf("appMain: CONFIG: %+v\n", cfg.Default.Zip) if err := tlsconfig.ValidateTLSVersions(*tlsMinVersion, *tlsMaxVersion); err != nil { fatal(err, "invalid TLS version") } -- GitLab From fd653cb8b358b010d08a10215827598d87be8911 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Sun, 22 Nov 2020 17:17:59 +1100 Subject: [PATCH 10/14] Fix import --- main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/main.go b/main.go index 99ae9460..ef9e6aa9 100644 --- a/main.go +++ b/main.go @@ -16,7 +16,6 @@ import ( "gitlab.com/gitlab-org/labkit/errortracking" - cfg "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/logging" "gitlab.com/gitlab-org/gitlab-pages/internal/request" "gitlab.com/gitlab-org/gitlab-pages/internal/tlsconfig" -- GitLab From aeff2242f8b8fe51bb02539bf73b90bb6cf2ed1b Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Fri, 27 Nov 2020 11:34:36 +1100 Subject: [PATCH 11/14] Use sync.Once instead of checking for nil --- internal/serving/disk/zip/serving.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/serving/disk/zip/serving.go b/internal/serving/disk/zip/serving.go index b552050c..61d186da 100644 --- a/internal/serving/disk/zip/serving.go +++ b/internal/serving/disk/zip/serving.go @@ -1,6 +1,8 @@ package zip import ( + "sync" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/serving" "gitlab.com/gitlab-org/gitlab-pages/internal/serving/disk" @@ -9,13 +11,14 @@ import ( ) var instance serving.Serving +var once sync.Once // Instance returns a serving instance that is capable of reading files // from a zip archives opened from a URL, most likely stored in object storage func Instance() serving.Serving { - if instance == nil { + once.Do(func() { instance = disk.New(vfs.Instrumented(zip.New(config.Default.Zip))) - } + }) return instance } -- GitLab From 9c51d46d39c1a2614aafca02bf1f3376a88e0280 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Mon, 30 Nov 2020 11:09:30 +1100 Subject: [PATCH 12/14] Add Reconfigure to serving and VFS interfaces --- internal/serving/disk/serving.go | 6 +++++ internal/serving/disk/zip/serving.go | 12 ++++++--- internal/serving/serverless/serverless.go | 6 +++++ internal/serving/serving.go | 3 +++ internal/vfs/local/vfs.go | 6 +++++ internal/vfs/vfs.go | 6 +++++ internal/vfs/zip/vfs.go | 32 ++++++++++++++++++----- 7 files changed, 62 insertions(+), 9 deletions(-) diff --git a/internal/serving/disk/serving.go b/internal/serving/disk/serving.go index 30c821ea..fbcdf9f2 100644 --- a/internal/serving/disk/serving.go +++ b/internal/serving/disk/serving.go @@ -3,6 +3,7 @@ package disk import ( "os" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/httperrors" "gitlab.com/gitlab-org/gitlab-pages/internal/serving" "gitlab.com/gitlab-org/gitlab-pages/internal/vfs" @@ -40,6 +41,11 @@ func (s *Disk) ServeNotFoundHTTP(h serving.Handler) { httperrors.Serve404(h.Writer) } +// Reconfigure VFS +func (s *Disk) Reconfigure(cfg *config.Config) error { + return s.reader.vfs.Reconfigure(cfg) +} + // New returns a serving instance that is capable of reading files // from the VFS func New(vfs vfs.VFS) serving.Serving { diff --git a/internal/serving/disk/zip/serving.go b/internal/serving/disk/zip/serving.go index 61d186da..b31d9090 100644 --- a/internal/serving/disk/zip/serving.go +++ b/internal/serving/disk/zip/serving.go @@ -3,6 +3,8 @@ package zip import ( "sync" + "gitlab.com/gitlab-org/labkit/log" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/serving" "gitlab.com/gitlab-org/gitlab-pages/internal/serving/disk" @@ -10,14 +12,18 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/internal/vfs/zip" ) -var instance serving.Serving -var once sync.Once +var ( + once sync.Once + instance = disk.New(vfs.Instrumented(zip.New(config.Default.Zip))) +) // Instance returns a serving instance that is capable of reading files // from a zip archives opened from a URL, most likely stored in object storage func Instance() serving.Serving { once.Do(func() { - instance = disk.New(vfs.Instrumented(zip.New(config.Default.Zip))) + if err := instance.Reconfigure(config.Default); err != nil { + log.WithError(err).Fatal("failed to reconfigure zip serving") + } }) return instance diff --git a/internal/serving/serverless/serverless.go b/internal/serving/serverless/serverless.go index e1881362..f8bd4e87 100644 --- a/internal/serving/serverless/serverless.go +++ b/internal/serving/serverless/serverless.go @@ -4,6 +4,7 @@ import ( "errors" "net/http/httputil" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/httperrors" "gitlab.com/gitlab-org/gitlab-pages/internal/serving" "gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab/api" @@ -65,3 +66,8 @@ func (s *Serverless) ServeFileHTTP(h serving.Handler) bool { func (s *Serverless) ServeNotFoundHTTP(h serving.Handler) { httperrors.Serve404(h.Writer) } + +// Reconfigure noop +func (s *Serverless) Reconfigure(*config.Config) error { + return nil +} diff --git a/internal/serving/serving.go b/internal/serving/serving.go index 6fde8216..786ee569 100644 --- a/internal/serving/serving.go +++ b/internal/serving/serving.go @@ -1,7 +1,10 @@ package serving +import "gitlab.com/gitlab-org/gitlab-pages/internal/config" + // Serving is an interface used to define a serving driver type Serving interface { ServeFileHTTP(Handler) bool ServeNotFoundHTTP(Handler) + Reconfigure(config *config.Config) error } diff --git a/internal/vfs/local/vfs.go b/internal/vfs/local/vfs.go index bdcd5160..ea54e8e8 100644 --- a/internal/vfs/local/vfs.go +++ b/internal/vfs/local/vfs.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/vfs" ) @@ -43,3 +44,8 @@ func (fs VFS) Root(ctx context.Context, path string) (vfs.Root, error) { func (fs *VFS) Name() string { return "local" } + +func (fs *VFS) Reconfigure(*config.Config) error { + // noop + return nil +} diff --git a/internal/vfs/vfs.go b/internal/vfs/vfs.go index 7bd51db2..2304f903 100644 --- a/internal/vfs/vfs.go +++ b/internal/vfs/vfs.go @@ -6,6 +6,7 @@ import ( log "github.com/sirupsen/logrus" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/metrics" ) @@ -13,6 +14,7 @@ import ( type VFS interface { Root(ctx context.Context, path string) (Root, error) Name() string + Reconfigure(config *config.Config) error } func Instrumented(fs VFS) VFS { @@ -50,3 +52,7 @@ func (i *instrumentedVFS) Root(ctx context.Context, path string) (Root, error) { func (i *instrumentedVFS) Name() string { return i.fs.Name() } + +func (i *instrumentedVFS) Reconfigure(cfg *config.Config) error { + return i.fs.Reconfigure(cfg) +} diff --git a/internal/vfs/zip/vfs.go b/internal/vfs/zip/vfs.go index d90d4796..b27424c6 100644 --- a/internal/vfs/zip/vfs.go +++ b/internal/vfs/zip/vfs.go @@ -56,12 +56,7 @@ func New(cfg *config.ZipServing) vfs.VFS { openTimeout: cfg.OpenTimeout, } - zipVFS.cache = cache.New(zipVFS.cacheExpirationInterval, zipVFS.cacheCleanupInterval) - zipVFS.cache.OnEvicted(func(s string, i interface{}) { - metrics.ZipCachedEntries.WithLabelValues("archive").Dec() - - i.(*zipArchive).onEvicted() - }) + zipVFS.resetCache() // TODO: To be removed with https://gitlab.com/gitlab-org/gitlab-pages/-/issues/480 zipVFS.dataOffsetCache = newLruCache("data-offset", defaultDataOffsetItems, defaultDataOffsetExpirationInterval) @@ -70,6 +65,31 @@ func New(cfg *config.ZipServing) vfs.VFS { return zipVFS } +// Reconfigure will update the zipVFS configuration values and will reset the +// cache +func (fs *zipVFS) Reconfigure(cfg *config.Config) error { + fs.cacheLock.Lock() + defer fs.cacheLock.Unlock() + + fs.openTimeout = cfg.Zip.OpenTimeout + fs.cacheExpirationInterval = cfg.Zip.ExpirationInterval + fs.cacheRefreshInterval = cfg.Zip.RefreshInterval + fs.cacheCleanupInterval = cfg.Zip.CleanupInterval + + fs.resetCache() + + return nil +} + +func (fs *zipVFS) resetCache() { + fs.cache = cache.New(fs.cacheExpirationInterval, fs.cacheCleanupInterval) + fs.cache.OnEvicted(func(s string, i interface{}) { + metrics.ZipCachedEntries.WithLabelValues("archive").Dec() + + i.(*zipArchive).onEvicted() + }) +} + func (fs *zipVFS) keyFromPath(path string) (string, error) { // We assume that our URL is https://.../artifacts.zip?content-sign=aaa // our caching key is `https://.../artifacts.zip` -- GitLab From 14e61effff77b6dde9effc665a13c82bec509067 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Mon, 30 Nov 2020 21:57:22 +1100 Subject: [PATCH 13/14] Call reconfigure explicitly --- app.go | 5 +++++ internal/serving/disk/zip/serving.go | 15 +-------------- internal/serving/disk/zip/serving_test.go | 2 ++ 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/app.go b/app.go index 5468036d..ccd107fb 100644 --- a/app.go +++ b/app.go @@ -29,6 +29,7 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/internal/middleware" "gitlab.com/gitlab-org/gitlab-pages/internal/netutil" "gitlab.com/gitlab-org/gitlab-pages/internal/request" + "gitlab.com/gitlab-org/gitlab-pages/internal/serving/disk/zip" "gitlab.com/gitlab-org/gitlab-pages/internal/source" "gitlab.com/gitlab-org/gitlab-pages/internal/tlsconfig" "gitlab.com/gitlab-org/gitlab-pages/metrics" @@ -510,6 +511,10 @@ func runApp(config appConfig) { cfg.Default.Zip.RefreshInterval = config.ZipCacheRefresh cfg.Default.Zip.OpenTimeout = config.ZipeOpenTimeout + if err := zip.Instance().Reconfigure(cfg.Default); err != nil { + fatal(err, "failed to reconfigure zip VFS") + } + a.Run() } diff --git a/internal/serving/disk/zip/serving.go b/internal/serving/disk/zip/serving.go index b31d9090..6d21f771 100644 --- a/internal/serving/disk/zip/serving.go +++ b/internal/serving/disk/zip/serving.go @@ -1,10 +1,6 @@ package zip import ( - "sync" - - "gitlab.com/gitlab-org/labkit/log" - "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/serving" "gitlab.com/gitlab-org/gitlab-pages/internal/serving/disk" @@ -12,19 +8,10 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/internal/vfs/zip" ) -var ( - once sync.Once - instance = disk.New(vfs.Instrumented(zip.New(config.Default.Zip))) -) +var instance = disk.New(vfs.Instrumented(zip.New(config.Default.Zip))) // Instance returns a serving instance that is capable of reading files // from a zip archives opened from a URL, most likely stored in object storage func Instance() serving.Serving { - once.Do(func() { - if err := instance.Reconfigure(config.Default); err != nil { - log.WithError(err).Fatal("failed to reconfigure zip serving") - } - }) - return instance } diff --git a/internal/serving/disk/zip/serving_test.go b/internal/serving/disk/zip/serving_test.go index 5bf16d7b..40b8e02a 100644 --- a/internal/serving/disk/zip/serving_test.go +++ b/internal/serving/disk/zip/serving_test.go @@ -62,6 +62,8 @@ func TestZip_ServeFileHTTP(t *testing.T) { OpenTimeout: 5 * time.Second, } s := Instance() + err := s.Reconfigure(config.Default) + require.NoError(t, err) for name, test := range tests { t.Run(name, func(t *testing.T) { -- GitLab From d13acbb585182980c0fcdba72df26001fafbce38 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Mon, 30 Nov 2020 22:35:06 +1100 Subject: [PATCH 14/14] Remove default config Add todo to reconfigure other vfs --- app.go | 16 +++++++++++----- internal/config/config.go | 5 ----- internal/serving/disk/zip/serving.go | 2 +- internal/serving/disk/zip/serving_test.go | 15 +++++++++------ 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/app.go b/app.go index ccd107fb..c411470b 100644 --- a/app.go +++ b/app.go @@ -506,12 +506,18 @@ func runApp(config appConfig) { log.WithError(err).Warn("Loading extended MIME database failed") } - cfg.Default.Zip.ExpirationInterval = config.ZipCacheExpiry - cfg.Default.Zip.CleanupInterval = config.ZipCacheCleanup - cfg.Default.Zip.RefreshInterval = config.ZipCacheRefresh - cfg.Default.Zip.OpenTimeout = config.ZipeOpenTimeout + c := &cfg.Config{ + Zip: &cfg.ZipServing{ + ExpirationInterval: config.ZipCacheExpiry, + CleanupInterval: config.ZipCacheCleanup, + RefreshInterval: config.ZipCacheRefresh, + OpenTimeout: config.ZipeOpenTimeout, + }, + } - if err := zip.Instance().Reconfigure(cfg.Default); err != nil { + // TODO: reconfigure all VFS' + // https://gitlab.com/gitlab-org/gitlab-pages/-/issues/512 + if err := zip.Instance().Reconfigure(c); err != nil { fatal(err, "failed to reconfigure zip VFS") } diff --git a/internal/config/config.go b/internal/config/config.go index 66f1a2b9..c52beef8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,11 +4,6 @@ import ( "time" ) -// Default configuration that can be accessed by different packages -var Default = &Config{ - Zip: &ZipServing{}, -} - type Config struct { Zip *ZipServing } diff --git a/internal/serving/disk/zip/serving.go b/internal/serving/disk/zip/serving.go index 6d21f771..6db0be10 100644 --- a/internal/serving/disk/zip/serving.go +++ b/internal/serving/disk/zip/serving.go @@ -8,7 +8,7 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/internal/vfs/zip" ) -var instance = disk.New(vfs.Instrumented(zip.New(config.Default.Zip))) +var instance = disk.New(vfs.Instrumented(zip.New(&config.ZipServing{}))) // Instance returns a serving instance that is capable of reading files // from a zip archives opened from a URL, most likely stored in object storage diff --git a/internal/serving/disk/zip/serving_test.go b/internal/serving/disk/zip/serving_test.go index 40b8e02a..e64a761a 100644 --- a/internal/serving/disk/zip/serving_test.go +++ b/internal/serving/disk/zip/serving_test.go @@ -55,14 +55,17 @@ func TestZip_ServeFileHTTP(t *testing.T) { }, } - config.Default.Zip = &config.ZipServing{ - ExpirationInterval: 10 * time.Second, - CleanupInterval: 5 * time.Second, - RefreshInterval: 5 * time.Second, - OpenTimeout: 5 * time.Second, + cfg := &config.Config{ + Zip: &config.ZipServing{ + ExpirationInterval: 10 * time.Second, + CleanupInterval: 5 * time.Second, + RefreshInterval: 5 * time.Second, + OpenTimeout: 5 * time.Second, + }, } + s := Instance() - err := s.Reconfigure(config.Default) + err := s.Reconfigure(cfg) require.NoError(t, err) for name, test := range tests { -- GitLab