Skip to content
Snippets Groups Projects
Commit b4c2abe9 authored by Stan Hu's avatar Stan Hu
Browse files

Add Azure Blob storage cache adapter and documentation

This commit makes it possible to use Azure Blob Storage as a runner
cache. Currently customers wanting to use Azure Blob Storage have to set
up a Minio Gateway, which isn't ideal because it requires customers to
maintain their own proxy server for Azure. We have a number of customers
who want native support for Azure Blob Storage.

Azure Blob storage will be fully supported for storing LFS, CI
artifacts, uploads, packages, etc. in 13.4
(gitlab#25877).
parent 53ff2e10
No related branches found
No related tags found
No related merge requests found
Pipeline #185566695 failed
Showing
with 4973 additions and 0 deletions
package azblob
import (
"bytes"
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"errors"
"net/http"
"net/url"
"sort"
"strings"
"time"
"github.com/Azure/azure-pipeline-go/pipeline"
)
// NewSharedKeyCredential creates an immutable SharedKeyCredential containing the
// storage account's name and either its primary or secondary key.
func NewSharedKeyCredential(accountName, accountKey string) (*SharedKeyCredential, error) {
bytes, err := base64.StdEncoding.DecodeString(accountKey)
if err != nil {
return &SharedKeyCredential{}, err
}
return &SharedKeyCredential{accountName: accountName, accountKey: bytes}, nil
}
// SharedKeyCredential contains an account's name and its primary or secondary key.
// It is immutable making it shareable and goroutine-safe.
type SharedKeyCredential struct {
// Only the NewSharedKeyCredential method should set these; all other methods should treat them as read-only
accountName string
accountKey []byte
}
// AccountName returns the Storage account's name.
func (f SharedKeyCredential) AccountName() string {
return f.accountName
}
func (f SharedKeyCredential) getAccountKey() []byte {
return f.accountKey
}
// noop function to satisfy StorageAccountCredential interface
func (f SharedKeyCredential) getUDKParams() *UserDelegationKey {
return nil
}
// New creates a credential policy object.
func (f *SharedKeyCredential) New(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.Policy {
return pipeline.PolicyFunc(func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) {
// Add a x-ms-date header if it doesn't already exist
if d := request.Header.Get(headerXmsDate); d == "" {
request.Header[headerXmsDate] = []string{time.Now().UTC().Format(http.TimeFormat)}
}
stringToSign, err := f.buildStringToSign(request)
if err != nil {
return nil, err
}
signature := f.ComputeHMACSHA256(stringToSign)
authHeader := strings.Join([]string{"SharedKey ", f.accountName, ":", signature}, "")
request.Header[headerAuthorization] = []string{authHeader}
response, err := next.Do(ctx, request)
if err != nil && response != nil && response.Response() != nil && response.Response().StatusCode == http.StatusForbidden {
// Service failed to authenticate request, log it
po.Log(pipeline.LogError, "===== HTTP Forbidden status, String-to-Sign:\n"+stringToSign+"\n===============================\n")
}
return response, err
})
}
// credentialMarker is a package-internal method that exists just to satisfy the Credential interface.
func (*SharedKeyCredential) credentialMarker() {}
// Constants ensuring that header names are correctly spelled and consistently cased.
const (
headerAuthorization = "Authorization"
headerCacheControl = "Cache-Control"
headerContentEncoding = "Content-Encoding"
headerContentDisposition = "Content-Disposition"
headerContentLanguage = "Content-Language"
headerContentLength = "Content-Length"
headerContentMD5 = "Content-MD5"
headerContentType = "Content-Type"
headerDate = "Date"
headerIfMatch = "If-Match"
headerIfModifiedSince = "If-Modified-Since"
headerIfNoneMatch = "If-None-Match"
headerIfUnmodifiedSince = "If-Unmodified-Since"
headerRange = "Range"
headerUserAgent = "User-Agent"
headerXmsDate = "x-ms-date"
headerXmsVersion = "x-ms-version"
)
// ComputeHMACSHA256 generates a hash signature for an HTTP request or for a SAS.
func (f SharedKeyCredential) ComputeHMACSHA256(message string) (base64String string) {
h := hmac.New(sha256.New, f.accountKey)
h.Write([]byte(message))
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
func (f *SharedKeyCredential) buildStringToSign(request pipeline.Request) (string, error) {
// https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
headers := request.Header
contentLength := headers.Get(headerContentLength)
if contentLength == "0" {
contentLength = ""
}
canonicalizedResource, err := f.buildCanonicalizedResource(request.URL)
if err != nil {
return "", err
}
stringToSign := strings.Join([]string{
request.Method,
headers.Get(headerContentEncoding),
headers.Get(headerContentLanguage),
contentLength,
headers.Get(headerContentMD5),
headers.Get(headerContentType),
"", // Empty date because x-ms-date is expected (as per web page above)
headers.Get(headerIfModifiedSince),
headers.Get(headerIfMatch),
headers.Get(headerIfNoneMatch),
headers.Get(headerIfUnmodifiedSince),
headers.Get(headerRange),
buildCanonicalizedHeader(headers),
canonicalizedResource,
}, "\n")
return stringToSign, nil
}
func buildCanonicalizedHeader(headers http.Header) string {
cm := map[string][]string{}
for k, v := range headers {
headerName := strings.TrimSpace(strings.ToLower(k))
if strings.HasPrefix(headerName, "x-ms-") {
cm[headerName] = v // NOTE: the value must not have any whitespace around it.
}
}
if len(cm) == 0 {
return ""
}
keys := make([]string, 0, len(cm))
for key := range cm {
keys = append(keys, key)
}
sort.Strings(keys)
ch := bytes.NewBufferString("")
for i, key := range keys {
if i > 0 {
ch.WriteRune('\n')
}
ch.WriteString(key)
ch.WriteRune(':')
ch.WriteString(strings.Join(cm[key], ","))
}
return string(ch.Bytes())
}
func (f *SharedKeyCredential) buildCanonicalizedResource(u *url.URL) (string, error) {
// https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
cr := bytes.NewBufferString("/")
cr.WriteString(f.accountName)
if len(u.Path) > 0 {
// Any portion of the CanonicalizedResource string that is derived from
// the resource's URI should be encoded exactly as it is in the URI.
// -- https://msdn.microsoft.com/en-gb/library/azure/dd179428.aspx
cr.WriteString(u.EscapedPath())
} else {
// a slash is required to indicate the root path
cr.WriteString("/")
}
// params is a map[string][]string; param name is key; params values is []string
params, err := url.ParseQuery(u.RawQuery) // Returns URL decoded values
if err != nil {
return "", errors.New("parsing query parameters must succeed, otherwise there might be serious problems in the SDK/generated code")
}
if len(params) > 0 { // There is at least 1 query parameter
paramNames := []string{} // We use this to sort the parameter key names
for paramName := range params {
paramNames = append(paramNames, paramName) // paramNames must be lowercase
}
sort.Strings(paramNames)
for _, paramName := range paramNames {
paramValues := params[paramName]
sort.Strings(paramValues)
// Join the sorted key values separated by ','
// Then prepend "keyName:"; then add this string to the buffer
cr.WriteString("\n" + paramName + ":" + strings.Join(paramValues, ","))
}
}
return string(cr.Bytes()), nil
}
package azblob
import (
"context"
"errors"
"sync/atomic"
"runtime"
"sync"
"time"
"github.com/Azure/azure-pipeline-go/pipeline"
)
// TokenRefresher represents a callback method that you write; this method is called periodically
// so you can refresh the token credential's value.
type TokenRefresher func(credential TokenCredential) time.Duration
// TokenCredential represents a token credential (which is also a pipeline.Factory).
type TokenCredential interface {
Credential
Token() string
SetToken(newToken string)
}
// NewTokenCredential creates a token credential for use with role-based access control (RBAC) access to Azure Storage
// resources. You initialize the TokenCredential with an initial token value. If you pass a non-nil value for
// tokenRefresher, then the function you pass will be called immediately so it can refresh and change the
// TokenCredential's token value by calling SetToken. Your tokenRefresher function must return a time.Duration
// indicating how long the TokenCredential object should wait before calling your tokenRefresher function again.
// If your tokenRefresher callback fails to refresh the token, you can return a duration of 0 to stop your
// TokenCredential object from ever invoking tokenRefresher again. Also, oen way to deal with failing to refresh a
// token is to cancel a context.Context object used by requests that have the TokenCredential object in their pipeline.
func NewTokenCredential(initialToken string, tokenRefresher TokenRefresher) TokenCredential {
tc := &tokenCredential{}
tc.SetToken(initialToken) // We don't set it above to guarantee atomicity
if tokenRefresher == nil {
return tc // If no callback specified, return the simple tokenCredential
}
tcwr := &tokenCredentialWithRefresh{token: tc}
tcwr.token.startRefresh(tokenRefresher)
runtime.SetFinalizer(tcwr, func(deadTC *tokenCredentialWithRefresh) {
deadTC.token.stopRefresh()
deadTC.token = nil // Sanity (not really required)
})
return tcwr
}
// tokenCredentialWithRefresh is a wrapper over a token credential.
// When this wrapper object gets GC'd, it stops the tokenCredential's timer
// which allows the tokenCredential object to also be GC'd.
type tokenCredentialWithRefresh struct {
token *tokenCredential
}
// credentialMarker is a package-internal method that exists just to satisfy the Credential interface.
func (*tokenCredentialWithRefresh) credentialMarker() {}
// Token returns the current token value
func (f *tokenCredentialWithRefresh) Token() string { return f.token.Token() }
// SetToken changes the current token value
func (f *tokenCredentialWithRefresh) SetToken(token string) { f.token.SetToken(token) }
// New satisfies pipeline.Factory's New method creating a pipeline policy object.
func (f *tokenCredentialWithRefresh) New(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.Policy {
return f.token.New(next, po)
}
///////////////////////////////////////////////////////////////////////////////
// tokenCredential is a pipeline.Factory is the credential's policy factory.
type tokenCredential struct {
token atomic.Value
// The members below are only used if the user specified a tokenRefresher callback function.
timer *time.Timer
tokenRefresher TokenRefresher
lock sync.Mutex
stopped bool
}
// credentialMarker is a package-internal method that exists just to satisfy the Credential interface.
func (*tokenCredential) credentialMarker() {}
// Token returns the current token value
func (f *tokenCredential) Token() string { return f.token.Load().(string) }
// SetToken changes the current token value
func (f *tokenCredential) SetToken(token string) { f.token.Store(token) }
// startRefresh calls refresh which immediately calls tokenRefresher
// and then starts a timer to call tokenRefresher in the future.
func (f *tokenCredential) startRefresh(tokenRefresher TokenRefresher) {
f.tokenRefresher = tokenRefresher
f.stopped = false // In case user calls StartRefresh, StopRefresh, & then StartRefresh again
f.refresh()
}
// refresh calls the user's tokenRefresher so they can refresh the token (by
// calling SetToken) and then starts another time (based on the returned duration)
// in order to refresh the token again in the future.
func (f *tokenCredential) refresh() {
d := f.tokenRefresher(f) // Invoke the user's refresh callback outside of the lock
if d > 0 { // If duration is 0 or negative, refresher wants to not be called again
f.lock.Lock()
if !f.stopped {
f.timer = time.AfterFunc(d, f.refresh)
}
f.lock.Unlock()
}
}
// stopRefresh stops any pending timer and sets stopped field to true to prevent
// any new timer from starting.
// NOTE: Stopping the timer allows the GC to destroy the tokenCredential object.
func (f *tokenCredential) stopRefresh() {
f.lock.Lock()
f.stopped = true
if f.timer != nil {
f.timer.Stop()
}
f.lock.Unlock()
}
// New satisfies pipeline.Factory's New method creating a pipeline policy object.
func (f *tokenCredential) New(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.Policy {
return pipeline.PolicyFunc(func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) {
if request.URL.Scheme != "https" {
// HTTPS must be used, otherwise the tokens are at the risk of being exposed
return nil, errors.New("token credentials require a URL using the https protocol scheme")
}
request.Header[headerAuthorization] = []string{"Bearer " + f.Token()}
return next.Do(ctx, request)
})
}
// +build linux darwin freebsd openbsd netbsd dragonfly solaris
package azblob
import (
"os"
"golang.org/x/sys/unix"
)
type mmf []byte
func newMMF(file *os.File, writable bool, offset int64, length int) (mmf, error) {
prot, flags := unix.PROT_READ, unix.MAP_SHARED // Assume read-only
if writable {
prot, flags = unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED
}
addr, err := unix.Mmap(int(file.Fd()), offset, length, prot, flags)
return mmf(addr), err
}
func (m *mmf) unmap() {
err := unix.Munmap(*m)
*m = nil
if err != nil {
panic("if we are unable to unmap the memory-mapped file, there is serious concern for memory corruption")
}
}
package azblob
import (
"os"
"reflect"
"syscall"
"unsafe"
)
type mmf []byte
func newMMF(file *os.File, writable bool, offset int64, length int) (mmf, error) {
prot, access := uint32(syscall.PAGE_READONLY), uint32(syscall.FILE_MAP_READ) // Assume read-only
if writable {
prot, access = uint32(syscall.PAGE_READWRITE), uint32(syscall.FILE_MAP_WRITE)
}
hMMF, errno := syscall.CreateFileMapping(syscall.Handle(file.Fd()), nil, prot, uint32(int64(length)>>32), uint32(int64(length)&0xffffffff), nil)
if hMMF == 0 {
return nil, os.NewSyscallError("CreateFileMapping", errno)
}
defer syscall.CloseHandle(hMMF)
addr, errno := syscall.MapViewOfFile(hMMF, access, uint32(offset>>32), uint32(offset&0xffffffff), uintptr(length))
m := mmf{}
h := (*reflect.SliceHeader)(unsafe.Pointer(&m))
h.Data = addr
h.Len = length
h.Cap = h.Len
return m, nil
}
func (m *mmf) unmap() {
addr := uintptr(unsafe.Pointer(&(([]byte)(*m)[0])))
*m = mmf{}
err := syscall.UnmapViewOfFile(addr)
if err != nil {
panic("if we are unable to unmap the memory-mapped file, there is serious concern for memory corruption")
}
}
package azblob
import (
"github.com/Azure/azure-pipeline-go/pipeline"
)
// PipelineOptions is used to configure a request policy pipeline's retry policy and logging.
type PipelineOptions struct {
// Log configures the pipeline's logging infrastructure indicating what information is logged and where.
Log pipeline.LogOptions
// Retry configures the built-in retry policy behavior.
Retry RetryOptions
// RequestLog configures the built-in request logging policy.
RequestLog RequestLogOptions
// Telemetry configures the built-in telemetry policy behavior.
Telemetry TelemetryOptions
// HTTPSender configures the sender of HTTP requests
HTTPSender pipeline.Factory
}
// NewPipeline creates a Pipeline using the specified credentials and options.
func NewPipeline(c Credential, o PipelineOptions) pipeline.Pipeline {
// Closest to API goes first; closest to the wire goes last
f := []pipeline.Factory{
NewTelemetryPolicyFactory(o.Telemetry),
NewUniqueRequestIDPolicyFactory(),
NewRetryPolicyFactory(o.Retry),
}
if _, ok := c.(*anonymousCredentialPolicyFactory); !ok {
// For AnonymousCredential, we optimize out the policy factory since it doesn't do anything
// NOTE: The credential's policy factory must appear close to the wire so it can sign any
// changes made by other factories (like UniqueRequestIDPolicyFactory)
f = append(f, c)
}
f = append(f,
NewRequestLogPolicyFactory(o.RequestLog),
pipeline.MethodFactoryMarker()) // indicates at what stage in the pipeline the method factory is invoked
return pipeline.NewPipeline(f, pipeline.Options{HTTPSender: o.HTTPSender, Log: o.Log})
}
package azblob
import (
"bytes"
"context"
"fmt"
"net/http"
"net/url"
"runtime"
"strings"
"time"
"github.com/Azure/azure-pipeline-go/pipeline"
)
// RequestLogOptions configures the retry policy's behavior.
type RequestLogOptions struct {
// LogWarningIfTryOverThreshold logs a warning if a tried operation takes longer than the specified
// duration (-1=no logging; 0=default threshold).
LogWarningIfTryOverThreshold time.Duration
}
func (o RequestLogOptions) defaults() RequestLogOptions {
if o.LogWarningIfTryOverThreshold == 0 {
// It would be good to relate this to https://azure.microsoft.com/en-us/support/legal/sla/storage/v1_2/
// But this monitors the time to get the HTTP response; NOT the time to download the response body.
o.LogWarningIfTryOverThreshold = 3 * time.Second // Default to 3 seconds
}
return o
}
// NewRequestLogPolicyFactory creates a RequestLogPolicyFactory object configured using the specified options.
func NewRequestLogPolicyFactory(o RequestLogOptions) pipeline.Factory {
o = o.defaults() // Force defaults to be calculated
return pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc {
// These variables are per-policy; shared by multiple calls to Do
var try int32
operationStart := time.Now() // If this is the 1st try, record the operation state time
return func(ctx context.Context, request pipeline.Request) (response pipeline.Response, err error) {
try++ // The first try is #1 (not #0)
// Log the outgoing request as informational
if po.ShouldLog(pipeline.LogInfo) {
b := &bytes.Buffer{}
fmt.Fprintf(b, "==> OUTGOING REQUEST (Try=%d)\n", try)
pipeline.WriteRequestWithResponse(b, prepareRequestForLogging(request), nil, nil)
po.Log(pipeline.LogInfo, b.String())
}
// Set the time for this particular retry operation and then Do the operation.
tryStart := time.Now()
response, err = next.Do(ctx, request) // Make the request
tryEnd := time.Now()
tryDuration := tryEnd.Sub(tryStart)
opDuration := tryEnd.Sub(operationStart)
logLevel, forceLog := pipeline.LogInfo, false // Default logging information
// If the response took too long, we'll upgrade to warning.
if o.LogWarningIfTryOverThreshold > 0 && tryDuration > o.LogWarningIfTryOverThreshold {
// Log a warning if the try duration exceeded the specified threshold
logLevel, forceLog = pipeline.LogWarning, true
}
if err == nil { // We got a response from the service
sc := response.Response().StatusCode
if ((sc >= 400 && sc <= 499) && sc != http.StatusNotFound && sc != http.StatusConflict && sc != http.StatusPreconditionFailed && sc != http.StatusRequestedRangeNotSatisfiable) || (sc >= 500 && sc <= 599) {
logLevel, forceLog = pipeline.LogError, true // Promote to Error any 4xx (except those listed is an error) or any 5xx
} else {
// For other status codes, we leave the level as is.
}
} else { // This error did not get an HTTP response from the service; upgrade the severity to Error
logLevel, forceLog = pipeline.LogError, true
}
if shouldLog := po.ShouldLog(logLevel); forceLog || shouldLog {
// We're going to log this; build the string to log
b := &bytes.Buffer{}
slow := ""
if o.LogWarningIfTryOverThreshold > 0 && tryDuration > o.LogWarningIfTryOverThreshold {
slow = fmt.Sprintf("[SLOW >%v]", o.LogWarningIfTryOverThreshold)
}
fmt.Fprintf(b, "==> REQUEST/RESPONSE (Try=%d/%v%s, OpTime=%v) -- ", try, tryDuration, slow, opDuration)
if err != nil { // This HTTP request did not get a response from the service
fmt.Fprint(b, "REQUEST ERROR\n")
} else {
if logLevel == pipeline.LogError {
fmt.Fprint(b, "RESPONSE STATUS CODE ERROR\n")
} else {
fmt.Fprint(b, "RESPONSE SUCCESSFULLY RECEIVED\n")
}
}
pipeline.WriteRequestWithResponse(b, prepareRequestForLogging(request), response.Response(), err)
if logLevel <= pipeline.LogError {
b.Write(stack()) // For errors (or lower levels), we append the stack trace (an expensive operation)
}
msg := b.String()
if forceLog {
pipeline.ForceLog(logLevel, msg)
}
if shouldLog {
po.Log(logLevel, msg)
}
}
return response, err
}
})
}
// RedactSigQueryParam redacts the 'sig' query parameter in URL's raw query to protect secret.
func RedactSigQueryParam(rawQuery string) (bool, string) {
rawQuery = strings.ToLower(rawQuery) // lowercase the string so we can look for ?sig= and &sig=
sigFound := strings.Contains(rawQuery, "?sig=")
if !sigFound {
sigFound = strings.Contains(rawQuery, "&sig=")
if !sigFound {
return sigFound, rawQuery // [?|&]sig= not found; return same rawQuery passed in (no memory allocation)
}
}
// [?|&]sig= found, redact its value
values, _ := url.ParseQuery(rawQuery)
for name := range values {
if strings.EqualFold(name, "sig") {
values[name] = []string{"REDACTED"}
}
}
return sigFound, values.Encode()
}
func prepareRequestForLogging(request pipeline.Request) *http.Request {
req := request
if sigFound, rawQuery := RedactSigQueryParam(req.URL.RawQuery); sigFound {
// Make copy so we don't destroy the query parameters we actually need to send in the request
req = request.Copy()
req.Request.URL.RawQuery = rawQuery
}
return prepareRequestForServiceLogging(req)
}
func stack() []byte {
buf := make([]byte, 1024)
for {
n := runtime.Stack(buf, false)
if n < len(buf) {
return buf[:n]
}
buf = make([]byte, 2*len(buf))
}
}
///////////////////////////////////////////////////////////////////////////////////////
// Redact phase useful for blob and file service only. For other services,
// this method can directly return request.Request.
///////////////////////////////////////////////////////////////////////////////////////
func prepareRequestForServiceLogging(request pipeline.Request) *http.Request {
req := request
if exist, key := doesHeaderExistCaseInsensitive(req.Header, xMsCopySourceHeader); exist {
req = request.Copy()
url, err := url.Parse(req.Header.Get(key))
if err == nil {
if sigFound, rawQuery := RedactSigQueryParam(url.RawQuery); sigFound {
url.RawQuery = rawQuery
req.Header.Set(xMsCopySourceHeader, url.String())
}
}
}
return req.Request
}
const xMsCopySourceHeader = "x-ms-copy-source"
func doesHeaderExistCaseInsensitive(header http.Header, key string) (bool, string) {
for keyInHeader := range header {
if strings.EqualFold(keyInHeader, key) {
return true, keyInHeader
}
}
return false, ""
}
package azblob
import (
"context"
"errors"
"io"
"io/ioutil"
"math/rand"
"net"
"net/http"
"strconv"
"strings"
"time"
"github.com/Azure/azure-pipeline-go/pipeline"
)
// RetryPolicy tells the pipeline what kind of retry policy to use. See the RetryPolicy* constants.
type RetryPolicy int32
const (
// RetryPolicyExponential tells the pipeline to use an exponential back-off retry policy
RetryPolicyExponential RetryPolicy = 0
// RetryPolicyFixed tells the pipeline to use a fixed back-off retry policy
RetryPolicyFixed RetryPolicy = 1
)
// RetryOptions configures the retry policy's behavior.
type RetryOptions struct {
// Policy tells the pipeline what kind of retry policy to use. See the RetryPolicy* constants.\
// A value of zero means that you accept our default policy.
Policy RetryPolicy
// MaxTries specifies the maximum number of attempts an operation will be tried before producing an error (0=default).
// A value of zero means that you accept our default policy. A value of 1 means 1 try and no retries.
MaxTries int32
// TryTimeout indicates the maximum time allowed for any single try of an HTTP request.
// A value of zero means that you accept our default timeout. NOTE: When transferring large amounts
// of data, the default TryTimeout will probably not be sufficient. You should override this value
// based on the bandwidth available to the host machine and proximity to the Storage service. A good
// starting point may be something like (60 seconds per MB of anticipated-payload-size).
TryTimeout time.Duration
// RetryDelay specifies the amount of delay to use before retrying an operation (0=default).
// When RetryPolicy is specified as RetryPolicyExponential, the delay increases exponentially
// with each retry up to a maximum specified by MaxRetryDelay.
// If you specify 0, then you must also specify 0 for MaxRetryDelay.
// If you specify RetryDelay, then you must also specify MaxRetryDelay, and MaxRetryDelay should be
// equal to or greater than RetryDelay.
RetryDelay time.Duration
// MaxRetryDelay specifies the maximum delay allowed before retrying an operation (0=default).
// If you specify 0, then you must also specify 0 for RetryDelay.
MaxRetryDelay time.Duration
// RetryReadsFromSecondaryHost specifies whether the retry policy should retry a read operation against another host.
// If RetryReadsFromSecondaryHost is "" (the default) then operations are not retried against another host.
// NOTE: Before setting this field, make sure you understand the issues around reading stale & potentially-inconsistent
// data at this webpage: https://docs.microsoft.com/en-us/azure/storage/common/storage-designing-ha-apps-with-ragrs
RetryReadsFromSecondaryHost string // Comment this our for non-Blob SDKs
}
func (o RetryOptions) retryReadsFromSecondaryHost() string {
return o.RetryReadsFromSecondaryHost // This is for the Blob SDK only
//return "" // This is for non-blob SDKs
}
func (o RetryOptions) defaults() RetryOptions {
// We assume the following:
// 1. o.Policy should either be RetryPolicyExponential or RetryPolicyFixed
// 2. o.MaxTries >= 0
// 3. o.TryTimeout, o.RetryDelay, and o.MaxRetryDelay >=0
// 4. o.RetryDelay <= o.MaxRetryDelay
// 5. Both o.RetryDelay and o.MaxRetryDelay must be 0 or neither can be 0
IfDefault := func(current *time.Duration, desired time.Duration) {
if *current == time.Duration(0) {
*current = desired
}
}
// Set defaults if unspecified
if o.MaxTries == 0 {
o.MaxTries = 4
}
switch o.Policy {
case RetryPolicyExponential:
IfDefault(&o.TryTimeout, 1*time.Minute)
IfDefault(&o.RetryDelay, 4*time.Second)
IfDefault(&o.MaxRetryDelay, 120*time.Second)
case RetryPolicyFixed:
IfDefault(&o.TryTimeout, 1*time.Minute)
IfDefault(&o.RetryDelay, 30*time.Second)
IfDefault(&o.MaxRetryDelay, 120*time.Second)
}
return o
}
func (o RetryOptions) calcDelay(try int32) time.Duration { // try is >=1; never 0
pow := func(number int64, exponent int32) int64 { // pow is nested helper function
var result int64 = 1
for n := int32(0); n < exponent; n++ {
result *= number
}
return result
}
delay := time.Duration(0)
switch o.Policy {
case RetryPolicyExponential:
delay = time.Duration(pow(2, try-1)-1) * o.RetryDelay
case RetryPolicyFixed:
if try > 1 { // Any try after the 1st uses the fixed delay
delay = o.RetryDelay
}
}
// Introduce some jitter: [0.0, 1.0) / 2 = [0.0, 0.5) + 0.8 = [0.8, 1.3)
// For casts and rounding - be careful, as per https://github.com/golang/go/issues/20757
delay = time.Duration(float32(delay) * (rand.Float32()/2 + 0.8)) // NOTE: We want math/rand; not crypto/rand
if delay > o.MaxRetryDelay {
delay = o.MaxRetryDelay
}
return delay
}
// NewRetryPolicyFactory creates a RetryPolicyFactory object configured using the specified options.
func NewRetryPolicyFactory(o RetryOptions) pipeline.Factory {
o = o.defaults() // Force defaults to be calculated
return pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc {
return func(ctx context.Context, request pipeline.Request) (response pipeline.Response, err error) {
// Before each try, we'll select either the primary or secondary URL.
primaryTry := int32(0) // This indicates how many tries we've attempted against the primary DC
// We only consider retrying against a secondary if we have a read request (GET/HEAD) AND this policy has a Secondary URL it can use
considerSecondary := (request.Method == http.MethodGet || request.Method == http.MethodHead) && o.retryReadsFromSecondaryHost() != ""
// Exponential retry algorithm: ((2 ^ attempt) - 1) * delay * random(0.8, 1.2)
// When to retry: connection failure or temporary/timeout. NOTE: StorageError considers HTTP 500/503 as temporary & is therefore retryable
// If using a secondary:
// Even tries go against primary; odd tries go against the secondary
// For a primary wait ((2 ^ primaryTries - 1) * delay * random(0.8, 1.2)
// If secondary gets a 404, don't fail, retry but future retries are only against the primary
// When retrying against a secondary, ignore the retry count and wait (.1 second * random(0.8, 1.2))
for try := int32(1); try <= o.MaxTries; try++ {
logf("\n=====> Try=%d\n", try)
// Determine which endpoint to try. It's primary if there is no secondary or if it is an add # attempt.
tryingPrimary := !considerSecondary || (try%2 == 1)
// Select the correct host and delay
if tryingPrimary {
primaryTry++
delay := o.calcDelay(primaryTry)
logf("Primary try=%d, Delay=%v\n", primaryTry, delay)
time.Sleep(delay) // The 1st try returns 0 delay
} else {
// For casts and rounding - be careful, as per https://github.com/golang/go/issues/20757
delay := time.Duration(float32(time.Second) * (rand.Float32()/2 + 0.8))
logf("Secondary try=%d, Delay=%v\n", try-primaryTry, delay)
time.Sleep(delay) // Delay with some jitter before trying secondary
}
// Clone the original request to ensure that each try starts with the original (unmutated) request.
requestCopy := request.Copy()
// For each try, seek to the beginning of the Body stream. We do this even for the 1st try because
// the stream may not be at offset 0 when we first get it and we want the same behavior for the
// 1st try as for additional tries.
err = requestCopy.RewindBody()
if err != nil {
return nil, errors.New("we must be able to seek on the Body Stream, otherwise retries would cause data corruption")
}
if !tryingPrimary {
requestCopy.URL.Host = o.retryReadsFromSecondaryHost()
requestCopy.Host = o.retryReadsFromSecondaryHost()
}
// Set the server-side timeout query parameter "timeout=[seconds]"
timeout := int32(o.TryTimeout.Seconds()) // Max seconds per try
if deadline, ok := ctx.Deadline(); ok { // If user's ctx has a deadline, make the timeout the smaller of the two
t := int32(deadline.Sub(time.Now()).Seconds()) // Duration from now until user's ctx reaches its deadline
logf("MaxTryTimeout=%d secs, TimeTilDeadline=%d sec\n", timeout, t)
if t < timeout {
timeout = t
}
if timeout < 0 {
timeout = 0 // If timeout ever goes negative, set it to zero; this happen while debugging
}
logf("TryTimeout adjusted to=%d sec\n", timeout)
}
q := requestCopy.Request.URL.Query()
q.Set("timeout", strconv.Itoa(int(timeout+1))) // Add 1 to "round up"
requestCopy.Request.URL.RawQuery = q.Encode()
logf("Url=%s\n", requestCopy.Request.URL.String())
// Set the time for this particular retry operation and then Do the operation.
tryCtx, tryCancel := context.WithTimeout(ctx, time.Second*time.Duration(timeout))
//requestCopy.Body = &deadlineExceededReadCloser{r: requestCopy.Request.Body}
response, err = next.Do(tryCtx, requestCopy) // Make the request
/*err = improveDeadlineExceeded(err)
if err == nil {
response.Response().Body = &deadlineExceededReadCloser{r: response.Response().Body}
}*/
logf("Err=%v, response=%v\n", err, response)
action := "" // This MUST get changed within the switch code below
switch {
case ctx.Err() != nil:
action = "NoRetry: Op timeout"
case !tryingPrimary && response != nil && response.Response() != nil && response.Response().StatusCode == http.StatusNotFound:
// If attempt was against the secondary & it returned a StatusNotFound (404), then
// the resource was not found. This may be due to replication delay. So, in this
// case, we'll never try the secondary again for this operation.
considerSecondary = false
action = "Retry: Secondary URL returned 404"
case err != nil:
// NOTE: Protocol Responder returns non-nil if REST API returns invalid status code for the invoked operation.
// Use ServiceCode to verify if the error is related to storage service-side,
// ServiceCode is set only when error related to storage service happened.
if stErr, ok := err.(StorageError); ok {
if stErr.Temporary() {
action = "Retry: StorageError with error service code and Temporary()"
} else if stErr.Response() != nil && isSuccessStatusCode(stErr.Response()) { // TODO: This is a temporarily work around, remove this after protocol layer fix the issue that net.Error is wrapped as storageError
action = "Retry: StorageError with success status code"
} else {
action = "NoRetry: StorageError not Temporary() and without retriable status code"
}
} else if netErr, ok := err.(net.Error); ok {
// Use non-retriable net.Error list, but not retriable list.
// As there are errors without Temporary() implementation,
// while need be retried, like 'connection reset by peer', 'transport connection broken' and etc.
// So the SDK do retry for most of the case, unless the error should not be retried for sure.
if !isNotRetriable(netErr) {
action = "Retry: net.Error and not in the non-retriable list"
} else {
action = "NoRetry: net.Error and in the non-retriable list"
}
} else if err == io.ErrUnexpectedEOF {
action = "Retry: unexpected EOF"
} else {
action = "NoRetry: unrecognized error"
}
default:
action = "NoRetry: successful HTTP request" // no error
}
logf("Action=%s\n", action)
// fmt.Println(action + "\n") // This is where we could log the retry operation; action is why we're retrying
if action[0] != 'R' { // Retry only if action starts with 'R'
if err != nil {
tryCancel() // If we're returning an error, cancel this current/last per-retry timeout context
} else {
// We wrap the last per-try context in a body and overwrite the Response's Body field with our wrapper.
// So, when the user closes the Body, the our per-try context gets closed too.
// Another option, is that the Last Policy do this wrapping for a per-retry context (not for the user's context)
if response == nil || response.Response() == nil {
// We do panic in the case response or response.Response() is nil,
// as for client, the response should not be nil if request is sent and the operations is executed successfully.
// Another option, is that execute the cancel function when response or response.Response() is nil,
// as in this case, current per-try has nothing to do in future.
return nil, errors.New("invalid state, response should not be nil when the operation is executed successfully")
}
response.Response().Body = &contextCancelReadCloser{cf: tryCancel, body: response.Response().Body}
}
break // Don't retry
}
if response != nil && response.Response() != nil && response.Response().Body != nil {
// If we're going to retry and we got a previous response, then flush its body to avoid leaking its TCP connection
body := response.Response().Body
io.Copy(ioutil.Discard, body)
body.Close()
}
// If retrying, cancel the current per-try timeout context
tryCancel()
}
return response, err // Not retryable or too many retries; return the last response/error
}
})
}
// contextCancelReadCloser helps to invoke context's cancelFunc properly when the ReadCloser is closed.
type contextCancelReadCloser struct {
cf context.CancelFunc
body io.ReadCloser
}
func (rc *contextCancelReadCloser) Read(p []byte) (n int, err error) {
return rc.body.Read(p)
}
func (rc *contextCancelReadCloser) Close() error {
err := rc.body.Close()
if rc.cf != nil {
rc.cf()
}
return err
}
// isNotRetriable checks if the provided net.Error isn't retriable.
func isNotRetriable(errToParse net.Error) bool {
// No error, so this is NOT retriable.
if errToParse == nil {
return true
}
// The error is either temporary or a timeout so it IS retriable (not not retriable).
if errToParse.Temporary() || errToParse.Timeout() {
return false
}
genericErr := error(errToParse)
// From here all the error are neither Temporary() nor Timeout().
switch err := errToParse.(type) {
case *net.OpError:
// The net.Error is also a net.OpError but the inner error is nil, so this is not retriable.
if err.Err == nil {
return true
}
genericErr = err.Err
}
switch genericErr.(type) {
case *net.AddrError, net.UnknownNetworkError, *net.DNSError, net.InvalidAddrError, *net.ParseError, *net.DNSConfigError:
// If the error is one of the ones listed, then it is NOT retriable.
return true
}
// If it's invalid header field name/value error thrown by http module, then it is NOT retriable.
// This could happen when metadata's key or value is invalid. (RoundTrip in transport.go)
if strings.Contains(genericErr.Error(), "invalid header field") {
return true
}
// Assume the error is retriable.
return false
}
var successStatusCodes = []int{http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent, http.StatusPartialContent}
func isSuccessStatusCode(resp *http.Response) bool {
if resp == nil {
return false
}
for _, i := range successStatusCodes {
if i == resp.StatusCode {
return true
}
}
return false
}
// According to https://github.com/golang/go/wiki/CompilerOptimizations, the compiler will inline this method and hopefully optimize all calls to it away
var logf = func(format string, a ...interface{}) {}
// Use this version to see the retry method's code path (import "fmt")
//var logf = fmt.Printf
/*
type deadlineExceededReadCloser struct {
r io.ReadCloser
}
func (r *deadlineExceededReadCloser) Read(p []byte) (int, error) {
n, err := 0, io.EOF
if r.r != nil {
n, err = r.r.Read(p)
}
return n, improveDeadlineExceeded(err)
}
func (r *deadlineExceededReadCloser) Seek(offset int64, whence int) (int64, error) {
// For an HTTP request, the ReadCloser MUST also implement seek
// For an HTTP response, Seek MUST not be called (or this will panic)
o, err := r.r.(io.Seeker).Seek(offset, whence)
return o, improveDeadlineExceeded(err)
}
func (r *deadlineExceededReadCloser) Close() error {
if c, ok := r.r.(io.Closer); ok {
c.Close()
}
return nil
}
// timeoutError is the internal struct that implements our richer timeout error.
type deadlineExceeded struct {
responseError
}
var _ net.Error = (*deadlineExceeded)(nil) // Ensure deadlineExceeded implements the net.Error interface at compile time
// improveDeadlineExceeded creates a timeoutError object that implements the error interface IF cause is a context.DeadlineExceeded error.
func improveDeadlineExceeded(cause error) error {
// If cause is not DeadlineExceeded, return the same error passed in.
if cause != context.DeadlineExceeded {
return cause
}
// Else, convert DeadlineExceeded to our timeoutError which gives a richer string message
return &deadlineExceeded{
responseError: responseError{
ErrorNode: pipeline.ErrorNode{}.Initialize(cause, 3),
},
}
}
// Error implements the error interface's Error method to return a string representation of the error.
func (e *deadlineExceeded) Error() string {
return e.ErrorNode.Error("context deadline exceeded; when creating a pipeline, consider increasing RetryOptions' TryTimeout field")
}
*/
package azblob
import (
"bytes"
"context"
"fmt"
"os"
"runtime"
"github.com/Azure/azure-pipeline-go/pipeline"
)
// TelemetryOptions configures the telemetry policy's behavior.
type TelemetryOptions struct {
// Value is a string prepended to each request's User-Agent and sent to the service.
// The service records the user-agent in logs for diagnostics and tracking of client requests.
Value string
}
// NewTelemetryPolicyFactory creates a factory that can create telemetry policy objects
// which add telemetry information to outgoing HTTP requests.
func NewTelemetryPolicyFactory(o TelemetryOptions) pipeline.Factory {
b := &bytes.Buffer{}
b.WriteString(o.Value)
if b.Len() > 0 {
b.WriteRune(' ')
}
fmt.Fprintf(b, "Azure-Storage/%s %s", serviceLibVersion, platformInfo)
telemetryValue := b.String()
return pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc {
return func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) {
request.Header.Set("User-Agent", telemetryValue)
return next.Do(ctx, request)
}
})
}
// NOTE: the ONLY function that should write to this variable is this func
var platformInfo = func() string {
// Azure-Storage/version (runtime; os type and version)”
// Azure-Storage/1.4.0 (NODE-VERSION v4.5.0; Windows_NT 10.0.14393)'
operatingSystem := runtime.GOOS // Default OS string
switch operatingSystem {
case "windows":
operatingSystem = os.Getenv("OS") // Get more specific OS information
case "linux": // accept default OS info
case "freebsd": // accept default OS info
}
return fmt.Sprintf("(%s; %s)", runtime.Version(), operatingSystem)
}()
package azblob
import (
"context"
"github.com/Azure/azure-pipeline-go/pipeline"
)
// NewUniqueRequestIDPolicyFactory creates a UniqueRequestIDPolicyFactory object
// that sets the request's x-ms-client-request-id header if it doesn't already exist.
func NewUniqueRequestIDPolicyFactory() pipeline.Factory {
return pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc {
// This is Policy's Do method:
return func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) {
id := request.Header.Get(xMsClientRequestID)
if id == "" { // Add a unique request ID if the caller didn't specify one already
request.Header.Set(xMsClientRequestID, newUUID().String())
}
return next.Do(ctx, request)
}
})
}
const xMsClientRequestID = "x-ms-client-request-id"
package azblob
import (
"context"
"io"
"net"
"net/http"
"strings"
"sync"
)
const CountToEnd = 0
// HTTPGetter is a function type that refers to a method that performs an HTTP GET operation.
type HTTPGetter func(ctx context.Context, i HTTPGetterInfo) (*http.Response, error)
// HTTPGetterInfo is passed to an HTTPGetter function passing it parameters
// that should be used to make an HTTP GET request.
type HTTPGetterInfo struct {
// Offset specifies the start offset that should be used when
// creating the HTTP GET request's Range header
Offset int64
// Count specifies the count of bytes that should be used to calculate
// the end offset when creating the HTTP GET request's Range header
Count int64
// ETag specifies the resource's etag that should be used when creating
// the HTTP GET request's If-Match header
ETag ETag
}
// FailedReadNotifier is a function type that represents the notification function called when a read fails
type FailedReadNotifier func(failureCount int, lastError error, offset int64, count int64, willRetry bool)
// RetryReaderOptions contains properties which can help to decide when to do retry.
type RetryReaderOptions struct {
// MaxRetryRequests specifies the maximum number of HTTP GET requests that will be made
// while reading from a RetryReader. A value of zero means that no additional HTTP
// GET requests will be made.
MaxRetryRequests int
doInjectError bool
doInjectErrorRound int
injectedError error
// NotifyFailedRead is called, if non-nil, after any failure to read. Expected usage is diagnostic logging.
NotifyFailedRead FailedReadNotifier
// TreatEarlyCloseAsError can be set to true to prevent retries after "read on closed response body". By default,
// retryReader has the following special behaviour: closing the response body before it is all read is treated as a
// retryable error. This is to allow callers to force a retry by closing the body from another goroutine (e.g. if the =
// read is too slow, caller may want to force a retry in the hope that the retry will be quicker). If
// TreatEarlyCloseAsError is true, then retryReader's special behaviour is suppressed, and "read on closed body" is instead
// treated as a fatal (non-retryable) error.
// Note that setting TreatEarlyCloseAsError only guarantees that Closing will produce a fatal error if the Close happens
// from the same "thread" (goroutine) as Read. Concurrent Close calls from other goroutines may instead produce network errors
// which will be retried.
TreatEarlyCloseAsError bool
}
// retryReader implements io.ReaderCloser methods.
// retryReader tries to read from response, and if there is retriable network error
// returned during reading, it will retry according to retry reader option through executing
// user defined action with provided data to get a new response, and continue the overall reading process
// through reading from the new response.
type retryReader struct {
ctx context.Context
info HTTPGetterInfo
countWasBounded bool
o RetryReaderOptions
getter HTTPGetter
// we support Close-ing during Reads (from other goroutines), so we protect the shared state, which is response
responseMu *sync.Mutex
response *http.Response
}
// NewRetryReader creates a retry reader.
func NewRetryReader(ctx context.Context, initialResponse *http.Response,
info HTTPGetterInfo, o RetryReaderOptions, getter HTTPGetter) io.ReadCloser {
return &retryReader{
ctx: ctx,
getter: getter,
info: info,
countWasBounded: info.Count != CountToEnd,
response: initialResponse,
responseMu: &sync.Mutex{},
o: o}
}
func (s *retryReader) setResponse(r *http.Response) {
s.responseMu.Lock()
defer s.responseMu.Unlock()
s.response = r
}
func (s *retryReader) Read(p []byte) (n int, err error) {
for try := 0; ; try++ {
//fmt.Println(try) // Comment out for debugging.
if s.countWasBounded && s.info.Count == CountToEnd {
// User specified an original count and the remaining bytes are 0, return 0, EOF
return 0, io.EOF
}
s.responseMu.Lock()
resp := s.response
s.responseMu.Unlock()
if resp == nil { // We don't have a response stream to read from, try to get one.
newResponse, err := s.getter(s.ctx, s.info)
if err != nil {
return 0, err
}
// Successful GET; this is the network stream we'll read from.
s.setResponse(newResponse)
resp = newResponse
}
n, err := resp.Body.Read(p) // Read from the stream (this will return non-nil err if forceRetry is called, from another goroutine, while it is running)
// Injection mechanism for testing.
if s.o.doInjectError && try == s.o.doInjectErrorRound {
if s.o.injectedError != nil {
err = s.o.injectedError
} else {
err = &net.DNSError{IsTemporary: true}
}
}
// We successfully read data or end EOF.
if err == nil || err == io.EOF {
s.info.Offset += int64(n) // Increments the start offset in case we need to make a new HTTP request in the future
if s.info.Count != CountToEnd {
s.info.Count -= int64(n) // Decrement the count in case we need to make a new HTTP request in the future
}
return n, err // Return the return to the caller
}
s.Close() // Error, close stream
s.setResponse(nil) // Our stream is no longer good
// Check the retry count and error code, and decide whether to retry.
retriesExhausted := try >= s.o.MaxRetryRequests
_, isNetError := err.(net.Error)
isUnexpectedEOF := err == io.ErrUnexpectedEOF
willRetry := (isNetError || isUnexpectedEOF || s.wasRetryableEarlyClose(err)) && !retriesExhausted
// Notify, for logging purposes, of any failures
if s.o.NotifyFailedRead != nil {
failureCount := try + 1 // because try is zero-based
s.o.NotifyFailedRead(failureCount, err, s.info.Offset, s.info.Count, willRetry)
}
if willRetry {
continue
// Loop around and try to get and read from new stream.
}
return n, err // Not retryable, or retries exhausted, so just return
}
}
// By default, we allow early Closing, from another concurrent goroutine, to be used to force a retry
// Is this safe, to close early from another goroutine? Early close ultimately ends up calling
// net.Conn.Close, and that is documented as "Any blocked Read or Write operations will be unblocked and return errors"
// which is exactly the behaviour we want.
// NOTE: that if caller has forced an early Close from a separate goroutine (separate from the Read)
// then there are two different types of error that may happen - either the one one we check for here,
// or a net.Error (due to closure of connection). Which one happens depends on timing. We only need this routine
// to check for one, since the other is a net.Error, which our main Read retry loop is already handing.
func (s *retryReader) wasRetryableEarlyClose(err error) bool {
if s.o.TreatEarlyCloseAsError {
return false // user wants all early closes to be errors, and so not retryable
}
// unfortunately, http.errReadOnClosedResBody is private, so the best we can do here is to check for its text
return strings.HasSuffix(err.Error(), ReadOnClosedBodyMessage)
}
const ReadOnClosedBodyMessage = "read on closed response body"
func (s *retryReader) Close() error {
s.responseMu.Lock()
defer s.responseMu.Unlock()
if s.response != nil && s.response.Body != nil {
return s.response.Body.Close()
}
return nil
}
package azblob
import (
"bytes"
"errors"
"fmt"
"strings"
"time"
)
// AccountSASSignatureValues is used to generate a Shared Access Signature (SAS) for an Azure Storage account.
// For more information, see https://docs.microsoft.com/rest/api/storageservices/constructing-an-account-sas
type AccountSASSignatureValues struct {
Version string `param:"sv"` // If not specified, this defaults to SASVersion
Protocol SASProtocol `param:"spr"` // See the SASProtocol* constants
StartTime time.Time `param:"st"` // Not specified if IsZero
ExpiryTime time.Time `param:"se"` // Not specified if IsZero
Permissions string `param:"sp"` // Create by initializing a AccountSASPermissions and then call String()
IPRange IPRange `param:"sip"`
Services string `param:"ss"` // Create by initializing AccountSASServices and then call String()
ResourceTypes string `param:"srt"` // Create by initializing AccountSASResourceTypes and then call String()
}
// NewSASQueryParameters uses an account's shared key credential to sign this signature values to produce
// the proper SAS query parameters.
func (v AccountSASSignatureValues) NewSASQueryParameters(sharedKeyCredential *SharedKeyCredential) (SASQueryParameters, error) {
// https://docs.microsoft.com/en-us/rest/api/storageservices/Constructing-an-Account-SAS
if v.ExpiryTime.IsZero() || v.Permissions == "" || v.ResourceTypes == "" || v.Services == "" {
return SASQueryParameters{}, errors.New("account SAS is missing at least one of these: ExpiryTime, Permissions, Service, or ResourceType")
}
if v.Version == "" {
v.Version = SASVersion
}
perms := &AccountSASPermissions{}
if err := perms.Parse(v.Permissions); err != nil {
return SASQueryParameters{}, err
}
v.Permissions = perms.String()
startTime, expiryTime, _ := FormatTimesForSASSigning(v.StartTime, v.ExpiryTime, time.Time{})
stringToSign := strings.Join([]string{
sharedKeyCredential.AccountName(),
v.Permissions,
v.Services,
v.ResourceTypes,
startTime,
expiryTime,
v.IPRange.String(),
string(v.Protocol),
v.Version,
""}, // That right, the account SAS requires a terminating extra newline
"\n")
signature := sharedKeyCredential.ComputeHMACSHA256(stringToSign)
p := SASQueryParameters{
// Common SAS parameters
version: v.Version,
protocol: v.Protocol,
startTime: v.StartTime,
expiryTime: v.ExpiryTime,
permissions: v.Permissions,
ipRange: v.IPRange,
// Account-specific SAS parameters
services: v.Services,
resourceTypes: v.ResourceTypes,
// Calculated SAS signature
signature: signature,
}
return p, nil
}
// The AccountSASPermissions type simplifies creating the permissions string for an Azure Storage Account SAS.
// Initialize an instance of this type and then call its String method to set AccountSASSignatureValues's Permissions field.
type AccountSASPermissions struct {
Read, Write, Delete, List, Add, Create, Update, Process bool
}
// String produces the SAS permissions string for an Azure Storage account.
// Call this method to set AccountSASSignatureValues's Permissions field.
func (p AccountSASPermissions) String() string {
var buffer bytes.Buffer
if p.Read {
buffer.WriteRune('r')
}
if p.Write {
buffer.WriteRune('w')
}
if p.Delete {
buffer.WriteRune('d')
}
if p.List {
buffer.WriteRune('l')
}
if p.Add {
buffer.WriteRune('a')
}
if p.Create {
buffer.WriteRune('c')
}
if p.Update {
buffer.WriteRune('u')
}
if p.Process {
buffer.WriteRune('p')
}
return buffer.String()
}
// Parse initializes the AccountSASPermissions's fields from a string.
func (p *AccountSASPermissions) Parse(s string) error {
*p = AccountSASPermissions{} // Clear out the flags
for _, r := range s {
switch r {
case 'r':
p.Read = true
case 'w':
p.Write = true
case 'd':
p.Delete = true
case 'l':
p.List = true
case 'a':
p.Add = true
case 'c':
p.Create = true
case 'u':
p.Update = true
case 'p':
p.Process = true
default:
return fmt.Errorf("Invalid permission character: '%v'", r)
}
}
return nil
}
// The AccountSASServices type simplifies creating the services string for an Azure Storage Account SAS.
// Initialize an instance of this type and then call its String method to set AccountSASSignatureValues's Services field.
type AccountSASServices struct {
Blob, Queue, File bool
}
// String produces the SAS services string for an Azure Storage account.
// Call this method to set AccountSASSignatureValues's Services field.
func (s AccountSASServices) String() string {
var buffer bytes.Buffer
if s.Blob {
buffer.WriteRune('b')
}
if s.Queue {
buffer.WriteRune('q')
}
if s.File {
buffer.WriteRune('f')
}
return buffer.String()
}
// Parse initializes the AccountSASServices' fields from a string.
func (a *AccountSASServices) Parse(s string) error {
*a = AccountSASServices{} // Clear out the flags
for _, r := range s {
switch r {
case 'b':
a.Blob = true
case 'q':
a.Queue = true
case 'f':
a.File = true
default:
return fmt.Errorf("Invalid service character: '%v'", r)
}
}
return nil
}
// The AccountSASResourceTypes type simplifies creating the resource types string for an Azure Storage Account SAS.
// Initialize an instance of this type and then call its String method to set AccountSASSignatureValues's ResourceTypes field.
type AccountSASResourceTypes struct {
Service, Container, Object bool
}
// String produces the SAS resource types string for an Azure Storage account.
// Call this method to set AccountSASSignatureValues's ResourceTypes field.
func (rt AccountSASResourceTypes) String() string {
var buffer bytes.Buffer
if rt.Service {
buffer.WriteRune('s')
}
if rt.Container {
buffer.WriteRune('c')
}
if rt.Object {
buffer.WriteRune('o')
}
return buffer.String()
}
// Parse initializes the AccountSASResourceType's fields from a string.
func (rt *AccountSASResourceTypes) Parse(s string) error {
*rt = AccountSASResourceTypes{} // Clear out the flags
for _, r := range s {
switch r {
case 's':
rt.Service = true
case 'c':
rt.Container = true
case 'o':
rt.Object = true
default:
return fmt.Errorf("Invalid resource type: '%v'", r)
}
}
return nil
}
package azblob
import (
"errors"
"net"
"net/url"
"strings"
"time"
)
// SASVersion indicates the SAS version.
const SASVersion = ServiceVersion
type SASProtocol string
const (
// SASProtocolHTTPS can be specified for a SAS protocol
SASProtocolHTTPS SASProtocol = "https"
// SASProtocolHTTPSandHTTP can be specified for a SAS protocol
SASProtocolHTTPSandHTTP SASProtocol = "https,http"
)
// FormatTimesForSASSigning converts a time.Time to a snapshotTimeFormat string suitable for a
// SASField's StartTime or ExpiryTime fields. Returns "" if value.IsZero().
func FormatTimesForSASSigning(startTime, expiryTime, snapshotTime time.Time) (string, string, string) {
ss := ""
if !startTime.IsZero() {
ss = formatSASTimeWithDefaultFormat(&startTime)
}
se := ""
if !expiryTime.IsZero() {
se = formatSASTimeWithDefaultFormat(&expiryTime)
}
sh := ""
if !snapshotTime.IsZero() {
sh = snapshotTime.Format(SnapshotTimeFormat)
}
return ss, se, sh
}
// SASTimeFormat represents the format of a SAS start or expiry time. Use it when formatting/parsing a time.Time.
const SASTimeFormat = "2006-01-02T15:04:05Z" //"2017-07-27T00:00:00Z" // ISO 8601
var SASTimeFormats = []string{"2006-01-02T15:04:05.0000000Z", SASTimeFormat, "2006-01-02T15:04Z", "2006-01-02"} // ISO 8601 formats, please refer to https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-a-service-sas for more details.
// formatSASTimeWithDefaultFormat format time with ISO 8601 in "yyyy-MM-ddTHH:mm:ssZ".
func formatSASTimeWithDefaultFormat(t *time.Time) string {
return formatSASTime(t, SASTimeFormat) // By default, "yyyy-MM-ddTHH:mm:ssZ" is used
}
// formatSASTime format time with given format, use ISO 8601 in "yyyy-MM-ddTHH:mm:ssZ" by default.
func formatSASTime(t *time.Time, format string) string {
if format != "" {
return t.Format(format)
}
return t.Format(SASTimeFormat) // By default, "yyyy-MM-ddTHH:mm:ssZ" is used
}
// parseSASTimeString try to parse sas time string.
func parseSASTimeString(val string) (t time.Time, timeFormat string, err error) {
for _, sasTimeFormat := range SASTimeFormats {
t, err = time.Parse(sasTimeFormat, val)
if err == nil {
timeFormat = sasTimeFormat
break
}
}
if err != nil {
err = errors.New("fail to parse time with IOS 8601 formats, please refer to https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-a-service-sas for more details")
}
return
}
// https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-a-service-sas
// A SASQueryParameters object represents the components that make up an Azure Storage SAS' query parameters.
// You parse a map of query parameters into its fields by calling NewSASQueryParameters(). You add the components
// to a query parameter map by calling AddToValues().
// NOTE: Changing any field requires computing a new SAS signature using a XxxSASSignatureValues type.
//
// This type defines the components used by all Azure Storage resources (Containers, Blobs, Files, & Queues).
type SASQueryParameters struct {
// All members are immutable or values so copies of this struct are goroutine-safe.
version string `param:"sv"`
services string `param:"ss"`
resourceTypes string `param:"srt"`
protocol SASProtocol `param:"spr"`
startTime time.Time `param:"st"`
expiryTime time.Time `param:"se"`
snapshotTime time.Time `param:"snapshot"`
ipRange IPRange `param:"sip"`
identifier string `param:"si"`
resource string `param:"sr"`
permissions string `param:"sp"`
signature string `param:"sig"`
cacheControl string `param:"rscc"`
contentDisposition string `param:"rscd"`
contentEncoding string `param:"rsce"`
contentLanguage string `param:"rscl"`
contentType string `param:"rsct"`
signedOid string `param:"skoid"`
signedTid string `param:"sktid"`
signedStart time.Time `param:"skt"`
signedExpiry time.Time `param:"ske"`
signedService string `param:"sks"`
signedVersion string `param:"skv"`
// private member used for startTime and expiryTime formatting.
stTimeFormat string
seTimeFormat string
}
func (p *SASQueryParameters) SignedOid() string {
return p.signedOid
}
func (p *SASQueryParameters) SignedTid() string {
return p.signedTid
}
func (p *SASQueryParameters) SignedStart() time.Time {
return p.signedStart
}
func (p *SASQueryParameters) SignedExpiry() time.Time {
return p.signedExpiry
}
func (p *SASQueryParameters) SignedService() string {
return p.signedService
}
func (p *SASQueryParameters) SignedVersion() string {
return p.signedVersion
}
func (p *SASQueryParameters) SnapshotTime() time.Time {
return p.snapshotTime
}
func (p *SASQueryParameters) Version() string {
return p.version
}
func (p *SASQueryParameters) Services() string {
return p.services
}
func (p *SASQueryParameters) ResourceTypes() string {
return p.resourceTypes
}
func (p *SASQueryParameters) Protocol() SASProtocol {
return p.protocol
}
func (p *SASQueryParameters) StartTime() time.Time {
return p.startTime
}
func (p *SASQueryParameters) ExpiryTime() time.Time {
return p.expiryTime
}
func (p *SASQueryParameters) IPRange() IPRange {
return p.ipRange
}
func (p *SASQueryParameters) Identifier() string {
return p.identifier
}
func (p *SASQueryParameters) Resource() string {
return p.resource
}
func (p *SASQueryParameters) Permissions() string {
return p.permissions
}
func (p *SASQueryParameters) Signature() string {
return p.signature
}
func (p *SASQueryParameters) CacheControl() string {
return p.cacheControl
}
func (p *SASQueryParameters) ContentDisposition() string {
return p.contentDisposition
}
func (p *SASQueryParameters) ContentEncoding() string {
return p.contentEncoding
}
func (p *SASQueryParameters) ContentLanguage() string {
return p.contentLanguage
}
func (p *SASQueryParameters) ContentType() string {
return p.contentType
}
// IPRange represents a SAS IP range's start IP and (optionally) end IP.
type IPRange struct {
Start net.IP // Not specified if length = 0
End net.IP // Not specified if length = 0
}
// String returns a string representation of an IPRange.
func (ipr *IPRange) String() string {
if len(ipr.Start) == 0 {
return ""
}
start := ipr.Start.String()
if len(ipr.End) == 0 {
return start
}
return start + "-" + ipr.End.String()
}
// NewSASQueryParameters creates and initializes a SASQueryParameters object based on the
// query parameter map's passed-in values. If deleteSASParametersFromValues is true,
// all SAS-related query parameters are removed from the passed-in map. If
// deleteSASParametersFromValues is false, the map passed-in map is unaltered.
func newSASQueryParameters(values url.Values, deleteSASParametersFromValues bool) SASQueryParameters {
p := SASQueryParameters{}
for k, v := range values {
val := v[0]
isSASKey := true
switch strings.ToLower(k) {
case "sv":
p.version = val
case "ss":
p.services = val
case "srt":
p.resourceTypes = val
case "spr":
p.protocol = SASProtocol(val)
case "snapshot":
p.snapshotTime, _ = time.Parse(SnapshotTimeFormat, val)
case "st":
p.startTime, p.stTimeFormat, _ = parseSASTimeString(val)
case "se":
p.expiryTime, p.seTimeFormat, _ = parseSASTimeString(val)
case "sip":
dashIndex := strings.Index(val, "-")
if dashIndex == -1 {
p.ipRange.Start = net.ParseIP(val)
} else {
p.ipRange.Start = net.ParseIP(val[:dashIndex])
p.ipRange.End = net.ParseIP(val[dashIndex+1:])
}
case "si":
p.identifier = val
case "sr":
p.resource = val
case "sp":
p.permissions = val
case "sig":
p.signature = val
case "rscc":
p.cacheControl = val
case "rscd":
p.contentDisposition = val
case "rsce":
p.contentEncoding = val
case "rscl":
p.contentLanguage = val
case "rsct":
p.contentType = val
case "skoid":
p.signedOid = val
case "sktid":
p.signedTid = val
case "skt":
p.signedStart, _ = time.Parse(SASTimeFormat, val)
case "ske":
p.signedExpiry, _ = time.Parse(SASTimeFormat, val)
case "sks":
p.signedService = val
case "skv":
p.signedVersion = val
default:
isSASKey = false // We didn't recognize the query parameter
}
if isSASKey && deleteSASParametersFromValues {
delete(values, k)
}
}
return p
}
// AddToValues adds the SAS components to the specified query parameters map.
func (p *SASQueryParameters) addToValues(v url.Values) url.Values {
if p.version != "" {
v.Add("sv", p.version)
}
if p.services != "" {
v.Add("ss", p.services)
}
if p.resourceTypes != "" {
v.Add("srt", p.resourceTypes)
}
if p.protocol != "" {
v.Add("spr", string(p.protocol))
}
if !p.startTime.IsZero() {
v.Add("st", formatSASTime(&(p.startTime), p.stTimeFormat))
}
if !p.expiryTime.IsZero() {
v.Add("se", formatSASTime(&(p.expiryTime), p.seTimeFormat))
}
if len(p.ipRange.Start) > 0 {
v.Add("sip", p.ipRange.String())
}
if p.identifier != "" {
v.Add("si", p.identifier)
}
if p.resource != "" {
v.Add("sr", p.resource)
}
if p.permissions != "" {
v.Add("sp", p.permissions)
}
if p.signedOid != "" {
v.Add("skoid", p.signedOid)
v.Add("sktid", p.signedTid)
v.Add("skt", p.signedStart.Format(SASTimeFormat))
v.Add("ske", p.signedExpiry.Format(SASTimeFormat))
v.Add("sks", p.signedService)
v.Add("skv", p.signedVersion)
}
if p.signature != "" {
v.Add("sig", p.signature)
}
if p.cacheControl != "" {
v.Add("rscc", p.cacheControl)
}
if p.contentDisposition != "" {
v.Add("rscd", p.contentDisposition)
}
if p.contentEncoding != "" {
v.Add("rsce", p.contentEncoding)
}
if p.contentLanguage != "" {
v.Add("rscl", p.contentLanguage)
}
if p.contentType != "" {
v.Add("rsct", p.contentType)
}
return v
}
// Encode encodes the SAS query parameters into URL encoded form sorted by key.
func (p *SASQueryParameters) Encode() string {
v := url.Values{}
p.addToValues(v)
return v.Encode()
}
package azblob
// https://docs.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes
const (
// ServiceCodeNone is the default value. It indicates that the error was related to the service or that the service didn't return a code.
ServiceCodeNone ServiceCodeType = ""
// ServiceCodeAccountAlreadyExists means the specified account already exists.
ServiceCodeAccountAlreadyExists ServiceCodeType = "AccountAlreadyExists"
// ServiceCodeAccountBeingCreated means the specified account is in the process of being created (403).
ServiceCodeAccountBeingCreated ServiceCodeType = "AccountBeingCreated"
// ServiceCodeAccountIsDisabled means the specified account is disabled (403).
ServiceCodeAccountIsDisabled ServiceCodeType = "AccountIsDisabled"
// ServiceCodeAuthenticationFailed means the server failed to authenticate the request. Make sure the value of the Authorization header is formed correctly including the signature (403).
ServiceCodeAuthenticationFailed ServiceCodeType = "AuthenticationFailed"
// ServiceCodeConditionHeadersNotSupported means the condition headers are not supported (400).
ServiceCodeConditionHeadersNotSupported ServiceCodeType = "ConditionHeadersNotSupported"
// ServiceCodeConditionNotMet means the condition specified in the conditional header(s) was not met for a read/write operation (304/412).
ServiceCodeConditionNotMet ServiceCodeType = "ConditionNotMet"
// ServiceCodeEmptyMetadataKey means the key for one of the metadata key-value pairs is empty (400).
ServiceCodeEmptyMetadataKey ServiceCodeType = "EmptyMetadataKey"
// ServiceCodeInsufficientAccountPermissions means read operations are currently disabled or Write operations are not allowed or The account being accessed does not have sufficient permissions to execute this operation (403).
ServiceCodeInsufficientAccountPermissions ServiceCodeType = "InsufficientAccountPermissions"
// ServiceCodeInternalError means the server encountered an internal error. Please retry the request (500).
ServiceCodeInternalError ServiceCodeType = "InternalError"
// ServiceCodeInvalidAuthenticationInfo means the authentication information was not provided in the correct format. Verify the value of Authorization header (400).
ServiceCodeInvalidAuthenticationInfo ServiceCodeType = "InvalidAuthenticationInfo"
// ServiceCodeInvalidHeaderValue means the value provided for one of the HTTP headers was not in the correct format (400).
ServiceCodeInvalidHeaderValue ServiceCodeType = "InvalidHeaderValue"
// ServiceCodeInvalidHTTPVerb means the HTTP verb specified was not recognized by the server (400).
ServiceCodeInvalidHTTPVerb ServiceCodeType = "InvalidHttpVerb"
// ServiceCodeInvalidInput means one of the request inputs is not valid (400).
ServiceCodeInvalidInput ServiceCodeType = "InvalidInput"
// ServiceCodeInvalidMd5 means the MD5 value specified in the request is invalid. The MD5 value must be 128 bits and Base64-encoded (400).
ServiceCodeInvalidMd5 ServiceCodeType = "InvalidMd5"
// ServiceCodeInvalidMetadata means the specified metadata is invalid. It includes characters that are not permitted (400).
ServiceCodeInvalidMetadata ServiceCodeType = "InvalidMetadata"
// ServiceCodeInvalidQueryParameterValue means an invalid value was specified for one of the query parameters in the request URI (400).
ServiceCodeInvalidQueryParameterValue ServiceCodeType = "InvalidQueryParameterValue"
// ServiceCodeInvalidRange means the range specified is invalid for the current size of the resource (416).
ServiceCodeInvalidRange ServiceCodeType = "InvalidRange"
// ServiceCodeInvalidResourceName means the specified resource name contains invalid characters (400).
ServiceCodeInvalidResourceName ServiceCodeType = "InvalidResourceName"
// ServiceCodeInvalidURI means the requested URI does not represent any resource on the server (400).
ServiceCodeInvalidURI ServiceCodeType = "InvalidUri"
// ServiceCodeInvalidXMLDocument means the specified XML is not syntactically valid (400).
ServiceCodeInvalidXMLDocument ServiceCodeType = "InvalidXmlDocument"
// ServiceCodeInvalidXMLNodeValue means the value provided for one of the XML nodes in the request body was not in the correct format (400).
ServiceCodeInvalidXMLNodeValue ServiceCodeType = "InvalidXmlNodeValue"
// ServiceCodeMd5Mismatch means the MD5 value specified in the request did not match the MD5 value calculated by the server (400).
ServiceCodeMd5Mismatch ServiceCodeType = "Md5Mismatch"
// ServiceCodeMetadataTooLarge means the size of the specified metadata exceeds the maximum size permitted (400).
ServiceCodeMetadataTooLarge ServiceCodeType = "MetadataTooLarge"
// ServiceCodeMissingContentLengthHeader means the Content-Length header was not specified (411).
ServiceCodeMissingContentLengthHeader ServiceCodeType = "MissingContentLengthHeader"
// ServiceCodeMissingRequiredQueryParameter means a required query parameter was not specified for this request (400).
ServiceCodeMissingRequiredQueryParameter ServiceCodeType = "MissingRequiredQueryParameter"
// ServiceCodeMissingRequiredHeader means a required HTTP header was not specified (400).
ServiceCodeMissingRequiredHeader ServiceCodeType = "MissingRequiredHeader"
// ServiceCodeMissingRequiredXMLNode means a required XML node was not specified in the request body (400).
ServiceCodeMissingRequiredXMLNode ServiceCodeType = "MissingRequiredXmlNode"
// ServiceCodeMultipleConditionHeadersNotSupported means multiple condition headers are not supported (400).
ServiceCodeMultipleConditionHeadersNotSupported ServiceCodeType = "MultipleConditionHeadersNotSupported"
// ServiceCodeOperationTimedOut means the operation could not be completed within the permitted time (500).
ServiceCodeOperationTimedOut ServiceCodeType = "OperationTimedOut"
// ServiceCodeOutOfRangeInput means one of the request inputs is out of range (400).
ServiceCodeOutOfRangeInput ServiceCodeType = "OutOfRangeInput"
// ServiceCodeOutOfRangeQueryParameterValue means a query parameter specified in the request URI is outside the permissible range (400).
ServiceCodeOutOfRangeQueryParameterValue ServiceCodeType = "OutOfRangeQueryParameterValue"
// ServiceCodeRequestBodyTooLarge means the size of the request body exceeds the maximum size permitted (413).
ServiceCodeRequestBodyTooLarge ServiceCodeType = "RequestBodyTooLarge"
// ServiceCodeResourceTypeMismatch means the specified resource type does not match the type of the existing resource (409).
ServiceCodeResourceTypeMismatch ServiceCodeType = "ResourceTypeMismatch"
// ServiceCodeRequestURLFailedToParse means the url in the request could not be parsed (400).
ServiceCodeRequestURLFailedToParse ServiceCodeType = "RequestUrlFailedToParse"
// ServiceCodeResourceAlreadyExists means the specified resource already exists (409).
ServiceCodeResourceAlreadyExists ServiceCodeType = "ResourceAlreadyExists"
// ServiceCodeResourceNotFound means the specified resource does not exist (404).
ServiceCodeResourceNotFound ServiceCodeType = "ResourceNotFound"
// ServiceCodeServerBusy means the server is currently unable to receive requests. Please retry your request or Ingress/egress is over the account limit or operations per second is over the account limit (503).
ServiceCodeServerBusy ServiceCodeType = "ServerBusy"
// ServiceCodeUnsupportedHeader means one of the HTTP headers specified in the request is not supported (400).
ServiceCodeUnsupportedHeader ServiceCodeType = "UnsupportedHeader"
// ServiceCodeUnsupportedXMLNode means one of the XML nodes specified in the request body is not supported (400).
ServiceCodeUnsupportedXMLNode ServiceCodeType = "UnsupportedXmlNode"
// ServiceCodeUnsupportedQueryParameter means one of the query parameters specified in the request URI is not supported (400).
ServiceCodeUnsupportedQueryParameter ServiceCodeType = "UnsupportedQueryParameter"
// ServiceCodeUnsupportedHTTPVerb means the resource doesn't support the specified HTTP verb (405).
ServiceCodeUnsupportedHTTPVerb ServiceCodeType = "UnsupportedHttpVerb"
)
package azblob
import (
"bytes"
"encoding/xml"
"fmt"
"net/http"
"sort"
"github.com/Azure/azure-pipeline-go/pipeline"
)
func init() {
// wire up our custom error handling constructor
responseErrorFactory = newStorageError
}
// ServiceCodeType is a string identifying a storage service error.
// For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/status-and-error-codes2
type ServiceCodeType string
// StorageError identifies a responder-generated network or response parsing error.
type StorageError interface {
// ResponseError implements error's Error(), net.Error's Temporary() and Timeout() methods & Response().
ResponseError
// ServiceCode returns a service error code. Your code can use this to make error recovery decisions.
ServiceCode() ServiceCodeType
}
// storageError is the internal struct that implements the public StorageError interface.
type storageError struct {
responseError
serviceCode ServiceCodeType
details map[string]string
}
// newStorageError creates an error object that implements the error interface.
func newStorageError(cause error, response *http.Response, description string) error {
return &storageError{
responseError: responseError{
ErrorNode: pipeline.ErrorNode{}.Initialize(cause, 3),
response: response,
description: description,
},
serviceCode: ServiceCodeType(response.Header.Get("x-ms-error-code")),
}
}
// ServiceCode returns service-error information. The caller may examine these values but should not modify any of them.
func (e *storageError) ServiceCode() ServiceCodeType {
return e.serviceCode
}
// Error implements the error interface's Error method to return a string representation of the error.
func (e *storageError) Error() string {
b := &bytes.Buffer{}
fmt.Fprintf(b, "===== RESPONSE ERROR (ServiceCode=%s) =====\n", e.serviceCode)
fmt.Fprintf(b, "Description=%s, Details: ", e.description)
if len(e.details) == 0 {
b.WriteString("(none)\n")
} else {
b.WriteRune('\n')
keys := make([]string, 0, len(e.details))
// Alphabetize the details
for k := range e.details {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
fmt.Fprintf(b, " %s: %+v\n", k, e.details[k])
}
}
req := pipeline.Request{Request: e.response.Request}.Copy() // Make a copy of the response's request
pipeline.WriteRequestWithResponse(b, prepareRequestForLogging(req), e.response, nil)
return e.ErrorNode.Error(b.String())
}
// Temporary returns true if the error occurred due to a temporary condition (including an HTTP status of 500 or 503).
func (e *storageError) Temporary() bool {
if e.response != nil {
if (e.response.StatusCode == http.StatusInternalServerError) || (e.response.StatusCode == http.StatusServiceUnavailable) {
return true
}
}
return e.ErrorNode.Temporary()
}
// UnmarshalXML performs custom unmarshalling of XML-formatted Azure storage request errors.
func (e *storageError) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) {
tokName := ""
var t xml.Token
for t, err = d.Token(); err == nil; t, err = d.Token() {
switch tt := t.(type) {
case xml.StartElement:
tokName = tt.Name.Local
break
case xml.CharData:
switch tokName {
case "Message":
e.description = string(tt)
default:
if e.details == nil {
e.details = map[string]string{}
}
e.details[tokName] = string(tt)
}
}
}
return nil
}
package azblob
import (
"errors"
"fmt"
"io"
"strconv"
)
// httpRange defines a range of bytes within an HTTP resource, starting at offset and
// ending at offset+count. A zero-value httpRange indicates the entire resource. An httpRange
// which has an offset but na zero value count indicates from the offset to the resource's end.
type httpRange struct {
offset int64
count int64
}
func (r httpRange) pointers() *string {
if r.offset == 0 && r.count == CountToEnd { // Do common case first for performance
return nil // No specified range
}
endOffset := "" // if count == CountToEnd (0)
if r.count > 0 {
endOffset = strconv.FormatInt((r.offset+r.count)-1, 10)
}
dataRange := fmt.Sprintf("bytes=%v-%s", r.offset, endOffset)
return &dataRange
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
func validateSeekableStreamAt0AndGetCount(body io.ReadSeeker) (int64, error) {
if body == nil { // nil body's are "logically" seekable to 0 and are 0 bytes long
return 0, nil
}
err := validateSeekableStreamAt0(body)
if err != nil {
return 0, err
}
count, err := body.Seek(0, io.SeekEnd)
if err != nil {
return 0, errors.New("body stream must be seekable")
}
body.Seek(0, io.SeekStart)
return count, nil
}
// return an error if body is not a valid seekable stream at 0
func validateSeekableStreamAt0(body io.ReadSeeker) error {
if body == nil { // nil body's are "logically" seekable to 0
return nil
}
if pos, err := body.Seek(0, io.SeekCurrent); pos != 0 || err != nil {
// Help detect programmer error
if err != nil {
return errors.New("body stream must be seekable")
}
return errors.New("body stream must be set to position 0")
}
return nil
}
package azblob
import (
"crypto/rand"
"fmt"
"strconv"
)
// The UUID reserved variants.
const (
reservedNCS byte = 0x80
reservedRFC4122 byte = 0x40
reservedMicrosoft byte = 0x20
reservedFuture byte = 0x00
)
// A UUID representation compliant with specification in RFC 4122 document.
type uuid [16]byte
// NewUUID returns a new uuid using RFC 4122 algorithm.
func newUUID() (u uuid) {
u = uuid{}
// Set all bits to randomly (or pseudo-randomly) chosen values.
rand.Read(u[:])
u[8] = (u[8] | reservedRFC4122) & 0x7F // u.setVariant(ReservedRFC4122)
var version byte = 4
u[6] = (u[6] & 0xF) | (version << 4) // u.setVersion(4)
return
}
// String returns an unparsed version of the generated UUID sequence.
func (u uuid) String() string {
return fmt.Sprintf("%x-%x-%x-%x-%x", u[0:4], u[4:6], u[6:8], u[8:10], u[10:])
}
// ParseUUID parses a string formatted as "003020100-0504-0706-0809-0a0b0c0d0e0f"
// or "{03020100-0504-0706-0809-0a0b0c0d0e0f}" into a UUID.
func parseUUID(uuidStr string) uuid {
char := func(hexString string) byte {
i, _ := strconv.ParseUint(hexString, 16, 8)
return byte(i)
}
if uuidStr[0] == '{' {
uuidStr = uuidStr[1:] // Skip over the '{'
}
// 03020100 - 05 04 - 07 06 - 08 09 - 0a 0b 0c 0d 0e 0f
// 1 11 1 11 11 1 12 22 2 22 22 22 33 33 33
// 01234567 8 90 12 3 45 67 8 90 12 3 45 67 89 01 23 45
uuidVal := uuid{
char(uuidStr[0:2]),
char(uuidStr[2:4]),
char(uuidStr[4:6]),
char(uuidStr[6:8]),
char(uuidStr[9:11]),
char(uuidStr[11:13]),
char(uuidStr[14:16]),
char(uuidStr[16:18]),
char(uuidStr[19:21]),
char(uuidStr[21:23]),
char(uuidStr[24:26]),
char(uuidStr[26:28]),
char(uuidStr[28:30]),
char(uuidStr[30:32]),
char(uuidStr[32:34]),
char(uuidStr[34:36]),
}
return uuidVal
}
func (u uuid) bytes() []byte {
return u[:]
}
// Copyright 2017 Microsoft Corporation. All rights reserved.
// Use of this source code is governed by an MIT
// license that can be found in the LICENSE file.
/*
Package azblob allows you to manipulate Azure Storage containers and blobs objects.
URL Types
The most common types you'll work with are the XxxURL types. The methods of these types make requests
against the Azure Storage Service.
- ServiceURL's methods perform operations on a storage account.
- ContainerURL's methods perform operations on an account's container.
- BlockBlobURL's methods perform operations on a container's block blob.
- AppendBlobURL's methods perform operations on a container's append blob.
- PageBlobURL's methods perform operations on a container's page blob.
- BlobURL's methods perform operations on a container's blob regardless of the blob's type.
Internally, each XxxURL object contains a URL and a request pipeline. The URL indicates the endpoint where each HTTP
request is sent and the pipeline indicates how the outgoing HTTP request and incoming HTTP response is processed.
The pipeline specifies things like retry policies, logging, deserialization of HTTP response payloads, and more.
Pipelines are threadsafe and may be shared by multiple XxxURL objects. When you create a ServiceURL, you pass
an initial pipeline. When you call ServiceURL's NewContainerURL method, the new ContainerURL object has its own
URL but it shares the same pipeline as the parent ServiceURL object.
To work with a blob, call one of ContainerURL's 4 NewXxxBlobURL methods depending on how you want to treat the blob.
To treat the blob as a block blob, append blob, or page blob, call NewBlockBlobURL, NewAppendBlobURL, or NewPageBlobURL
respectively. These three types are all identical except for the methods they expose; each type exposes the methods
relevant to the type of blob represented. If you're not sure how you want to treat a blob, you can call NewBlobURL;
this returns an object whose methods are relevant to any kind of blob. When you call ContainerURL's NewXxxBlobURL,
the new XxxBlobURL object has its own URL but it shares the same pipeline as the parent ContainerURL object. You
can easily switch between blob types (method sets) by calling a ToXxxBlobURL method.
If you'd like to use a different pipeline with a ServiceURL, ContainerURL, or XxxBlobURL object, then call the XxxURL
object's WithPipeline method passing in the desired pipeline. The WithPipeline methods create a new XxxURL object
with the same URL as the original but with the specified pipeline.
Note that XxxURL objects use little memory, are goroutine-safe, and many objects share the same pipeline. This means that
XxxURL objects share a lot of system resources making them very efficient.
All of XxxURL's methods that make HTTP requests return rich error handling information so you can discern network failures,
transient failures, timeout failures, service failures, etc. See the StorageError interface for more information and an
example of how to do deal with errors.
URL and Shared Access Signature Manipulation
The library includes a BlobURLParts type for deconstructing and reconstructing URLs. And you can use the following types
for generating and parsing Shared Access Signature (SAS)
- Use the AccountSASSignatureValues type to create a SAS for a storage account.
- Use the BlobSASSignatureValues type to create a SAS for a container or blob.
- Use the SASQueryParameters type to turn signature values in to query parameres or to parse query parameters.
To generate a SAS, you must use the SharedKeyCredential type.
Credentials
When creating a request pipeline, you must specify one of this package's credential types.
- Call the NewAnonymousCredential function for requests that contain a Shared Access Signature (SAS).
- Call the NewSharedKeyCredential function (with an account name & key) to access any account resources. You must also use this
to generate Shared Access Signatures.
HTTP Request Policy Factories
This package defines several request policy factories for use with the pipeline package.
Most applications will not use these factories directly; instead, the NewPipeline
function creates these factories, initializes them (via the PipelineOptions type)
and returns a pipeline object for use by the XxxURL objects.
However, for advanced scenarios, developers can access these policy factories directly
and even create their own and then construct their own pipeline in order to affect HTTP
requests and responses performed by the XxxURL objects. For example, developers can
introduce their own logging, random failures, request recording & playback for fast
testing, HTTP request pacing, alternate retry mechanisms, metering, metrics, etc. The
possibilities are endless!
Below are the request pipeline policy factory functions that are provided with this
package:
- NewRetryPolicyFactory Enables rich retry semantics for failed HTTP requests.
- NewRequestLogPolicyFactory Enables rich logging support for HTTP requests/responses & failures.
- NewTelemetryPolicyFactory Enables simple modification of the HTTP request's User-Agent header so each request reports the SDK version & language/runtime making the requests.
- NewUniqueRequestIDPolicyFactory Adds a x-ms-client-request-id header with a unique UUID value to an HTTP request to help with diagnosing failures.
Also, note that all the NewXxxCredential functions return request policy factory objects which get injected into the pipeline.
*/
package azblob
// TokenCredential Use this to access resources using Role-Based Access Control (RBAC).
This diff is collapsed.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment