feat: support GraphQL Upload scalar via multipart requests (080c5219) · Commits · GitLab.org / API / client-go

example_graphql_upload_test.go

0 → 100644

+65 −0

Original line number	Diff line number	Diff line
		package gitlab_test

		import (
		"fmt"
		"strings"

		gitlab "gitlab.com/gitlab-org/api/client-go/v2"
		)

		// Example_graphQLFileUpload demonstrates uploading a file through the GraphQL
		// API. A *gitlab.GraphQLUpload can be used anywhere a GraphQL mutation expects
		// an Upload scalar: as a variable of a query passed to client.GraphQL.Do
		// directly, as done here, or as a field of a service's options struct.
		func Example_graphQLFileUpload() {
		// Note: The setupGraphQLUploadMock() function below is ONLY for the example purpose
		// and has nothing to do with how a user will use client-go.
		// In production, you would use a real authenticated GitLab client.
		client, server := setupGraphQLUploadMock()
		defer server.Close()

		// Any io.Reader works as the upload's content, for example an *os.File
		// opened from disk. Its data is buffered by NewGraphQLUpload, so the
		// upload can be sent again if the request is retried.
		avatar, err := gitlab.NewGraphQLUpload(strings.NewReader("fake-avatar-bytes"), "avatar.png", "image/png")
		if err != nil {
		fmt.Println(err)
		return
		}

		var response struct {
		Data struct {
		AchievementsCreate struct {
		Achievement struct {
		Name string `json:"name"`
		AvatarURL string `json:"avatarUrl"`
		} `json:"achievement"`
		} `json:"achievementsCreate"`
		} `json:"data"`
		}

		_, _ = client.GraphQL.Do(
		gitlab.GraphQLQuery{
		Query: `
		mutation ($input: AchievementsCreateInput!) {
		achievementsCreate(input: $input) {
		achievement { name avatarUrl }
		}
		}`,
		Variables: map[string]any{
		"input": map[string]any{
		"namespaceId": "gid://gitlab/Namespace/10",
		"name": "First Commit",
		"avatar": avatar,
		},
		},
		},
		&response,
		)

		achievement := response.Data.AchievementsCreate.Achievement
		fmt.Printf("Created %s with avatar %s\n", achievement.Name, achievement.AvatarURL)

		// Output:
		// Created First Commit with avatar https://gitlab.example.com/uploads/avatar.png
		}

graphql.go

+174 −1

Original line number	Diff line number	Diff line
		@@ -2,16 +2,26 @@ package gitlab

		import (
		"encoding/json"
		"errors"
		"fmt"
		"io"
		"net/http"
		"regexp"
		"strconv"
		"strings"

		retryablehttp "github.com/hashicorp/go-retryablehttp"

		"gitlab.com/gitlab-org/api/client-go/v2/internal/graphql"
		)

		const (
		// GraphQLAPIEndpoint defines the endpoint URI for the GraphQL backend
		GraphQLAPIEndpoint = "/api/graphql"

		// graphQLUploadDefaultContentType is the Content-Type sent for an
		// upload that does not specify one.
		graphQLUploadDefaultContentType = "application/octet-stream"
		)

		type (
		@@ -28,6 +38,64 @@ type (
		Variables map[string]any `json:"variables,omitempty"`
		}

		// GraphQLUpload is a file to upload as part of a GraphQL query. Create
		// one with [NewGraphQLUpload].
		//
		// Set one wherever a mutation input expects a GraphQL Upload scalar,
		// nested at any depth inside GraphQLQuery.Variables, and Do sends the
		// query as a multipart request following the GraphQL multipart request
		// specification, https://github.com/jaydenseric/graphql-multipart-request-spec,
		// which GitLab implements through apollo_upload_server. An upload always
		// marshals to null in the query itself; the file travels in its own part
		// of the multipart body.
		//
		// Uploading several files works the same way, including for arguments
		// that take a list of uploads: assign an upload to every element that
		// carries a file. Assigning the same *GraphQLUpload to more than one
		// variable sends its content once and points every variable at it.
		//
		// An upload holds its own copy of the file, so it can be sent as many
		// times as needed, and is safe to use from several queries at once.
		//
		// Do finds an upload by the position it marshals to, which it derives the
		// way encoding/json does. A type that marshals itself - one implementing
		// json.Marshaler or encoding.TextMarshaler - decides its own JSON shape,
		// so an upload held inside one has no position that can be derived, and
		// Do reports an error rather than sending the query without the file. The
		// same goes for an upload under a map key that is neither a string nor an
		// integer. Hold uploads in plain maps, slices and structs to avoid this.
		//
		// Example:
		//
		// avatar, err := gitlab.NewGraphQLUpload(f, "avatar.png", "image/png")
		// if err != nil {
		// return err
		// }
		//
		// input := map[string]any{
		// "namespaceId": "gid://gitlab/Namespace/1",
		// "name": "First Commit",
		// "avatar": avatar,
		// }
		//
		// Experimental: reflection derives an upload's position from the marshaled
		// query, and an edge case it does not handle correctly may require this
		// type or its behavior to change in a breaking way to fix.
		GraphQLUpload struct {
		// content holds the file's data, buffered by NewGraphQLUpload so
		// that every request - including a retried one - can write it
		// again.
		content []byte

		// filename is reported to GitLab as the uploaded file's name.
		// GitLab does not treat a part without a filename as an uploaded
		// file, so NewGraphQLUpload requires it.
		filename string

		// contentType is sent as the file part's Content-Type.
		contentType string
		}

		GenericGraphQLErrors struct {
		Errors []struct {
		Message string `json:"message"`
		@@ -78,9 +146,26 @@ func (e *GraphQLResponseError) Error() string {
		// }
		// _, err := client.GraphQL.Do(GraphQLQuery{Query: `query { project(fullPath: "gitlab-org/gitlab") { id } }`}, &response, gitlab.WithContext(ctx))
		//
		// When the query's variables contain one or more GraphQLUpload values, the
		// query is sent as a multipart request that carries the files alongside it.
		//
		// Attention: This API is experimental and may be subject to breaking changes to improve the API in the future.
		func (g GraphQL) Do(query GraphQLQuery, response any, options ...RequestOptionFunc) (Response, error) {
		request, err := g.client.NewRequest(http.MethodPost, "", query, options)
		var (
		request *retryablehttp.Request
		err error
		)

		uploads, err := graphql.CollectUploads[GraphQLUpload](query.Variables)
		if err != nil {
		return nil, fmt.Errorf("failed to create GraphQL request: %w", err)
		}

		if len(uploads) > 0 {
		request, err = g.newUploadRequest(query, uploads, options)
		} else {
		request, err = g.client.NewRequest(http.MethodPost, "", query, options)
		}
		if err != nil {
		return nil, fmt.Errorf("failed to create GraphQL request: %w", err)
		}
		@@ -104,6 +189,94 @@ func (g *GraphQL) Do(query GraphQLQuery, response any, options ...RequestOptionF
		return resp, nil
		}

		// NewGraphQLUpload returns a GraphQLUpload that sends the data of content
		// under the given filename, which is required as GitLab does not treat a
		// part without a filename as an uploaded file. contentType is sent as the
		// file part's Content-Type and defaults to application/octet-stream.
		//
		// content is read in full and buffered in memory, so the returned upload is
		// independent of it: the query it is used in can be sent repeatedly and
		// retried, and a failure to build one part of a query does not leave the
		// other uploads half consumed. content is not closed, so a caller that
		// opened a file keeps ownership of it.
		//
		// Experimental: see [GraphQLUpload].
		func NewGraphQLUpload(content io.Reader, filename, contentType string) (*GraphQLUpload, error) {
		if content == nil {
		return nil, fmt.Errorf("GraphQL upload %q has no content", filename)
		}
		if filename == "" {
		return nil, errors.New("GraphQL upload has no filename")
		}
		if contentType == "" {
		contentType = graphQLUploadDefaultContentType
		}

		buf, err := io.ReadAll(content)
		if err != nil {
		return nil, fmt.Errorf("failed to read content of GraphQL upload %q: %w", filename, err)
		}

		return &GraphQLUpload{
		content: buf,
		filename: filename,
		contentType: contentType,
		}, nil
		}

		// MarshalJSON implements json.Marshaler and always returns a JSON null. The
		// GraphQL multipart request specification requires the query to hold null at
		// every upload's position, with the file sent as a separate part of the
		// multipart body.
		func (GraphQLUpload) MarshalJSON() ([]byte, error) {
		return []byte("null"), nil
		}

		// newUploadRequest builds the multipart/form-data request carrying query and
		// its uploads, following the GraphQL multipart request specification
		// (https://github.com/jaydenseric/graphql-multipart-request-spec).
		func (g GraphQL) newUploadRequest(query GraphQLQuery, uploads []graphql.Ref[GraphQLUpload], options []RequestOptionFunc) (retryablehttp.Request, error) {
		files := make([]graphql.File, 0, len(uploads))
		for _, ref := range uploads {
		// An upload that did not come from NewGraphQLUpload holds no content
		// and no filename, and cannot be sent as a file.
		if ref.Value.filename == "" {
		return nil, fmt.Errorf("GraphQL upload at %s was not created with NewGraphQLUpload", strings.Join(ref.Paths, ", "))
		}

		files = append(files, graphql.File{
		Filename: ref.Value.filename,
		ContentType: ref.Value.contentType,
		Content: ref.Value.content,
		Paths: ref.Paths,
		})
		}

		operations, err := json.Marshal(query)
		if err != nil {
		return nil, fmt.Errorf("failed to marshal GraphQL operations: %w", err)
		}

		body, contentType, err := graphql.BuildMultipartBody(operations, files)
		if err != nil {
		return nil, err
		}

		request, err := g.client.NewRequest(http.MethodPost, "", nil, options)
		if err != nil {
		return nil, err
		}

		// Set the body from a byte slice rather than a buffer, so that the
		// request stays replayable when it is retried.
		if err := request.SetBody(body); err != nil {
		return nil, fmt.Errorf("failed to set GraphQL multipart body: %w", err)
		}
		request.Header.Set("Content-Type", contentType)

		return request, nil
		}

		// gidGQL is a global ID. It is used by GraphQL to uniquely identify resources.
		type gidGQL struct {
		Type string

graphql_test.go

+409 −0

Original line number	Diff line number	Diff line
		@@ -2,9 +2,14 @@ package gitlab

		import (
		"encoding/json"
		"errors"
		"fmt"
		"io"
		"mime/multipart"
		"net/http"
		"strings"
		"testing"
		"testing/iotest"

		"github.com/stretchr/testify/assert"
		"github.com/stretchr/testify/require"
		@@ -127,6 +132,378 @@ func TestGraphQL_Do_Success_With_Variables(t *testing.T) {
		assert.Equal(t, "any-id", response.Data.Project.ID)
		}

		// mutationErrorsResponse is the response shape shared by the upload tests,
		// which all send a mutation and only care that it was accepted.
		type mutationErrorsResponse struct {
		Data struct {
		AchievementsCreate struct {
		Errors []string `json:"errors"`
		} `json:"achievementsCreate"`
		} `json:"data"`
		}

		const (
		uploadTestMutation = `mutation ($input: AchievementsCreateInput!) { achievementsCreate(input: $input) { errors } }`
		uploadListTestMutation = `mutation ($files: [Upload!]!) { designManagementUpload(files: $files) { errors } }`
		)

		// newTestUpload returns an upload of content, failing the test if it cannot
		// be created.
		func newTestUpload(t testing.T, content, filename, contentType string) GraphQLUpload {
		t.Helper()

		upload, err := NewGraphQLUpload(strings.NewReader(content), filename, contentType)
		require.NoError(t, err)

		return upload
		}

		// readUploadPart returns the content and header of the multipart file part
		// with the given name.
		func readUploadPart(t testing.T, r http.Request, name string) (string, *multipart.FileHeader) {
		t.Helper()

		file, header, err := r.FormFile(name)
		require.NoError(t, err)
		defer file.Close()

		content, err := io.ReadAll(file)
		require.NoError(t, err)

		return string(content), header
		}

		func TestGraphQL_Do_Success_With_FileUpload(t *testing.T) {
		t.Parallel()

		// GIVEN a query holding an upload in its variables
		mux, client := setup(t)
		mux.HandleFunc("/api/graphql", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, http.MethodPost)
		assert.Contains(t, r.Header.Get("Content-Type"), "multipart/form-data;")

		if !assert.NoError(t, r.ParseMultipartForm(1<<20)) {
		return
		}

		// THEN the upload is sent as null in the operations, pointed at by the
		// map, and carried by its own part
		assert.JSONEq(t,
		`{ "query": "`+uploadTestMutation+`", "variables": { "input": { "avatar": null, "name": "First Commit" } } }`,
		r.FormValue("operations"),
		)
		assert.JSONEq(t, `{ "0": ["variables.input.avatar"] }`, r.FormValue("map"))

		content, header := readUploadPart(t, r, "0")
		assert.Equal(t, "fake-avatar-bytes", content)
		assert.Equal(t, "avatar.png", header.Filename)
		assert.Equal(t, "image/png", header.Header.Get("Content-Type"))

		fmt.Fprint(w, `{ "data": { "achievementsCreate": { "errors": [] } } }`)
		})

		// WHEN
		var response mutationErrorsResponse
		_, err := client.GraphQL.Do(
		GraphQLQuery{
		Query: uploadTestMutation,
		Variables: map[string]any{
		"input": map[string]any{
		"name": "First Commit",
		"avatar": newTestUpload(t, "fake-avatar-bytes", "avatar.png", "image/png"),
		},
		},
		},
		&response)

		// THEN
		require.NoError(t, err)
		assert.Empty(t, response.Data.AchievementsCreate.Errors)
		}

		func TestGraphQL_Do_FileUpload_MultipleFiles(t *testing.T) {
		t.Parallel()

		// GIVEN a query holding two uploads, one of them without a content type
		mux, client := setup(t)
		mux.HandleFunc("/api/graphql", func(w http.ResponseWriter, r *http.Request) {
		if !assert.NoError(t, r.ParseMultipartForm(1<<20)) {
		return
		}

		// THEN each upload gets its own part, numbered by sorted path so that
		// the parts are the same on every run
		assert.JSONEq(t, `{
		"0": ["variables.input.attachment"],
		"1": ["variables.input.avatar"]
		}`, r.FormValue("map"))

		attachment, attachmentHeader := readUploadPart(t, r, "0")
		assert.Equal(t, "attachment-bytes", attachment)
		// AND an upload without a content type falls back to a generic one
		assert.Equal(t, "application/octet-stream", attachmentHeader.Header.Get("Content-Type"))

		avatar, avatarHeader := readUploadPart(t, r, "1")
		assert.Equal(t, "avatar-bytes", avatar)
		assert.Equal(t, "image/png", avatarHeader.Header.Get("Content-Type"))

		fmt.Fprint(w, `{ "data": { "achievementsCreate": { "errors": [] } } }`)
		})

		// WHEN
		var response mutationErrorsResponse
		_, err := client.GraphQL.Do(
		GraphQLQuery{
		Query: uploadTestMutation,
		Variables: map[string]any{
		"input": map[string]any{
		"avatar": newTestUpload(t, "avatar-bytes", "avatar.png", "image/png"),
		"attachment": newTestUpload(t, "attachment-bytes", "attachment.bin", ""),
		},
		},
		},
		&response)

		// THEN
		require.NoError(t, err)
		}

		func TestGraphQL_Do_FileUpload_ListVariable(t *testing.T) {
		t.Parallel()

		// GIVEN a query whose variable is a list of uploads
		mux, client := setup(t)
		mux.HandleFunc("/api/graphql", func(w http.ResponseWriter, r *http.Request) {
		if !assert.NoError(t, r.ParseMultipartForm(1<<20)) {
		return
		}

		// THEN the list keeps its length in the operations, with every element
		// nulled out and mapped by index
		assert.JSONEq(t,
		`{ "query": "`+uploadListTestMutation+`", "variables": { "files": [null, null] } }`,
		r.FormValue("operations"),
		)
		assert.JSONEq(t, `{
		"0": ["variables.files.0"],
		"1": ["variables.files.1"]
		}`, r.FormValue("map"))

		first, _ := readUploadPart(t, r, "0")
		assert.Equal(t, "first", first)
		second, _ := readUploadPart(t, r, "1")
		assert.Equal(t, "second", second)

		fmt.Fprint(w, `{ "data": { "achievementsCreate": { "errors": [] } } }`)
		})

		// WHEN
		var response mutationErrorsResponse
		_, err := client.GraphQL.Do(
		GraphQLQuery{
		Query: uploadListTestMutation,
		Variables: map[string]any{
		"files": []*GraphQLUpload{
		newTestUpload(t, "first", "first.png", ""),
		newTestUpload(t, "second", "second.png", ""),
		},
		},
		},
		&response)

		// THEN
		require.NoError(t, err)
		}

		func TestGraphQL_Do_FileUpload_SharedUpload(t *testing.T) {
		t.Parallel()

		// GIVEN one upload used by two variables
		upload := newTestUpload(t, "shared-bytes", "shared.png", "")

		mux, client := setup(t)
		mux.HandleFunc("/api/graphql", func(w http.ResponseWriter, r *http.Request) {
		if !assert.NoError(t, r.ParseMultipartForm(1<<20)) {
		return
		}

		// THEN its content is sent once, with both paths pointing at it
		assert.JSONEq(t, `{ "0": ["variables.avatar", "variables.logo"] }`, r.FormValue("map"))

		content, _ := readUploadPart(t, r, "0")
		assert.Equal(t, "shared-bytes", content)

		_, _, err := r.FormFile("1")
		assert.Error(t, err)

		fmt.Fprint(w, `{ "data": { "achievementsCreate": { "errors": [] } } }`)
		})

		// WHEN
		var response mutationErrorsResponse
		_, err := client.GraphQL.Do(
		GraphQLQuery{
		Variables: map[string]any{"avatar": upload, "logo": upload},
		},
		&response)

		// THEN
		require.NoError(t, err)
		}

		func TestNewGraphQLUpload(t *testing.T) {
		t.Parallel()

		t.Run("defaults the content type", func(t *testing.T) {
		t.Parallel()

		// WHEN an upload is created without a content type
		upload, err := NewGraphQLUpload(strings.NewReader("avatar-bytes"), "avatar.png", "")

		// THEN it falls back to a generic one and keeps a copy of the content
		require.NoError(t, err)
		assert.Equal(t, "avatar.png", upload.filename)
		assert.Equal(t, "application/octet-stream", upload.contentType)
		assert.Equal(t, "avatar-bytes", string(upload.content))
		})

		tests := []struct {
		name string
		content io.Reader
		filename string
		wantErr string
		}{
		{
		name: "missing content",
		filename: "avatar.png",
		wantErr: `GraphQL upload "avatar.png" has no content`,
		},
		{
		name: "missing filename",
		content: strings.NewReader("avatar-bytes"),
		wantErr: "GraphQL upload has no filename",
		},
		{
		name: "unreadable content",
		content: iotest.ErrReader(errors.New("broken content")),
		filename: "avatar.png",
		wantErr: `failed to read content of GraphQL upload "avatar.png": broken content`,
		},
		}

		for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
		t.Parallel()

		// WHEN an upload that cannot be sent is created
		upload, err := NewGraphQLUpload(tt.content, tt.filename, "")

		// THEN it is rejected right away, rather than when a query using
		// it is sent
		require.EqualError(t, err, tt.wantErr)
		assert.Nil(t, upload)
		})
		}
		}

		func TestGraphQL_Do_FileUpload_UploadWithoutConstructor(t *testing.T) {
		t.Parallel()

		// GIVEN an upload that was not created by NewGraphQLUpload, and so holds
		// nothing to send
		mux, client := setup(t)
		mux.HandleFunc("/api/graphql", func(_ http.ResponseWriter, _ *http.Request) {
		assert.Fail(t, "no request should be sent for an invalid upload")
		})

		// WHEN
		_, err := client.GraphQL.Do(
		GraphQLQuery{
		Query: uploadTestMutation,
		Variables: map[string]any{"input": map[string]any{"avatar": &GraphQLUpload{}}},
		},
		nil)

		// THEN the query fails before anything is sent
		require.Error(t, err)
		assert.Contains(t, err.Error(), "GraphQL upload at variables.input.avatar was not created with NewGraphQLUpload")
		}

		func TestGraphQL_Do_FileUpload_UploadIsReusable(t *testing.T) {
		t.Parallel()

		// GIVEN an upload sent by two separate queries
		var contents []string
		mux, client := setup(t)
		mux.HandleFunc("/api/graphql", func(w http.ResponseWriter, r *http.Request) {
		if !assert.NoError(t, r.ParseMultipartForm(1<<20)) {
		return
		}
		content, _ := readUploadPart(t, r, "0")
		contents = append(contents, content)

		fmt.Fprint(w, `{ "data": { "achievementsCreate": { "errors": [] } } }`)
		})

		upload := newTestUpload(t, "fake-avatar-bytes", "avatar.png", "")
		query := GraphQLQuery{
		Query: uploadTestMutation,
		Variables: map[string]any{"input": map[string]any{"avatar": upload}},
		}

		// WHEN
		var response mutationErrorsResponse
		_, err := client.GraphQL.Do(query, &response)
		require.NoError(t, err)

		_, err = client.GraphQL.Do(query, &response)
		require.NoError(t, err)

		// THEN both requests carry the file, as the upload holds its own copy of
		// the content instead of a reader that is drained by the first query
		assert.Equal(t, []string{"fake-avatar-bytes", "fake-avatar-bytes"}, contents)
		}

		func TestGraphQL_Do_FileUpload_RetriedRequestReplaysBody(t *testing.T) {
		t.Parallel()

		// GIVEN a server that fails the first attempt
		var bodies []string
		mux, client := setup(t)
		mux.HandleFunc("/api/graphql", func(w http.ResponseWriter, r *http.Request) {
		body, err := io.ReadAll(r.Body)
		if !assert.NoError(t, err) {
		return
		}
		bodies = append(bodies, string(body))

		if len(bodies) == 1 {
		w.WriteHeader(http.StatusInternalServerError)
		return
		}
		fmt.Fprint(w, `{ "data": { "achievementsCreate": { "errors": [] } } }`)
		})

		// WHEN
		var response mutationErrorsResponse
		_, err := client.GraphQL.Do(
		GraphQLQuery{
		Query: uploadTestMutation,
		Variables: map[string]any{
		"input": map[string]any{
		"avatar": newTestUpload(t, "fake-avatar-bytes", "avatar.png", ""),
		},
		},
		},
		&response)

		// THEN the retried request carries the same multipart body
		require.NoError(t, err)
		require.Len(t, bodies, 2)
		assert.Equal(t, bodies[0], bodies[1])
		assert.Contains(t, bodies[1], "fake-avatar-bytes")
		}

		func TestGraphQL_Do_ErrorWithMessages(t *testing.T) {
		t.Parallel()

		@@ -176,3 +553,35 @@ func TestGraphQL_Do_ErrorNoMessages(t *testing.T) {
		// THEN
		assert.ErrorContains(t, err, `{key: whuat} (no additional error messages)`)
		}

		// graphQLUploadOpaqueInput marshals itself, so the position of an upload
		// held inside it cannot be derived from its Go type.
		type graphQLUploadOpaqueInput struct {
		Avatar *GraphQLUpload
		}

		func (graphQLUploadOpaqueInput) MarshalJSON() ([]byte, error) {
		return []byte(`"opaque"`), nil
		}

		func TestGraphQL_Do_FileUpload_UploadInUndeterminablePosition(t *testing.T) {
		t.Parallel()

		// GIVEN a query holding an upload inside a type that marshals itself
		mux, client := setup(t)
		mux.HandleFunc("/api/graphql", func(http.ResponseWriter, *http.Request) {
		assert.Fail(t, "the query must not reach the server")
		})

		upload := newTestUpload(t, "avatar.png", "avatar-bytes", "image/png")

		// WHEN the query is sent
		_, err := client.GraphQL.Do(GraphQLQuery{
		Query: uploadTestMutation,
		Variables: map[string]any{"input": graphQLUploadOpaqueInput{Avatar: upload}},
		}, nil)

		// THEN it is refused, rather than silently sent without the file
		require.Error(t, err)
		assert.ErrorContains(t, err, "which marshals itself to JSON")
		}

internal/graphql/uploads.go

0 → 100644

+481 −0

File added.

Preview size limit exceeded, changes collapsed.

internal/graphql/uploads_test.go

0 → 100644

+856 −0

File added.

Preview size limit exceeded, changes collapsed.