Build built-in steps when building the step-runner

All built-in step binaries will be embedded into the step-runner binary

.gitignore

@@ -6,3 +6,5 @@ step-runner
 .docker-env
 Library/
 out/
+steps/bin/*
+!steps/bin/README.md

.gitlab/ci/test.gitlab-ci.yml

@@ -22,7 +22,9 @@ check-generated:
 
 go-test:
   stage: test
-  image: golang:$GO_VERSION
+  image: golang:${GO_VERSION}
+  before_script:
+    - make build-builtin-steps
   script:
     - go install gotest.tools/gotestsum@latest
     - gotestsum --junitfile=report.xml

Makefile

@@ -40,6 +40,8 @@ $(TARGETS): GOOS=$(firstword $(subst /, ,$@))
 $(TARGETS): GOARCH=$(lastword $(subst /, ,$@))
 $(TARGETS): BINARY=$(BIN_PATH)/step-runner-$(subst /,-,$@)
 $(TARGETS):
+	@TARGET=$@ $(MAKE) build-builtin-steps
+
 	@mkdir -p $(BIN_PATH)
 	CGO_ENABLED=0 GOOS="$(GOOS)" GOARCH="$(GOARCH)" go build \
 		-ldflags '-X "$(MODULE_NAME)/cmd.stepRunnerVersion=$(STEP_RUNNER_VERSION)"' \
@@ -74,7 +76,7 @@ generate:
 	$(MAKE) .generate-proto
 
 .PHONY: test
-test: generate
+test: generate build-builtin-steps
 	CI_STEPS_DEBUG=true go test ./...
 	@git --no-pager diff --compact-summary --exit-code -- go.mod go.sum && echo 'Go modules are tidy and complete!'
 	@git --no-pager diff --compact-summary --exit-code -- ./internal/plugin/proto && echo 'proto code is up-to-date!'
@@ -113,7 +115,7 @@ $(PROTOVALIDATE_DIST):
 	@rm "$(local)/protovalidate.zip"
 
 .PHONY: clean
-clean:
+clean: clean-builtin-steps
 	@rm -rf $(BIN_PATH)
 
 .PHONY: image
@@ -134,3 +136,15 @@ $(GOIMPORTS):
 go-fmt: DIRECTORY := ./pkg ./cmd main.go
 go-fmt: $(GOIMPORTS)
 	$(GOIMPORTS) -w -local gitlab.com/gitlab-org/step-runner $(DIRECTORY)
+
+.PHONY: build-builtin-steps
+build-builtin-steps: TARGET ?= $(LOCAL_OS_ARCH)
+build-builtin-steps: STEP_DIRS = $(shell find steps/source -type f -name Makefile -exec dirname {} \;)
+build-builtin-steps:
+	@for dir in $(STEP_DIRS); do \
+		TARGET="$(TARGET)" $(MAKE) -C $$dir build; \
+	done
+
+.PHONY: clean-builtin-steps
+clean-builtin-steps:
+	@find steps/bin -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \;

steps/bin/README.md

+## What is the purpose of this README?
+
+To explain to engineers the purpose of this folder, and to ensure that the `//go:embed bin`
+directive has something to embed on a new machine when built-in steps have not been generated.
+
+## What is the bin folder for?
+
+The `steps/bin` directory hosts "built-in" steps. These are generated from the built-in steps found in
+`steps/source`.
+
+Like any other steps, built-in steps are run in a separate process to the step-runner. Each built-in step
+is written to the file system before being executed.
+
+## How to generate built-in steps
+
+From the step-runner source folder, run `make build` to build the step-runner and dependent built-in steps.

steps/source/oci/publish/Makefile

+STEP_BIN_PATH := $(subst steps/source/,steps/bin/,$(CURDIR))
+LOCAL_OS_ARCH := $(lastword $(shell go version))
+TARGET ?= $(LOCAL_OS_ARCH)
+
+.PHONY: create-bin-step-dir
+create-bin-step-dir:
+	@rm -rf $(STEP_BIN_PATH)
+	@mkdir -p $(STEP_BIN_PATH)
+	@cp step.yml $(STEP_BIN_PATH)/step.yml
+
+# $(TARGET) compiles a go binary for the specified target platform
+.PHONY: $(TARGET)
+$(TARGET): GOOS=$(firstword $(subst /, ,$@))
+$(TARGET): GOARCH=$(lastword $(subst /, ,$@))
+$(TARGET): BINARY=$(STEP_BIN_PATH)/run
+$(TARGET):
+	CGO_ENABLED=0 GOOS="$(GOOS)" GOARCH="$(GOARCH)" go build -o "$(BINARY)"
+
+# build compiles the step into the bin directory
+build: create-bin-step-dir $(TARGET)

steps/source/oci/publish/main.go

+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+)
+
+var (
+	outputFile = flag.String("output", "", "")
+)
+
+func main() {
+	flag.Parse()
+
+	if outputFile == nil || *outputFile == "" {
+		panic("must specify an output file to write to")
+	}
+
+	err := os.WriteFile(*outputFile, []byte(`{"name":"message", "value":"publish step"}`), 0640)
+	if err != nil {
+		panic(fmt.Errorf("cannot write to output file: %w", err))
+	}
+}

steps/source/oci/publish/step.yml

+spec:
+  outputs:
+    message:
+      type: string
+---
+exec:
+  command: ["${{step_dir}}/run", "--output", "${{output_file}}"]