Commit df55acf6 authored by Lee Brown's avatar Lee Brown

add web-app to gitlab cicd

parent 2cd374a1
......@@ -34,7 +34,7 @@ cache:
.deploy_tmpl: &deploy_tmpl
<<: *job_tmpl
script:
- 'devops deploy -service=${SERVICE} -project=${PROJECT_NAME} -env=${TARGET_ENV} -enable_https=${ENABLE_HTTPS} -enable_elb=${ENABLE_ELB} -primary_host=${PRIMARY_HOST} -host_names=${HOST_NAMES} -private_bucket=${S3_BUCKET_PRIVATE} -public_bucket=${S3_BUCKET_PUBLIC}'
- 'devops deploy -service=${SERVICE} -project=${PROJECT_NAME} -env=${TARGET_ENV} -enable_https=${ENABLE_HTTPS} -enable_elb=${ENABLE_ELB} -primary_host=${PRIMARY_HOST} -host_names=${HOST_NAMES} -private_bucket=${S3_BUCKET_PRIVATE} -public_bucket=${S3_BUCKET_PUBLIC} -static_files_s3={STATIC_FILES_S3_ENABLED} -static_files_cloudfront={STATIC_FILES_CLOUDFRONT_ENABLED} -static_files_img_resize={STATIC_FILES_IMG_RESIZE_ENABLED}'
.migrate_tmpl: &migrate_tmpl
<<: *job_tmpl
......@@ -54,6 +54,45 @@ db:migrate:dev:
TARGET_ENV: 'dev'
AWS_USE_ROLE: 'true'
webapp:build:dev:
<<: *build_tmpl
stage: build:dev
tags:
- dev
only:
- master
- dev
- dev-web-app
variables:
TARGET_ENV: 'dev'
SERVICE: 'web-app'
AWS_USE_ROLE: 'true'
webapp:deploy:dev:
<<: *deploy_tmpl
stage: deploy:dev
tags:
- dev
only:
- master
- dev
- dev-web-app
dependencies:
- 'webapp:build:dev'
- 'db:migrate:dev'
variables:
TARGET_ENV: 'dev'
SERVICE: 'web-app'
ENABLE_HTTPS: 1
ENABLE_ELB: 0
PRIMARY_HOST: 'eproc.tech'
HOST_NAMES: 'www.eproc.tech, dev.eproc.tech'
S3_BUCKET_PRIVATE: 'saas-starter-kit-private'
S3_BUCKET_PUBLIC: 'saas-starter-kit-public'
STATIC_FILES_S3_ENABLED: 'true'
STATIC_FILES_CLOUDFRONT_ENABLED: 'false'
STATIC_FILES_IMG_RESIZE_ENABLED: 'true'
AWS_USE_ROLE: 'true'
webapi:build:dev:
<<: *build_tmpl
stage: build:dev
......@@ -84,14 +123,15 @@ webapi:deploy:dev:
SERVICE: 'web-api'
ENABLE_HTTPS: 1
ENABLE_ELB: 0
PRIMARY_HOST: 'eproc.tech'
HOST_NAMES: 'www.eproc.tech, api.eproc.tech'
PRIMARY_HOST: 'api.eproc.tech'
HOST_NAMES: 'api.dev.eproc.tech'
S3_BUCKET_PRIVATE: 'saas-starter-kit-private'
S3_BUCKET_PUBLIC: 'saas-starter-kit-public'
STATIC_FILES_S3_ENABLED: 'false'
STATIC_FILES_CLOUDFRONT_ENABLED: 'false'
STATIC_FILES_IMG_RESIZE_ENABLED: 'false'
AWS_USE_ROLE: 'true'
#ddlogscollector:deploy:stage:
# <<: *deploy_stage_tmpl
# variables:
......
......@@ -38,6 +38,7 @@ COPY --from=builder /static /static
COPY --from=builder /templates /templates
ENV TEMPLATE_DIR=/templates
ENV STATIC_DIR=/static
ARG service
ENV SERVICE_NAME $service
......
......@@ -20,8 +20,8 @@ import (
"syscall"
"time"
"geeks-accelerator/oss/saas-starter-kit/internal/mid"
"geeks-accelerator/oss/saas-starter-kit/cmd/web-app/handlers"
"geeks-accelerator/oss/saas-starter-kit/internal/mid"
"geeks-accelerator/oss/saas-starter-kit/internal/platform/devops"
"geeks-accelerator/oss/saas-starter-kit/internal/platform/flag"
img_resize "geeks-accelerator/oss/saas-starter-kit/internal/platform/img-resize"
......@@ -80,10 +80,10 @@ func main() {
HostNames []string `envconfig:"HOST_NAMES" example:"www.eproc.tech"`
EnableHTTPS bool `default:"false" envconfig:"ENABLE_HTTPS"`
TemplateDir string `default:"./templates" envconfig:"TEMPLATE_DIR"`
StaticDir string `default:"./static" envconfig:"STATIC_DIR"`
StaticS3 struct {
S3Enabled bool `envconfig:"ENABLED"`
S3KeyPrefix string `default:"public/web_app/static" envconfig:"KEY_PREFIX"`
StaticFiles struct {
Dir string `default:"./static" envconfig:"STATIC_DIR"`
S3Enabled bool `envconfig:"S3_ENABLED"`
S3Prefix string `default:"public/web_app/static" envconfig:"S3_PREFIX"`
CloudFrontEnabled bool `envconfig:"CLOUDFRONT_ENABLED"`
ImgResizeEnabled bool `envconfig:"IMG_RESIZE_ENABLED"`
}
......@@ -369,8 +369,8 @@ func main() {
// s3UrlFormatter is a help function used by to convert an s3 key to
// a publicly available image URL.
var staticS3UrlFormatter func(string) string
if cfg.Service.StaticS3.S3Enabled || cfg.Service.StaticS3.CloudFrontEnabled || cfg.Service.StaticS3.ImgResizeEnabled {
s3UrlFormatter, err := devops.S3UrlFormatter(awsSession, cfg.Aws.S3BucketPublic, cfg.Service.StaticS3.S3KeyPrefix, cfg.Service.StaticS3.CloudFrontEnabled)
if cfg.Service.StaticFiles.S3Enabled || cfg.Service.StaticFiles.CloudFrontEnabled || cfg.Service.StaticFiles.ImgResizeEnabled {
s3UrlFormatter, err := devops.S3UrlFormatter(awsSession, cfg.Aws.S3BucketPublic, cfg.Service.StaticFiles.S3Prefix, cfg.Service.StaticFiles.CloudFrontEnabled)
if err != nil {
log.Fatalf("main : S3UrlFormatter failed : %+v", err)
}
......@@ -379,7 +379,7 @@ func main() {
// When the path starts with a forward slash its referencing a local file,
// make sure the static file prefix is included
if strings.HasPrefix(p, "/") {
p = filepath.Join(cfg.Service.StaticS3.S3KeyPrefix, p)
p = filepath.Join(cfg.Service.StaticFiles.S3Prefix, p)
}
return s3UrlFormatter(p)
}
......@@ -400,7 +400,7 @@ func main() {
// templates should be updated to use a fully qualified URL for either the public file on S3
// on from the cloudfront distribution.
var staticUrlFormatter func(string) string
if cfg.Service.StaticS3.S3Enabled || cfg.Service.StaticS3.CloudFrontEnabled {
if cfg.Service.StaticFiles.S3Enabled || cfg.Service.StaticFiles.CloudFrontEnabled {
staticUrlFormatter = staticS3UrlFormatter
} else {
baseUrl, err := url.Parse(cfg.Service.BaseUrl)
......@@ -508,12 +508,12 @@ func main() {
// Image Formatter - additional functions exposed to templates for resizing images
// to support response web applications.
imgResizeS3KeyPrefix := filepath.Join(cfg.Service.StaticS3.S3KeyPrefix, "images/responsive")
imgResizeS3KeyPrefix := filepath.Join(cfg.Service.StaticFiles.S3Prefix, "images/responsive")
imgSrcAttr := func(ctx context.Context, p string, sizes []int, includeOrig bool) template.HTMLAttr {
u := staticUrlFormatter(p)
var srcAttr string
if cfg.Service.StaticS3.ImgResizeEnabled {
if cfg.Service.StaticFiles.ImgResizeEnabled {
srcAttr, _ = img_resize.S3ImgSrc(ctx, redisClient, staticS3UrlFormatter, awsSession, cfg.Aws.S3BucketPublic, imgResizeS3KeyPrefix, u, sizes, includeOrig)
} else {
srcAttr = fmt.Sprintf("src=\"%s\"", u)
......@@ -544,7 +544,7 @@ func main() {
}
tmplFuncs["S3ImgUrl"] = func(ctx context.Context, p string, size int) string {
imgUrl := staticUrlFormatter(p)
if cfg.Service.StaticS3.ImgResizeEnabled {
if cfg.Service.StaticFiles.ImgResizeEnabled {
imgUrl, _ = img_resize.S3ImgUrl(ctx, redisClient, staticS3UrlFormatter, awsSession, cfg.Aws.S3BucketPublic, imgResizeS3KeyPrefix, imgUrl, size)
}
return imgUrl
......@@ -635,7 +635,7 @@ func main() {
if cfg.HTTP.Host != "" {
api := http.Server{
Addr: cfg.HTTP.Host,
Handler: handlers.APP(shutdown, log, cfg.Service.StaticDir, cfg.Service.TemplateDir, masterDb, redisClient, renderer, serviceMiddlewares...),
Handler: handlers.APP(shutdown, log, cfg.Service.StaticFiles.Dir, cfg.Service.TemplateDir, masterDb, redisClient, renderer, serviceMiddlewares...),
ReadTimeout: cfg.HTTP.ReadTimeout,
WriteTimeout: cfg.HTTP.WriteTimeout,
MaxHeaderBytes: 1 << 20,
......@@ -652,7 +652,7 @@ func main() {
if cfg.HTTPS.Host != "" {
api := http.Server{
Addr: cfg.HTTPS.Host,
Handler: handlers.APP(shutdown, log, cfg.Service.StaticDir, cfg.Service.TemplateDir, masterDb, redisClient, renderer, serviceMiddlewares...),
Handler: handlers.APP(shutdown, log, cfg.Service.StaticFiles.Dir, cfg.Service.TemplateDir, masterDb, redisClient, renderer, serviceMiddlewares...),
ReadTimeout: cfg.HTTPS.ReadTimeout,
WriteTimeout: cfg.HTTPS.WriteTimeout,
MaxHeaderBytes: 1 << 20,
......
package devops
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)
// SyncS3StaticFiles copies the local files from the static directory to s3
// with public-read enabled.
func SyncS3StaticFiles(awsSession *session.Session, staticS3Bucket, staticS3Prefix, staticDir string) error {
uploader := s3manager.NewUploader(awsSession)
di := NewDirectoryIterator(staticS3Bucket, staticS3Prefix, staticDir, "public-read")
if err := uploader.UploadWithIterator(aws.BackgroundContext(), di); err != nil {
return err
}
return nil
}
......@@ -3,6 +3,7 @@ package cicd
import (
"encoding/json"
"fmt"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"io/ioutil"
"net/url"
"path/filepath"
......@@ -262,6 +263,18 @@ func EcrPurgeImages(req *serviceBuildRequest) ([]*ecr.ImageIdentifier, error) {
return delIds, nil
}
// SyncPublicS3Files copies the local files from the static directory to s3 with public-read enabled.
func SyncPublicS3Files(awsSession *session.Session, staticS3Bucket, staticS3Prefix, staticDir string) error {
uploader := s3manager.NewUploader(awsSession)
di := NewDirectoryIterator(staticS3Bucket, staticS3Prefix, staticDir, "public-read")
if err := uploader.UploadWithIterator(aws.BackgroundContext(), di); err != nil {
return err
}
return nil
}
// EcsReadTaskDefinition reads a task definition file and json decodes it.
func EcsReadTaskDefinition(serviceDir, targetEnv string) ([]byte, error) {
checkPaths := []string{
......
......@@ -142,19 +142,26 @@ func ecrRepositoryName(projectName string) string {
// releaseImage returns the name used for tagging a release image will always include one with environment and
// service name. If the env var CI_COMMIT_REF_NAME is set, it will be appended.
func releaseImage(env, serviceName, repositoryUri string) string {
func releaseTag(env, serviceName string) string {
tag1 := env + "-" + serviceName
// Generate tags for the release image.
var releaseImage string
var releaseTag string
if v := os.Getenv("CI_COMMIT_REF_NAME"); v != "" {
tag2 := tag1 + "-" + v
releaseImage = repositoryUri + ":" + tag2
releaseTag = tag2
} else {
releaseImage = repositoryUri + ":" + tag1
releaseTag = tag1
}
return releaseImage
return releaseTag
}
// releaseImage returns the name used for tagging a release image will always include one with environment and
// service name. If the env var CI_COMMIT_REF_NAME is set, it will be appended.
func releaseImage(env, serviceName, repositoryUri string) string {
return repositoryUri + ":" + releaseTag(env, serviceName)
}
// dBInstanceIdentifier returns the database name.
......
......@@ -64,7 +64,12 @@ type ServiceDeployFlags struct {
DockerFile string `validate:"omitempty" example:"./cmd/web-api/Dockerfile"`
EnableLambdaVPC bool `validate:"omitempty" example:"false"`
EnableEcsElb bool `validate:"omitempty" example:"false"`
RecreateService bool `validate:"omitempty" example:"false"`
StaticFilesS3Enable bool `validate:"omitempty" example:"false"`
StaticFilesCloudfrontEnable bool `validate:"omitempty" example:"false"`
StaticFilesImgResizeEnable bool `validate:"omitempty" example:"false"`
RecreateService bool `validate:"omitempty" example:"false"`
}
// serviceDeployRequest defines the details needed to execute a service deployment.
......@@ -105,10 +110,16 @@ type serviceDeployRequest struct {
CloudWatchLogGroupName string `validate:"required"`
CloudWatchLogGroup *cloudwatchlogs.CreateLogGroupInput
S3BucketTempPrefix string `validate:"required_with=S3BucketPrivateName S3BucketPublicName"`
S3BucketPrivateName string `validate:"omitempty"`
S3BucketPublicName string `validate:"omitempty"`
S3Buckets []S3Bucket
S3BucketTempPrefix string `validate:"required_with=S3BucketPrivateName S3BucketPublicName"`
S3BucketPrivateName string `validate:"omitempty"`
S3BucketPublicName string `validate:"omitempty"`
S3BucketPublicKeyPrefix string `validate:"omitempty"`
S3Buckets []S3Bucket
StaticFilesS3Enable bool `validate:"omitempty"`
StaticFilesS3Prefix string `validate:"omitempty"`
StaticFilesCloudfrontEnable bool `validate:"omitempty"`
StaticFilesImgResizeEnable bool `validate:"omitempty"`
EnableEcsElb bool `validate:"omitempty"`
ElbLoadBalancerName string `validate:"omitempty"`
......@@ -169,9 +180,14 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
req = serviceDeployRequest{
serviceRequest: sr,
EnableHTTPS: flags.EnableHTTPS,
ServiceHostPrimary: flags.ServiceHostPrimary,
ServiceHostNames: flags.ServiceHostNames,
EnableHTTPS: flags.EnableHTTPS,
ServiceHostPrimary: flags.ServiceHostPrimary,
ServiceHostNames: flags.ServiceHostNames,
StaticFilesS3Enable: flags.StaticFilesS3Enable,
StaticFilesCloudfrontEnable: flags.StaticFilesCloudfrontEnable,
StaticFilesImgResizeEnable: flags.StaticFilesImgResizeEnable,
S3BucketPrivateName: flags.S3BucketPrivateName,
S3BucketPublicName: flags.S3BucketPublicName,
EnableLambdaVPC: flags.EnableLambdaVPC,
......@@ -337,6 +353,16 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
})
}
// The S3 key prefix used as the origin when cloud front is enabled.
if req.S3BucketPublicKeyPrefix == "" {
req.S3BucketPublicKeyPrefix = "public"
}
// The S3 prefix used to upload static files served to public.
if req.StaticFilesS3Prefix == "" {
req.StaticFilesS3Prefix = filepath.Join(req.S3BucketPublicKeyPrefix, releaseTag(req.Env, req.ServiceName), "static")
}
// Set default AWS ECR Repository Name.
req.EcrRepositoryName = ecrRepositoryName(req.ProjectName)
log.Printf("\t\t\tSet ECR Repository Name to '%s'.", req.EcrRepositoryName)
......@@ -988,7 +1014,6 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
log.Printf("\t\t\tUpdated bucket policy")
}
}
log.Printf("\t%s\tS3 buckets configured successfully.\n", tests.Success)
}
......@@ -2307,6 +2332,11 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
"{HOST_PRIMARY}": req.ServiceHostPrimary,
"{HOST_NAMES}": strings.Join(req.ServiceHostNames, ","),
"{STATIC_FILES_S3_ENABLED}": "false",
"{STATIC_FILES_S3_PREFIX}": "",
"{STATIC_FILES_CLOUDFRONT_ENABLED}": "false",
"{STATIC_FILES_IMG_RESIZE_ENABLED}": "false",
"{CACHE_HOST}": "", // Not enabled by default
"{DB_HOST}": "",
......@@ -2359,6 +2389,21 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
placeholders["{APP_BASE_URL}"] = fmt.Sprintf("%s://%s/", appSchema, req.ServiceHostPrimary)
}
// Static files served from S3.
if req.StaticFilesS3Enable {
placeholders["{STATIC_FILES_S3_ENABLED}"] = "true"
}
// Static files served from CloudFront.
if req.StaticFilesCloudfrontEnable {
placeholders["{STATIC_FILES_CLOUDFRONT_ENABLED}"] = "true"
}
// Support for resizing static images files to be responsive.
if req.StaticFilesImgResizeEnable {
placeholders["{STATIC_FILES_IMG_RESIZE_ENABLED}"] = "true"
}
// When db is set, update the placeholders.
if db != nil {
placeholders["{DB_HOST}"] = db.Host
......@@ -3157,6 +3202,20 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
}
}
// When static files are enabled to be to stored on S3, we need to upload all of them.
if req.StaticFilesS3Enable {
log.Println("\tSync static files to public S3 bucket")
staticDir := filepath.Join(req.ServiceDir, "static")
err := SyncPublicS3Files(req.awsSession(), req.S3BucketPublicName, req.StaticFilesS3Prefix, staticDir)
if err != nil {
return errors.Wrapf(err, "Failed to sync static files from %s to s3://%s/%s '%s'", staticDir, req.S3BucketPublicName, req.StaticFilesS3Prefix)
}
log.Printf("\t%s\tFiles uploaded.\n", tests.Success)
}
// Wait for the updated or created service to enter a stable state.
{
log.Println("\tWaiting for service to enter stable state.")
......
......@@ -82,6 +82,9 @@ func main() {
cli.StringFlag{Name: "project", Usage: "name of project", Destination: &deployFlags.ProjectName},
cli.BoolFlag{Name: "enable_elb", Usage: "enable deployed to use Elastic Load Balancer", Destination: &deployFlags.EnableEcsElb},
cli.BoolTFlag{Name: "lambda_vpc", Usage: "deploy lambda behind VPC", Destination: &deployFlags.EnableLambdaVPC},
cli.BoolFlag{Name: "static_files_s3", Usage: "service static files from S3", Destination: &deployFlags.StaticFilesS3Enable},
cli.BoolFlag{Name: "static_files_cloudfront", Usage: "serve static files from Cloudfront", Destination: &deployFlags.StaticFilesCloudfrontEnable},
cli.BoolFlag{Name: "static_files_img_resize", Usage: "enable response images from service", Destination: &deployFlags.StaticFilesImgResizeEnable},
cli.BoolFlag{Name: "recreate_service", Usage: "skip docker push after build", Destination: &deployFlags.RecreateService},
},
Action: func(c *cli.Context) error {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment