Commit 5678d183 authored by badsectorlabs's avatar badsectorlabs

Initial commit

parents
__pycache__
output
*.pyc
.idea
.DS_Store
*.pid
images-source
\ No newline at end of file
variables:
# Set git strategy, recursive in case there are submodules
GIT_STRATEGY: clone
GIT_SUBMODULE_STRATEGY: recursive
# Keys and secrets are defined in the project CI settings and exposed as env variables
AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
AWS_DEFAULT_REGION: "us-east-1"
# Define two stages, if the site fails to build it will not be deployed
stages:
- build
- deploy
build:
stage: build
image: apihackers/pelican # This image contains everything needed to build a static pelican site
artifacts: # artifacts are files that will be passed to the next CI stage and can be downloaded from the GitLab web
# frontend as zips
paths:
- output # This is the directory we want to save and pass to the next stage
expire_in: 1 week # Keep it around for a week in case we need to roll back
script: # The script block is the series of commands that will be run in the container defined in `image`
- pelican content -o output -s publishconf.py # Build the site using the publish config into the output directory
- ls -lart output
only:
- master # Only run this step on the master branch. No reason to spend resources on incomplete feature branches
deploy-prod:
stage: deploy
image: badsectorlabs/aws-compress-and-deploy # This is a custom image for minifying and working with AWS
variables: # You can set per-stage variables like this
DESC: "Prod build, commit: $CI_COMMIT_SHA" # There are tons of built in env variables during the CI process
S3_BUCKET: blog.badsectorlabs.com
CLOUDFRONT_DISTRIBUTION_ID: $CLOUDFRONT_DISTRIBUTION # Again, the secrets are stored in GitLab, not in the code!
script:
- cd output # Assumes the static site is in 'output' which is automatically created because the last step had
# 'output' as an artifact
- echo [+] ls before minification
- ls -lart .
- echo "$DESC" > version.html
- echo [+] minifying HTML
- find . -iname \*.html | xargs -I {} htmlminify -o {} {}
- echo [+] minifying CSS
- find . -iname \*.css | xargs -I {} uglifycss --output {} {}
- echo [+] minifying JS
- find . -iname \*.js | xargs -I {} uglifyjs -o {} {}
- echo [+] ls after minification
- ls -lart .
- echo [+] Syncing all files to $S3_BUCKET
- aws s3 sync . s3://$S3_BUCKET --region us-east-2
- echo [+] Invalidating Cloudfront cache # This step is necessary or you wont see the changes until the TTL expires
- aws cloudfront create-invalidation --distribution-id $CLOUDFRONT_DISTRIBUTION_ID --paths '/*'
environment: # environments are just ways to control what is deployed where, for a simple blog straight to prod is ok
name: master-prod
only:
- master
when: manual # This causes GitLab to wait until you click the run button before executing this stage
PY?=python3
PELICAN?=pelican
PELICANOPTS=
BASEDIR=$(CURDIR)
INPUTDIR=$(BASEDIR)/content
OUTPUTDIR=$(BASEDIR)/output
CONFFILE=$(BASEDIR)/pelicanconf.py
PUBLISHCONF=$(BASEDIR)/publishconf.py
FTP_HOST=localhost
FTP_USER=anonymous
FTP_TARGET_DIR=/
SSH_HOST=localhost
SSH_PORT=22
SSH_USER=root
SSH_TARGET_DIR=/var/www
S3_BUCKET=blog.badsectorlabs.com
CLOUDFILES_USERNAME=my_rackspace_username
CLOUDFILES_API_KEY=my_rackspace_api_key
CLOUDFILES_CONTAINER=my_cloudfiles_container
DROPBOX_DIR=~/Dropbox/Public/
GITHUB_PAGES_BRANCH=gh-pages
DEBUG ?= 0
ifeq ($(DEBUG), 1)
PELICANOPTS += -D
endif
RELATIVE ?= 0
ifeq ($(RELATIVE), 1)
PELICANOPTS += --relative-urls
endif
help:
@echo 'Makefile for a pelican Web site '
@echo ' '
@echo 'Usage: '
@echo ' make html (re)generate the web site '
@echo ' make clean remove the generated files '
@echo ' make regenerate regenerate files upon modification '
@echo ' make publish generate using production settings '
@echo ' make serve [PORT=8000] serve site at http://localhost:8000'
@echo ' make serve-global [SERVER=0.0.0.0] serve (as root) to $(SERVER):80 '
@echo ' make devserver [PORT=8000] start/restart develop_server.sh '
@echo ' make stopserver stop local server '
@echo ' make ssh_upload upload the web site via SSH '
@echo ' make rsync_upload upload the web site via rsync+ssh '
@echo ' make dropbox_upload upload the web site via Dropbox '
@echo ' make ftp_upload upload the web site via FTP '
@echo ' make s3_upload upload the web site via S3 '
@echo ' make cf_upload upload the web site via Cloud Files'
@echo ' make github upload the web site via gh-pages '
@echo ' '
@echo 'Set the DEBUG variable to 1 to enable debugging, e.g. make DEBUG=1 html '
@echo 'Set the RELATIVE variable to 1 to enable relative urls '
@echo ' '
html:
$(PELICAN) $(INPUTDIR) -o $(OUTPUTDIR) -s $(CONFFILE) $(PELICANOPTS)
clean:
[ ! -d $(OUTPUTDIR) ] || rm -rf $(OUTPUTDIR)
regenerate:
$(PELICAN) -r $(INPUTDIR) -o $(OUTPUTDIR) -s $(CONFFILE) $(PELICANOPTS)
serve:
ifdef PORT
cd $(OUTPUTDIR) && $(PY) -m pelican.server $(PORT)
else
cd $(OUTPUTDIR) && $(PY) -m pelican.server
endif
serve-global:
ifdef SERVER
cd $(OUTPUTDIR) && $(PY) -m pelican.server 80 $(SERVER)
else
cd $(OUTPUTDIR) && $(PY) -m pelican.server 80 0.0.0.0
endif
devserver:
ifdef PORT
$(BASEDIR)/develop_server.sh restart $(PORT)
else
$(BASEDIR)/develop_server.sh restart
endif
stopserver:
$(BASEDIR)/develop_server.sh stop
@echo 'Stopped Pelican and SimpleHTTPServer processes running in background.'
publish:
$(PELICAN) $(INPUTDIR) -o $(OUTPUTDIR) -s $(PUBLISHCONF) $(PELICANOPTS)
ssh_upload: publish
scp -P $(SSH_PORT) -r $(OUTPUTDIR)/* $(SSH_USER)@$(SSH_HOST):$(SSH_TARGET_DIR)
rsync_upload: publish
rsync -e "ssh -p $(SSH_PORT)" -P -rvzc --delete $(OUTPUTDIR)/ $(SSH_USER)@$(SSH_HOST):$(SSH_TARGET_DIR) --cvs-exclude
dropbox_upload: publish
cp -r $(OUTPUTDIR)/* $(DROPBOX_DIR)
ftp_upload: publish
lftp ftp://$(FTP_USER)@$(FTP_HOST) -e "mirror -R $(OUTPUTDIR) $(FTP_TARGET_DIR) ; quit"
s3_upload: publish
s3cmd sync $(OUTPUTDIR)/ s3://$(S3_BUCKET) --acl-public --delete-removed --guess-mime-type --no-mime-magic --no-preserve
cf_upload: publish
cd $(OUTPUTDIR) && swift -v -A https://auth.api.rackspacecloud.com/v1.0 -U $(CLOUDFILES_USERNAME) -K $(CLOUDFILES_API_KEY) upload -c $(CLOUDFILES_CONTAINER) .
github: publish
ghp-import -m "Generate Pelican site" -b $(GITHUB_PAGES_BRANCH) $(OUTPUTDIR)
git push origin $(GITHUB_PAGES_BRANCH)
.PHONY: html help clean regenerate serve serve-global devserver stopserver publish ssh_upload rsync_upload dropbox_upload ftp_upload s3_upload cf_upload github
# Bad Sector Labs Blog
This blog is a static site built with [Pelican](https://blog.getpelican.com/) and based on the
[m.css](http://mcss.mosra.cz/) theme.
For more details, read
[this blog post](https://blog.badsectorlabs.com/pelican-gitlab-cicd-docker-aws-awesome-static-site.html).
Pelican + Gitlab CI/CD + Docker + AWS = Awesome Static Site
###########################################################
:date: 2018-02-26 12:04
:category: DevOps
:tags: Gitlab, DevOps, Python
:summary: How I leverage Gitlab CI/CD and Pelican to create a static blog hosted on AWS S3
.. role:: raw-html(raw)
:format: html
All the code referenced in this post (and even this post itself) is available
`on gitlab <https://gitlab.com/badsectorlabs/blog>`_.
Choosing a static site generator
================================
Setting out to start a blog, there are tons of options. The classic `Wordpress <https://wordpress.com/>`_,
the upstart `ghost <https://ghost.org/>`_ or the many static site generators like
`jeckyll <https://jekyllrb.com/>`_, `Hugo <https://gohugo.io/>`_, `Octopress <http://octopress.org/>`_ (abandoned), and
`Pelican <https://blog.getpelican.com/>`_. After looking at each option, I settled on pelican because I wanted a static
site (one less server to deal with), it's written in Python (one of my preferred languages) and it has an `extensive
library of themes <http://www.pelicanthemes.com/>`_ to use as a base. I decided on the `m.css theme
<http://mcss.mosra.cz/>`_ because I liked its dark theme and lack of javascript (shout out to anyone reading this via
the Tor browser) and it has great support for code. I've only had to make a few small tweaks to m.css to make it my own.
Getting started with Pelican is simple, follow the m.css `quickstart <http://mcss.mosra.cz/pelican/#quick-start>`_.
Pelican has a cool feature which makes tweaking themes or writing content easy - the devserver.
.. code:: bash
$ cd /dir/of/pelican/blog
$ make devserver
<lots of output>
Pelican and HTTP server processes now running in background.
$
Now Pelican is watching your files for changes, and will re-compile articles when you save a change. Keep an eye on the
terminal running the devserver though, if a change causes an error in Pelican it will show up there and your browser
will not see anything new.
Creating content is as easy as writing a `reStructuredText <http://docutils.sourceforge.net/rst.html>`_ document in the
``content`` directory. reStructuredText is awesome, and if you've used Markdown (Pelican also supports Markdown if you
prefer) before, it has the same general feel. The m.css
:raw-html:`<em><a href="http://mcss.mosra.cz/pelican/writing-content/">writing content</a></em>` guide is a great primer
on reST. The only issue I have with reST is that markup can't be nested, so italicising a link is not as simple as
wrapping it in ``*``. For instance, you would think that last *writing content* link would be written as
.. code:: rst
*`writing content <http://mcss.mosra.cz/pelican/writing-content/>`_*
But that is not allowed, so you have to define a directive at the top of the document to allow raw HTML and use it
in-line later, as so:
.. code:: rst
.. At the top of the document before any content
.. role:: raw-html(raw)
:format: html
.. In-line
:raw-html:`<em><a href="http://mcss.mosra.cz/pelican/writing-content/">writing content</a></em>`
Hosting a static site
=====================
Just like static site generators, there are a few static site hosts to choose from:
`Google <https://cloud.google.com/storage/docs/hosting-static-website>`_, `GitHub Pages <https://pages.github.com/>`_,
`GitLab Pages <https://about.gitlab.com/features/pages/>`_, and `Amazon's S3 <https://aws.amazon.com/s3/>`_.
I choose S3, mostly because I am already familiar with AWS and am using it extensively for another project
(`hamiltix.net <https://www.hamiltix.net>`_) which will be detailed in a future post. For the first 12 months on AWS you
get 5GB of S3 storage free, as well as 20k get requests and 2k put requests per month. Combine this with
`Cloudfront <https://aws.amazon.com/cloudfront/>`_ (AWS's CDN) and even if reddit tries to hug you to death you should
have no issues keeping your site up. In fact, if you want to use SSL/TLS with your S3 static site (hint:
`you do <https://security.googleblog.com/2016/09/moving-towards-more-secure-web.html>`_) you *have* to use Cloudfront.
Instead of walking through another S3 and Cloudfront setup, just follow the `same guide I used
<https://medium.com/@sbuckpesch/setup-aws-s3-static-website-hosting-using-ssl-acm-34d41d32e394>`_.
CI/CD - Putting it all together, automatically
==============================================
This is where the magic happens. On every push to master, your static site should build, minify, upload, and invalidate
the Cloudfront cache. This way you can write a post in a feature branch, and when you merge it into master your blog
updates without any additional actions! Gitlab is my git host of choice because it can be self-hosted and is very
powerful. Additionally, Gitlab.com offers unlimited free private repos with unlimited collaborators. But my favorite
feature of Gitlab is its built-in CI/CD. No longer do you need a seperate service to test/build/deploy your code, it's
all built right into your version control. Layer docker on top of this and you get easy, reporducable builds and all it
takes is one yaml file in the root of your repo!
Getting started with Gitlab CI/CD can be a little intimidating, and I've found using other projects ``gitlab-ci.yml``
files as templates is the best way to get started. For instance, here is the ``gitlab-ci.yml`` file for this blog
(if you're on mobile, sorry in advance; there is no good way to show code on mobile without wrapping which kills
context):
.. code:: yaml
variables:
# Set git strategy, recursive in case there are submodules
GIT_STRATEGY: clone
GIT_SUBMODULE_STRATEGY: recursive
# Keys and secrets are defined in the project CI settings and exposed as env variables
AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
AWS_DEFAULT_REGION: "us-east-1"
# Define two stages, if the site fails to build it will not be deployed
stages:
- build
- deploy
build:
stage: build
image: apihackers/pelican # This image contains everything needed to build a static pelican site
artifacts: # artifacts are files that will be passed to the next CI stage and can be downloaded from the GitLab web
# frontend as zips
paths:
- output # This is the directory we want to save and pass to the next stage
expire_in: 1 week # Keep it around for a week in case we need to roll back
script: # The script block is the series of commands that will be run in the container defined in `image`
- pelican content -o output -s publishconf.py # Build the site using the publish config into the output directory
- ls -lart output
only:
- master # Only run this step on the master branch. No reason to spend resources on incomplete feature branches
deploy-prod:
stage: deploy
image: badsectorlabs/aws-compress-and-deploy # This is a custom image for minifying and working with AWS
variables: # You can set per-stage variables like this
DESC: "Prod build, commit: $CI_COMMIT_SHA" # There are tons of built in env variables during the CI process
S3_BUCKET: blog.badsectorlabs.com
CLOUDFRONT_DISTRIBUTION_ID: $CLOUDFRONT_DISTRIBUTION # Again, the secrets are stored in GitLab, not in the code!
script:
- cd output # Assumes the static site is in 'output' which is automatically created because the last step had
# 'output' as an artifact
- echo [+] ls before minification
- ls -lart .
- echo "$DESC" > version.html
- echo [+] minifying HTML
- find . -iname \*.html | xargs -I {} htmlminify -o {} {}
- echo [+] minifying CSS
- find . -iname \*.css | xargs -I {} uglifycss --output {} {}
- echo [+] minifying JS
- find . -iname \*.js | xargs -I {} uglifyjs -o {} {}
- echo [+] ls after minification
- ls -lart .
- echo [+] Syncing all files to $S3_BUCKET
- aws s3 sync . s3://$S3_BUCKET --region us-east-2
- echo [+] Invalidating Cloudfront cache # This step is necessary or you wont see the changes until the TTL expires
- aws cloudfront create-invalidation --distribution-id $CLOUDFRONT_DISTRIBUTION_ID --paths '/*'
environment: # environments are just ways to control what is deployed where, for a simple blog straight to prod is ok
name: master-prod
only:
- master
when: manual # This causes GitLab to wait until you click the run button before executing this stage
.. block-success:: Success!
And there you have it. A clean, static, no-javascript blog with posts you can write in reST and deploy with a git
push.
Questions or comments?
blog (at) badsectorlabs.com
#!/usr/bin/env bash
##
# This section should match your Makefile
##
PY=${PY:-python3}
PELICAN=${PELICAN:-pelican}
PELICANOPTS=
BASEDIR=$(pwd)
INPUTDIR=$BASEDIR/content
OUTPUTDIR=$BASEDIR/output
CONFFILE=$BASEDIR/pelicanconf.py
###
# Don't change stuff below here unless you are sure
###
SRV_PID=$BASEDIR/srv.pid
PELICAN_PID=$BASEDIR/pelican.pid
function usage(){
echo "usage: $0 (stop) (start) (restart) [port]"
echo "This starts Pelican in debug and reload mode and then launches"
echo "an HTTP server to help site development. It doesn't read"
echo "your Pelican settings, so if you edit any paths in your Makefile"
echo "you will need to edit your settings as well."
exit 3
}
function alive() {
kill -0 $1 >/dev/null 2>&1
}
function shut_down(){
PID=$(cat $SRV_PID)
if [[ $? -eq 0 ]]; then
if alive $PID; then
echo "Stopping HTTP server"
kill $PID
else
echo "Stale PID, deleting"
fi
rm $SRV_PID
else
echo "HTTP server PIDFile not found"