Commit d7dc7ed7 authored by Kamil Trzciński's avatar Kamil Trzciński

Add simple GitLab Pages daemon with custom CNAME and TLS support

parent 61debe70
# Created by .ignore support plugin (hsz.mobi)
## GitLab Pages Daemon
This is simple HTTP server written in Go made to serve GitLab Pages with CNAMEs and SNI using HTTP/HTTP2.
This is made to work in small-to-medium scale environments.
In large environment it can be time consuming to list all directories, and CNAMEs.
### How it generates routes
1. It reads the `pages-root` directory to list all groups
2. It looks for `CNAME` files in `pages-root/group/project` directory, reads them and creates mapping for custom CNAMEs.
3. It generates virtual-host from these data.
4. Periodically (every second) it checks the `pages-root` directory if it was modified to reload all mappings.
To force route refresh, CNAME reload or TLS certificate reload: `touch pages-root`.
It will be done asynchronously, not interrupting current requests.
### How it serves content
1. When client initiates the TLS connection, the GitLab-Pages daemon looks in hash map for virtual hosts and tries to load TLS certificate from:
`pages-root/group/project/domain.{crt,key}`.
2. When client asks HTTP server the GitLab-Pages daemon looks in hash map for registered virtual hosts.
3. The URL.Path is split into `/<project>/<subpath>` and we daemon tries to load: `pages-root/group/project/public/subpath`.
4. If file was not found it will try to load `pages-root/group/<host>/public/<URL.Path>`.
5. If requested path is directory, the `index.html` will be served.
### How it should be run?
Ideally the GitLab Pages should run without load balancer.
If load balancer is required, the HTTP can be served in HTTP mode.
For HTTPS traffic load balancer should be run in TCP-mode.
If load balancer is run in SSL-offloading mode the custom TLS certificate will not work.
### How to run it
```
go build
./gitlab-pages -listen-https "" -listen-http ":8090" -pages-root path/to/gitlab/shared/pages
```
### License
MIT
package main
import (
"net/http"
"os"
"path/filepath"
"strings"
"crypto/tls"
"errors"
)
type domain struct {
Group string
Project string
CNAME bool
certificate *tls.Certificate
}
func (d *domain) notFound(w http.ResponseWriter, r *http.Request) {
http.NotFound(w, r)
}
func (d *domain) tryFile(w http.ResponseWriter, r *http.Request, projectName, subPath string) bool {
publicPath := filepath.Join(*pagesRoot, d.Group, projectName, "public")
fullPath := filepath.Join(publicPath, subPath)
fullPath = filepath.Clean(fullPath)
if !strings.HasPrefix(fullPath, publicPath) {
return false
}
fi, err := os.Lstat(fullPath)
if err != nil {
return false
}
// If this file is directory, open the index.html
if fi.IsDir() {
fullPath = filepath.Join(fullPath, "index.html")
fi, err = os.Lstat(fullPath)
if err != nil {
return false
}
}
// We don't allow to open non-regular files
if !fi.Mode().IsRegular() {
return false
}
// Open and serve content of file
file, err := os.Open(fullPath)
if err != nil {
return false
}
defer file.Close()
fi, err = file.Stat()
if err != nil {
return false
}
http.ServeContent(w, r, filepath.Base(file.Name()), fi.ModTime(), file)
return true
}
func (d *domain) serverGroup(w http.ResponseWriter, r *http.Request) {
// The Path always contains "/" at the beggining
split := strings.SplitN(r.URL.Path, "/", 3)
if len(split) >= 2 {
subPath := ""
if len(split) >= 3 {
subPath = split[2]
}
if d.tryFile(w, r, split[1], subPath) {
return
}
}
if d.tryFile(w, r, strings.ToLower(r.Host), r.URL.Path) {
return
}
d.notFound(w, r)
}
func (d *domain) serveCNAME(w http.ResponseWriter, r *http.Request) {
if d.tryFile(w, r, d.Project, r.URL.Path) {
return
}
d.notFound(w, r)
}
func (d *domain) ensureCertificate() (*tls.Certificate, error) {
if !d.CNAME {
return nil, errors.New("tls certificates can be loaded only for pages with CNAME")
}
if d.certificate != nil {
return d.certificate, nil
}
// Load keypair from shared/pages/group/project/domain.{crt,key}
certificateFile := filepath.Join(*pagesRoot, d.Group, d.Project, "domain.crt")
keyFile := filepath.Join(*pagesRoot, d.Group, d.Project, "domain.key")
tls, err := tls.LoadX509KeyPair(certificateFile, keyFile)
if err != nil {
return nil, err
}
d.certificate = &tls
return d.certificate, nil
}
func (d *domain) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if d.CNAME {
d.serveCNAME(w, r)
} else {
d.serverGroup(w, r)
}
}
package main
import (
"io/ioutil"
"log"
"os"
"path/filepath"
"strings"
"time"
)
type domains map[string]domain
type domainsUpdater func(domains domains)
func readGroups(domains domains) error {
groups, err := filepath.Glob(filepath.Join(*pagesRoot, "*/"))
if err != nil {
return err
}
for _, groupDir := range groups {
group := filepath.Base(groupDir)
groupName := strings.ToLower(group)
domains[groupName+"."+*pagesDomain] = domain{
Group: group,
CNAME: false,
}
}
return nil
}
func readCnames(domains domains) error {
cnames, err := filepath.Glob(filepath.Join(*pagesRoot, "*/*/CNAME"))
if err != nil {
return err
}
for _, cnamePath := range cnames {
cnameData, err := ioutil.ReadFile(cnamePath)
if err != nil {
continue
}
for _, cname := range strings.Fields(string(cnameData)) {
cname := strings.ToLower(cname)
if strings.HasSuffix(cname, "."+*pagesDomain) {
continue
}
domains[cname] = domain{
// TODO: make it nicer
Group: filepath.Base(filepath.Dir(filepath.Dir(cnamePath))),
Project: filepath.Base(filepath.Dir(cnamePath)),
CNAME: true,
}
}
}
return nil
}
func watchDomains(updater domainsUpdater) {
var lastModified time.Time
for {
fi, err := os.Stat(*pagesRoot)
if err != nil || !fi.IsDir() {
log.Println("Failed to read domains from", *pagesRoot, "due to:", err, fi.IsDir())
time.Sleep(time.Second)
continue
}
// If directory did not get modified we will reload
if !lastModified.Before(fi.ModTime()) {
time.Sleep(time.Second)
continue
}
lastModified = fi.ModTime()
started := time.Now()
domains := make(domains)
readGroups(domains)
readCnames(domains)
duration := time.Since(started)
log.Println("Updated", len(domains), "domains in", duration)
if updater != nil {
updater(domains)
}
}
}
package main
import (
"fmt"
"net/http"
"time"
)
type loggingResponseWriter struct {
rw http.ResponseWriter
status int
written int64
started time.Time
}
func newLoggingResponseWriter(rw http.ResponseWriter) loggingResponseWriter {
return loggingResponseWriter{
rw: rw,
started: time.Now(),
}
}
func (l *loggingResponseWriter) Header() http.Header {
return l.rw.Header()
}
func (l *loggingResponseWriter) Write(data []byte) (n int, err error) {
if l.status == 0 {
l.WriteHeader(http.StatusOK)
}
n, err = l.rw.Write(data)
l.written += int64(n)
return
}
func (l *loggingResponseWriter) WriteHeader(status int) {
if l.status != 0 {
return
}
l.status = status
l.rw.WriteHeader(status)
}
func (l *loggingResponseWriter) Log(r *http.Request) {
duration := time.Since(l.started)
fmt.Printf("%s %s - - [%s] %q %d %d %q %q %f\n",
r.Host, r.RemoteAddr, l.started,
fmt.Sprintf("%s %s %s", r.Method, r.RequestURI, r.Proto),
l.status, l.written, r.Referer(), r.UserAgent(), duration.Seconds(),
)
}
package main
import (
"crypto/tls"
"flag"
"fmt"
"log"
"net/http"
"strings"
"sync"
)
var listenHTTP = flag.String("listen-http", ":80", "The address to listen for HTTP requests")
var listenHTTPS = flag.String("listen-https", "", "The address to listen for HTTPS requests")
var pagesDomain = flag.String("pages-domain", "gitlab-example.com", "The domain to serve static pages")
var pagesRootCert = flag.String("root-cert", "", "The default certificate to serve static pages")
var pagesRootKey = flag.String("root-key", "", "The default certificate to serve static pages")
var serverHTTP = flag.Bool("serve-http", true, "Serve the pages under HTTP")
var http2proto = flag.Bool("http2", true, "Enable HTTP2 support")
var pagesRoot = flag.String("pages-root", "shared/pages", "The directory where pages are stored")
type theApp struct {
domains domains
}
func (a *theApp) ServeTLS(ch *tls.ClientHelloInfo) (*tls.Certificate, error) {
if ch.ServerName == "" {
return nil, nil
}
host := strings.ToLower(ch.ServerName)
if domain, ok := a.domains[host]; ok {
tls, _ := domain.ensureCertificate()
return tls, nil
}
return nil, nil
}
func (a *theApp) ServeHTTP(ww http.ResponseWriter, r *http.Request) {
w := newLoggingResponseWriter(ww)
defer w.Log(r)
// Add auto redirect
if r.TLS == nil && !*serverHTTP {
u := *r.URL
u.Scheme = "https"
u.Host = r.Host
u.User = nil
http.Redirect(&w, r, u.String(), 307)
return
}
host := strings.ToLower(r.Host)
domain, ok := a.domains[host]
if !ok {
http.NotFound(&w, r)
return
}
// Serve static file
domain.ServeHTTP(&w, r)
}
func (a *theApp) UpdateDomains(domains domains) {
fmt.Printf("Domains: %v", domains)
a.domains = domains
}
func main() {
var wg sync.WaitGroup
var app theApp
flag.Parse()
// Listen for HTTP
if *listenHTTP != "" {
wg.Add(1)
go func() {
defer wg.Done()
err := ListenAndServe(*listenHTTP, &app)
if err != nil {
log.Fatal(err)
}
}()
}
// Listen for HTTPS
if *listenHTTPS != "" {
wg.Add(1)
go func() {
defer wg.Done()
err := ListenAndServeTLS(*listenHTTPS, *pagesRootCert, *pagesRootKey, &app)
if err != nil {
log.Fatal(err)
}
}()
}
go watchDomains(app.UpdateDomains)
wg.Wait()
}
package main
import (
"crypto/tls"
"golang.org/x/net/http2"
"net/http"
)
type TLSHandler interface {
http.Handler
ServeTLS(*tls.ClientHelloInfo) (*tls.Certificate, error)
}
func ListenAndServe(addr string, handler http.Handler) error {
// create server
server := &http.Server{Addr: addr, Handler: handler}
if *http2proto {
err := http2.ConfigureServer(server, &http2.Server{})
if err != nil {
return err
}
}
return server.ListenAndServe()
}
func ListenAndServeTLS(addr string, certFile, keyFile string, handler TLSHandler) error {
// create server
server := &http.Server{Addr: addr, Handler: handler}
server.TLSConfig = &tls.Config{}
server.TLSConfig.GetCertificate = handler.ServeTLS
if *http2proto {
err := http2.ConfigureServer(server, &http2.Server{})
if err != nil {
return err
}
}
return server.ListenAndServeTLS(certFile, keyFile)
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment