Remove all Go related files ready for new architecture

parent b0e80c5f
/pkg/
/bin/
set-env-*.sh
*.db
.envrc
/node_modules/
*.log
*~
all:
echo 'Provide a target: pow clean'
minify:
curl -X POST -s --data-urlencode '[email protected]/s/js/app.js' https://javascript-minifier.com/raw > static/s/js/app.min.js
curl -X POST -s --data-urlencode '[email protected]/s/js/ie10.js' https://javascript-minifier.com/raw > static/s/js/ie10.min.js
curl -X POST -s --data-urlencode '[email protected]/s/css/styles.css' https://cssminifier.com/raw > static/s/css/styles.min.css
vendor:
gb vendor fetch github.com/boltdb/bolt
fmt:
find src/ -name '*.go' -exec go fmt {} ';'
build: fmt
gb build all
start: build
./bin/pow
test:
gb test -v
clean:
rm -rf bin/ pkg/
.PHONY: pow
# pow #
# cssminifier.com #
...
## Author ##
By [Andrew Chilton](https://chilts.org/), [@twitter](https://twitter.com/andychilton).
For [AppsAttic](https://appsattic.com/), [@AppsAttic](https://twitter.com/AppsAttic).
## License ##
[MIT](https://publish.li/mit-license-CPdxXSZb).
URL shortener. Simple, Quick and Fast!
(Ends)
__POW_NAKED_DOMAIN__ {
proxy / localhost:__POW_PORT__ {
transparent
}
tls [email protected]
log stdout
errors stderr
}
www.__POW_NAKED_DOMAIN__ {
redir http://__POW_NAKED_DOMAIN__{uri} 302
}
[program:gd-pow]
directory = /home/chilts/src/appsattic-pow.gd
command = /home/chilts/src/appsattic-pow.gd/bin/pow
user = chilts
autostart = true
autorestart = true
stdout_logfile = /var/log/chilts/gd-pow-stdout.log
stderr_logfile = /var/log/chilts/gd-pow-stderr.log
environment =
POW_PORT="__POW_PORT__",
POW_BASE_URL="__POW_BASE_URL__",
POW_NAKED_DOMAIN="__POW_NAKED_DOMAIN__",
POW_REDIS_ADDR="__POW_REDIS_ADDR__"
#!/bin/bash
## --------------------------------------------------------------------------------------------------------------------
#
# This script should be run from the parent directory as follows:
#
# ./scripts/00-release.sh
#
# It assumes the other scripts are also `./scripts/*.sh`.
#
## --------------------------------------------------------------------------------------------------------------------
set -e
./scripts/01-update-repo.sh
./scripts/02-install.sh
## --------------------------------------------------------------------------------------------------------------------
#!/bin/bash
## --------------------------------------------------------------------------------------------------------------------
set -e
git fetch
git rebase origin/master
## --------------------------------------------------------------------------------------------------------------------
#!/bin/bash
## --------------------------------------------------------------------------------------------------------------------
set -e
echo "Checking ask.sh is installed ..."
if [ ! $(which ask.sh) ]; then
echo "Please put ask.sh into ~/bin (should already be in your path from ~/.profile):"
echo ""
echo " mkdir ~/bin"
echo " wget -O ~/bin/ask.sh https://gist.githubusercontent.com/chilts/6b547307a6717d53e14f7403d58849dd/raw/ecead4db87ad4e7674efac5ab0e7a04845be642c/ask.sh"
echo " chmod +x ~/bin/ask.sh"
echo ""
exit 2
fi
echo
# General
POW_PORT=`ask.sh pow POW_PORT 'Which local port should the server listen on :'`
POW_NAKED_DOMAIN=`ask.sh pow POW_NAKED_DOMAIN 'What is the naked domain (e.g. localhost:1234 or pow.gd) :'`
POW_BASE_URL=`ask.sh pow POW_BASE_URL 'What is the base URL (e.g. http://localhost:1234 or https://pow.gd) :'`
POW_REDIS_ADDR=`ask.sh pow POW_REDIS_ADDR 'Which Redis server should be used for hits (e.g. ":6379") :'`
echo "Building code ..."
gb build
echo
# copy the supervisor script into place
echo "Copying supervisor config ..."
m4 \
-D __POW_PORT__=$POW_PORT \
-D __POW_NAKED_DOMAIN__=$POW_NAKED_DOMAIN \
-D __POW_BASE_URL__=$POW_BASE_URL \
-D __POW_REDIS_ADDR__=$POW_REDIS_ADDR \
etc/supervisor/conf.d/gd-pow.conf.m4 | sudo tee /etc/supervisor/conf.d/gd-pow.conf
echo
# restart supervisor
echo "Restarting supervisor ..."
sudo systemctl restart supervisor.service
echo
# copy the caddy conf
echo "Copying Caddy config config ..."
m4 \
-D __POW_PORT__=$POW_PORT \
-D __POW_NAKED_DOMAIN__=$POW_NAKED_DOMAIN \
-D __POW_BASE_URL__=$POW_BASE_URL \
-D __POW_REDIS_ADDR__=$POW_REDIS_ADDR \
etc/caddy/vhosts/gd.pow.conf.m4 | sudo tee /etc/caddy/vhosts/gd.pow.conf
echo
# restarting Caddy
echo "Restarting caddy ..."
sudo systemctl restart caddy.service
echo
## --------------------------------------------------------------------------------------------------------------------
package main
// https://gist.github.com/chilts/db1adfaddaae871b161d7eadab6b1278
import (
"bytes"
"html/template"
"log"
"net/http"
)
func serveFile(filename string) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, filename)
}
}
func fileServer(dirname string) http.Handler {
return http.FileServer(http.Dir(dirname))
}
func redirectFound(path string) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, path, http.StatusFound)
}
}
func redirectMovedPermanently(path string) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, path, http.StatusMovedPermanently)
}
}
func notFound(w http.ResponseWriter, r *http.Request) {
http.NotFound(w, r)
}
func internalServerError(w http.ResponseWriter, err error) {
log.Printf("Err: %s\n", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
}
func render(w http.ResponseWriter, tmpl *template.Template, tmplName string, data interface{}) {
buf := &bytes.Buffer{}
err := tmpl.ExecuteTemplate(buf, tmplName, data)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
buf.WriteTo(w)
}
package main
import (
"math/rand"
"time"
)
const idChars string = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
const idCharLen = len(idChars)
func init() {
rand.Seed(time.Now().UnixNano())
}
func Id(len int) string {
str := ""
for i := 0; i < len; i++ {
r := rand.Intn(idCharLen)
str = str + string(idChars[r])
}
return str
}
package main
import (
"errors"
"fmt"
"html/template"
"log"
"net/http"
"net/url"
"os"
"regexp"
"strings"
"time"
"github.com/boltdb/bolt"
"github.com/chilts/rod"
"github.com/garyburd/redigo/redis"
"github.com/gomiddleware/logger"
"github.com/gomiddleware/logit"
"github.com/gomiddleware/mux"
)
var urlBucketName = []byte("url")
var urlBucketNameStr = "url"
var statsBucketName = []byte("stats")
var statsBucketNameStr = "stats"
var doneBucketName = []byte("done") // as in "stats-done"
var doneBucketNameStr = "done" // as in "stats-done"
var (
ErrInvalidScheme = errors.New("URL scheme must be http or https")
ErrInvalidHost = errors.New("Host must contain letters/numbers, contain at least one dot and last component is at least 2 letters")
ErrHostCantHaveDashesHere = errors.New("Host can't have dashes next to dots anywhere")
ErrHostCantBeginEndWithDash = errors.New("Host can't begin or end with a dash")
)
var domainRegExp = regexp.MustCompile(`^([a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}$`)
var invalidDashRegExp = regexp.MustCompile(`(\.-)|(-\.)`)
var toDelete = []string{"RToXsy", "iyzqGc", "ZIWyvo"}
func check(err error) {
if err != nil {
log.Fatal(err)
}
}
func validateUrl(str string) (*url.URL, error) {
u, err := url.ParseRequestURI(str)
if err != nil {
return u, err
}
if u.Scheme != "https" && u.Scheme != "http" {
return u, ErrInvalidScheme
}
u.Host = strings.ToLower(u.Host)
if !domainRegExp.MatchString(u.Host) {
return u, ErrInvalidHost
}
// see if we match any of '.-' or '-.
if invalidDashRegExp.MatchString(u.Host) {
return u, ErrHostCantHaveDashesHere
}
// or if it starts or ends with a dash
if strings.HasPrefix(u.Host, "-") || strings.HasSuffix(u.Host, "-") {
return u, ErrHostCantBeginEndWithDash
}
return u, nil
}
func main() {
// setup the logger
lgr := logit.New(os.Stdout, "pow")
// setup
nakedDomain := os.Getenv("POW_NAKED_DOMAIN")
baseUrl := os.Getenv("POW_BASE_URL")
port := os.Getenv("POW_PORT")
if port == "" {
log.Fatal("Specify a port to listen on in the environment variable 'POW_PORT'")
}
// load up all templates
tmpl, err := template.New("").ParseGlob("./templates/*.html")
check(err)
// connect to Redis if specified
var redisPool *redis.Pool
redisAddr := os.Getenv("POW_REDIS_ADDR")
lgr.Print("redis-addr")
if redisAddr != "" {
fmt.Println("Configuring Redis")
redisPool = &redis.Pool{
MaxIdle: 3,
IdleTimeout: 240 * time.Second,
Dial: func() (redis.Conn, error) {
return redis.Dial("tcp", redisAddr)
},
}
lgr.Print("redis-configured")
} else {
fmt.Println("Not configuring Redis, hit counts will not function")
lgr.Print("redis-not-configured")
}
// open the datastore
db, err := bolt.Open("pow.db", 0600, &bolt.Options{Timeout: 1 * time.Second})
check(err)
defer db.Close()
// create the main buckets
err = db.Update(func(tx *bolt.Tx) error {
var err error
urlBucket, err := tx.CreateBucketIfNotExists(urlBucketName)
if err != nil {
return err
}
// delete abusive URLs
fmt.Printf("Removing URLs ...\n")
for _, v := range toDelete {
fmt.Printf("Removing URL=%s\n", v)
err = urlBucket.Delete([]byte(v))
if err != nil {
return err
}
}
fmt.Printf("Done\n")
_, err = tx.CreateBucketIfNotExists(statsBucketName)
if err != nil {
return err
}
return nil
})
check(err)
// Run the stats at regular intervals to process the hits from the previous hour.
go stats(redisPool, db)
// the mux
m := mux.New()
m.Use("/", logger.NewLogger(lgr))
// do some static routes before doing logging
m.All("/s", fileServer("static"))
m.Get("/favicon.ico", serveFile("./static/favicon.ico"))
m.Get("/robots.txt", serveFile("./static/robots.txt"))
m.Get("/", func(w http.ResponseWriter, r *http.Request) {
data := struct {
NakedDomain string
BaseUrl string
}{
nakedDomain,
baseUrl,
}
render(w, tmpl, "index.html", data)
})
m.Get("/new", func(w http.ResponseWriter, r *http.Request) {
data := struct {
BaseUrl string
}{
baseUrl,
}
render(w, tmpl, "new.html", data)
})
m.Post("/new", func(w http.ResponseWriter, r *http.Request) {
// validate the URL
u, err := validateUrl(r.FormValue("url"))
if err != nil {
internalServerError(w, err)
return
}
fmt.Printf("url=%s\n", u)
// setup a few things
var id string
now := time.Now().UTC()
shortUrl := ShortUrl{
Id: "", // filled in later
Url: u.String(),
Created: now,
Updated: now,
}
err = db.Update(func(tx *bolt.Tx) error {
// keep generating IDs until we find a unique one
for {
// generate a new Id
id = Id(6)
fmt.Printf("id=%s\n", id)
// see if it already exists
v, err := rod.Get(tx, urlBucketNameStr, id)
if err != nil {
return err
}
if v == nil {
// this id does not yet exist, so quite the loop
break
}
// ID exists, loop again ...
}
shortUrl.Id = id
return rod.PutJson(tx, urlBucketNameStr, id, shortUrl)
})
if err != nil {
internalServerError(w, err)
return
}
http.Redirect(w, r, "/"+id+"+", http.StatusFound)
})
m.Get("/:id", func(w http.ResponseWriter, r *http.Request) {
var preview bool
id := mux.Vals(r)["id"]
fmt.Printf("id=%s\n", id)
lgr := logger.LogFromRequest(r)
lgr.WithField("ShortUrlId", id)
// decide if we're redirecting or viewing the preview page (https://play.golang.org/p/Mkpb9gAzN1)
if strings.HasSuffix(id, "+") {
id = strings.TrimSuffix(id, "+")
preview = true
}
// get the shortUrl if it exists
var shortUrl *ShortUrl
err := db.View(func(tx *bolt.Tx) error {
return rod.GetJson(tx, urlBucketNameStr, id, &shortUrl)
})
if err != nil {
internalServerError(w, err)
return
}
if shortUrl == nil {
lgr.Print("no-short-url-found")
notFound(w, r)
return
}
if preview {
// get the stats (if it exists)
stats := Stats{}
err := db.View(func(tx *bolt.Tx) error {
return rod.GetJson(tx, statsBucketNameStr, id, &stats)
})
if err != nil {
internalServerError(w, err)
return
}
fmt.Printf("stats=%#v\n", stats)
lgr.Print("rendering-preview")
data := struct {
BaseUrl string
ShortUrl *ShortUrl
Stats *Stats
}{
baseUrl,
shortUrl,
&stats,
}
render(w, tmpl, "preview.html", data)
} else {
go incHits(redisPool, id)
http.Redirect(w, r, shortUrl.Url, http.StatusMovedPermanently)
}
})
// finally, check all routing was added correctly
check(m.Err)
// server
fmt.Printf("Starting server, listening on port %s\n", port)
errServer := http.ListenAndServe(":"+port, m)
check(errServer)
}
package main
import (
"fmt"
"log"
"time"
"github.com/boltdb/bolt"
"github.com/chilts/rod"
"github.com/garyburd/redigo/redis"
)
func incHits(pool *redis.Pool, id string) {
if pool == nil {
return
}
// get a connection
conn := pool.Get()
defer conn.Close()
fmt.Printf("incrementing hits for %s here\n", id)
// do ALL times in UTC
datetime := now().Format("20060102-15")
// just inc count:20060102-15:<id>
conn.Send("MULTI")
conn.Send("INCR", "count:"+datetime+":"+id)
conn.Send("SADD", "active:"+datetime, id)
_, err := conn.Do("EXEC")
if err != nil {
log.Printf("incHits: %s\n", err)
}
}
func stats(pool *redis.Pool, db *bolt.DB) {
if pool == nil {
log.Printf("Not setting up stats collection from Redis")
return
}
// every 15s, hit redis for 'active' ShortURLs
duration := time.Duration(15) * time.Second
ticker := time.NewTicker(duration)
for t := range ticker.C {
log.Println("Tick at", t)
processRandStat(pool, db)
}
}
func processRandStat(pool *redis.Pool, db *bolt.DB) {
// get a connection
conn := pool.Get()
defer conn.Close()
// do ALL times in UTC
t := now().Add(-60 * time.Minute)
datetime := t.Format("20060102-15")
fmt.Printf("Looking for an active ID in the previous hour ...\n")
// get one random ID
id, err := redis.String(conn.Do("SRANDMEMBER", "active:"+datetime))
if err != nil {
log.Printf(err.Error())
return
}
fmt.Printf("* id=%s\n", id)
// get it's hit count
hour := datetime + ":" + id
count, err := redis.Int64(conn.Do("GET", "count:"+hour))
if err != nil {
log.Printf(err.Error())
return
}
fmt.Printf("* count=%d\n", count)
// put these stats into Bolt
stats := Stats{}
err = db.Update(func(tx *bolt.Tx) error {
// firstly, let's see if these stats have already been processed
done, err := rod.GetString(tx, doneBucketNameStr, hour)
if err != nil {
return err
}
if done == "" {
fmt.Printf("not yet done\n")
// get the stats and increment the right slots
fmt.Printf("* 1 stats=%#v\n", stats)
rod.GetJson(tx, statsBucketNameStr, id, &stats)
fmt.Printf("* 2 stats=%#v\n", stats)
stats.Total += count
if stats.Daily == nil {
stats.Daily = make(map[string]int64)
}
stats.Daily[t.Format("20060102")] += count
if stats.Hourly == nil {
stats.Hourly = make(map[string]int64)
}
stats.Hourly[t.Format("15")] += count
if stats.DOTWly == nil {
stats.DOTWly = make(map[string]int64)
}
stats.DOTWly[t.Format("Mon")] += count
fmt.Printf("* 3 stats=%#v\n", stats)
//
err = rod.PutJson(tx, statsBucketNameStr, id, stats)
if err != nil {
return err
}
// and say we're done
err = rod.PutString(tx, doneBucketNameStr, hour, now().Format("20060102-150405.000000000"))
if err != nil {
return err
}
} else {
fmt.Printf("done:%s\n", done)
}
return nil
})
if err != nil {
log.Printf(err.Error())
}
// and finally, remove this hit from Redis
conn.Send("MULTI")
conn.Send("DEL", "count:"+hour)
conn.Send("SREM", "active:"+datetime, id)
_, err = conn.Do("EXEC")
if err != nil {
log.Printf("incHits: %s\n", err)
}
// ToDo: check if the "active:"+datetime now has zero members, and if so, remove the key
}