Switch from the old stats to some new stats

parent 644be71c
......@@ -24,6 +24,8 @@ var urlBucketName = []byte("url")
var urlBucketNameStr = "url"
var statsBucketName = []byte("stats")
var statsBucketNameStr = "stats"
var doneBucketName = []byte("done") // as in "stats-done"
var doneBucketNameStr = "done" // as in "stats-done"
var (
ErrInvalidScheme = errors.New("URL scheme must be http or https")
......@@ -90,8 +92,9 @@ func main() {
// connect to Redis if specified
var redisPool *redis.Pool
redisAddr := os.Getenv("POW_REDIS_ADDR")
lgr.Print("redis-addr")
if redisAddr != "" {
log.Println("Configuring Redis")
fmt.Println("Configuring Redis")
redisPool = &redis.Pool{
MaxIdle: 3,
IdleTimeout: 240 * time.Second,
......@@ -99,8 +102,10 @@ func main() {
return redis.Dial("tcp", redisAddr)
},
}
lgr.Print("redis-configured")
} else {
log.Println("Not configuring Redis, hit counts will not function")
fmt.Println("Not configuring Redis, hit counts will not function")
lgr.Print("redis-not-configured")
}
// open the datastore
......@@ -127,8 +132,9 @@ func main() {
return nil
})
// Set off the go-routine which deals with collecting the stats from Redis
// and putting them into BoltDB.
// For now, just read in the old stats and write them out to BoltDB.
statsOld(redisPool, db)
// Run the stats at regular intervals to process the hits from the previous hour.
go stats(redisPool, db)
// the mux
......@@ -236,14 +242,14 @@ func main() {
return
}
if shortUrl == nil {
lgr.Log("no-short-url-found")
lgr.Print("no-short-url-found")
notFound(w, r)
return
}
if preview {
// get the stats (if it exists)
var stats *Stats
stats := Stats{}
err := db.View(func(tx *bolt.Tx) error {
return rod.GetJson(tx, statsBucketNameStr, id, &stats)
})
......@@ -253,7 +259,7 @@ func main() {
}
fmt.Printf("stats=%#v\n", stats)
lgr.Log("preview.html")
lgr.Print("rendering-preview")
data := struct {
BaseUrl string
ShortUrl *ShortUrl
......@@ -261,7 +267,7 @@ func main() {
}{
baseUrl,
shortUrl,
stats,
&stats,
}
render(w, tmpl, "preview.html", data)
} else {
......@@ -274,7 +280,7 @@ func main() {
check(m.Err)
// server
log.Printf("Starting server, listening on port %s\n", port)
fmt.Printf("Starting server, listening on port %s\n", port)
errServer := http.ListenAndServe(":"+port, m)
check(errServer)
}
......@@ -15,24 +15,19 @@ func incHits(pool *redis.Pool, id string) {
return
}
// get a connection
conn := pool.Get()
defer conn.Close()
fmt.Printf("incrementing hits for %s here\n", id)
// do ALL times in UTC
t := now()
day := t.Format("20060102")
hour := t.Format("15")
dotw := t.Format("Mon")
fmt.Printf("time=%s\n", t)
datetime := now().Format("20060102-15")
// just inc count:20060102-15:<id>
conn.Send("MULTI")
conn.Send("INCR", "hits:"+id+":total")
conn.Send("HINCRBY", "hits:"+id+":day", day, 1)
conn.Send("HINCRBY", "hits:"+id+":hour", hour, 1)
conn.Send("HINCRBY", "hits:"+id+":dotw", dotw, 1)
conn.Send("SADD", "active", id)
conn.Send("INCR", "count:"+datetime+":"+id)
conn.Send("SADD", "active:"+datetime, id)
_, err := conn.Do("EXEC")
if err != nil {
log.Printf("incHits: %s\n", err)
......@@ -46,31 +41,132 @@ func stats(pool *redis.Pool, db *bolt.DB) {
}
// every 10s, hit redis for 'active' ShortURLs
duration := time.Duration(10) * time.Second
// duration := time.Duration(10) * time.Second
duration := time.Duration(3) * time.Second
ticker := time.NewTicker(duration)
for t := range ticker.C {
log.Println("Tick at", t)
statsRand(pool, db)
processRandStat(pool, db)
}
}
func statsRand(pool *redis.Pool, db *bolt.DB) {
func processRandStat(pool *redis.Pool, db *bolt.DB) {
// get a connection
conn := pool.Get()
defer conn.Close()
// firstly, get a random active Short URL
ids, err := redis.Strings(conn.Do("SRANDMEMBER", "active", 1))
// do ALL times in UTC
t := now().Add(-60 * time.Minute)
datetime := t.Format("20060102-15")
fmt.Printf("Looking for an active ID in the previous hour ...\n")
// get one random ID
id, err := redis.String(conn.Do("SRANDMEMBER", "active:"+datetime))
if err != nil {
log.Printf("statsRand: %s\n", err)
log.Printf(err.Error())
return
}
fmt.Printf("* id=%s\n", id)
// get it's hit count
hour := datetime + ":" + id
count, err := redis.Int64(conn.Do("GET", "count:"+hour))
if err != nil {
log.Printf(err.Error())
return
}
fmt.Printf("* count=%d\n", count)
// put these stats into Bolt
stats := Stats{}
err = db.Update(func(tx *bolt.Tx) error {
// firstly, let's see if these stats have already been processed
done, err := rod.GetString(tx, doneBucketNameStr, hour)
if err != nil {
return err
}
if done == "" {
fmt.Printf("not yet done\n")
// get the stats and increment the right slots
fmt.Printf("* 1 stats=%#v\n", stats)
rod.GetJson(tx, statsBucketNameStr, id, &stats)
fmt.Printf("* 2 stats=%#v\n", stats)
stats.Total += count
if stats.Daily == nil {
stats.Daily = make(map[string]int64)
}
stats.Daily[t.Format("20060102")] += count
if stats.Hourly == nil {
stats.Hourly = make(map[string]int64)
}
stats.Hourly[t.Format("15")] += count
if stats.DOTWly == nil {
stats.DOTWly = make(map[string]int64)
}
stats.DOTWly[t.Format("Mon")] += count
fmt.Printf("* 3 stats=%#v\n", stats)
//
err = rod.PutJson(tx, statsBucketNameStr, id, stats)
if err != nil {
return err
}
// and say we're done
err = rod.PutString(tx, doneBucketNameStr, hour, now().Format("20060102-150405.000000000"))
if err != nil {
return err
}
} else {
fmt.Printf("done:%s\n", done)
}
return nil
})
if err != nil {
log.Printf(err.Error())
}
// and finally, remove this hit from Redis
conn.Send("MULTI")
conn.Send("DEL", "count:"+hour)
conn.Send("SREM", "active:"+datetime, id)
_, err = conn.Do("EXEC")
if err != nil {
log.Printf("incHits: %s\n", err)
}
// ToDo: check if the "active:"+datetime now has zero members, and if so, remove the key
}
func statsOld(pool *redis.Pool, db *bolt.DB) {
if pool == nil {
log.Printf("Not setting up stats collection from Redis")
return
}
if len(ids) == 0 {
// no Short URLs are currently active
// get a connection
conn := pool.Get()
defer conn.Close()
// get all of the 'active' IDs
active, err := redis.Strings(conn.Do("SMEMBERS", "active"))
if err != nil {
log.Printf(err.Error())
return
}
id := ids[0]
fmt.Printf("id=%s\n", id)
for _, id := range active {
fmt.Printf("active=%s\n", id)
statsRandOld(pool, db, id)
}
}
func statsRandOld(pool *redis.Pool, db *bolt.DB, id string) {
// get a connection
conn := pool.Get()
defer conn.Close()
// total
total, err := redis.Int64(conn.Do("GET", "hits:"+id+":total"))
......@@ -117,4 +213,17 @@ func statsRand(pool *redis.Pool, db *bolt.DB) {
if err != nil {
log.Printf("statsRand: %s\n", err)
}
// finally, remove the keys used here, and remove it from the original set
// just inc count:20060102-15:<id>
conn.Send("MULTI")
conn.Send("DEL", "hits:"+id+":total")
conn.Send("DEL", "hits:"+id+":day")
conn.Send("DEL", "hits:"+id+":hour")
conn.Send("DEL", "hits:"+id+":dotw")
conn.Send("SREM", "active", id)
_, err = conn.Do("EXEC")
if err != nil {
log.Printf("incHits: %s\n", err)
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment