Commit e442c816 authored by John Jarvis's avatar John Jarvis

Updates to the oncall robot report

* pull production issues for incidents and changes
* add severity
* temporarily remove reports
parent 65a3c252
Pipeline #28132437 passed with stage
in 1 minute and 50 seconds
......@@ -4,14 +4,17 @@
[[projects]]
branch = "master"
name = "github.com/GeertJohan/go.rice"
packages = [".","embedded"]
packages = [
".",
"embedded"
]
revision = "c02ca9a983da5807ddf7d796784928f5be4afd09"
[[projects]]
branch = "master"
name = "github.com/PagerDuty/go-pagerduty"
packages = ["."]
revision = "078a3284fb0ee5256d2164b53b9ed44ff3b3b05e"
revision = "5f93c380e5f988559a9ed7bc11a60b3db7e06ef3"
[[projects]]
branch = "master"
......@@ -38,10 +41,10 @@
version = "v0.6.2"
[[projects]]
branch = "v2"
name = "gopkg.in/yaml.v2"
packages = ["."]
revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
version = "v2.2.1"
[solve-meta]
analyzer-name = "dep"
......
......@@ -38,6 +38,7 @@ type Project struct {
// Projects for project ids
type Projects struct {
Infrastructure Project `yaml:"infrastructure"`
Production Project `yaml:"production"`
ReportProject Project `yaml:"report_project"`
}
......
......@@ -39,6 +39,11 @@ func main() {
if config.Projects.Infrastructure.ID == 0 {
log.Fatalln("ERROR: It looks like you have an old settings file, please update using oncall-settings-example.yaml")
}
if config.Projects.Production.ID == 0 {
log.Fatalln("ERROR: It looks like you have an old settings file, you are missing the production project ID. please update using oncall-settings-example.yaml")
}
opts := oncall.ReportOptions{}
report := oncall.NewReport(config, opts)
title := "OnCall report for period: " +
......
......@@ -3,6 +3,7 @@ package oncall
import (
"fmt"
"log"
"strings"
"time"
gitlab "github.com/xanzy/go-gitlab"
......@@ -19,18 +20,20 @@ type GitLabHelper struct {
// GitLabHelperOptions options for the api helper
type GitLabHelperOptions struct {
APIToken string
ProjectID int
DayOffset int
APIToken string
ProjectID int
ProjectIDProd int
DayOffset int
}
// NewGitLabHelper creates a new api helper
func NewGitLabHelper(opts GitLabHelperOptions) *GitLabHelper {
r := GitLabHelper{
client: newGitLabClient(opts.APIToken),
apiToken: opts.APIToken,
projectID: opts.ProjectID,
dayOffset: opts.DayOffset,
client: newGitLabClient(opts.APIToken),
apiToken: opts.APIToken,
projectID: opts.ProjectID,
projectIDProd: opts.ProjectIDProd,
dayOffset: opts.DayOffset,
}
return &r
}
......@@ -89,7 +92,7 @@ func (g *GitLabHelper) GetIssuesOpenedDuringShift() []*gitlab.Issue {
return allIssues
}
// GetIssuesOpenedDuringShift gets opened issues during shift
// GetIssuesOpenedDuringShiftProd gets opened issues during shift for production issues
func (g *GitLabHelper) GetIssuesOpenedDuringShiftProd() []*gitlab.Issue {
var opts gitlab.ListProjectIssuesOptions
opts.ListOptions.PerPage = 100
......@@ -170,6 +173,16 @@ func (g *GitLabHelper) UploadFile(projectID int, fname string) *gitlab.ProjectFi
return pf
}
func filterSeverityLabels(labels []string) (filteredLabels []string) {
// severityTest := func(s string) bool { return strings.HasPrefix(s, "S") && len(s) <= 2 }
for _, l := range labels {
if strings.HasPrefix(l, "S") && len(l) <= 2 {
filteredLabels = append(filteredLabels, l)
}
}
return
}
func filterIssuesByLabel(label string, issues []*gitlab.Issue) (filteredIssues []*gitlab.Issue) {
for _, p := range issues {
if stringInSlice(label, p.Labels) {
......@@ -189,6 +202,15 @@ func filterIssuesByState(state string, issues []*gitlab.Issue) (filteredIssues [
return
}
func choose(ss []string, test func(string) bool) (ret []string) {
for _, s := range ss {
if test(s) {
ret = append(ret, s)
}
}
return
}
func stringInSlice(a string, list []string) bool {
for _, b := range list {
if b == a {
......
......@@ -2,17 +2,13 @@ package oncall
import (
"bytes"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"strings"
"sync"
"text/template"
"time"
"github.com/GeertJohan/go.rice"
gitlab "github.com/xanzy/go-gitlab"
config "gitlab.com/gl-infra/oncall-robot-assistant/config"
)
......@@ -24,7 +20,7 @@ type GraphData struct {
Info string
}
// ReportHelpers for clients needed for hte report
// ReportHelpers for clients needed for the report
type ReportHelpers struct {
GitLab *GitLabHelper
GitLabDev *GitLabHelper
......@@ -54,52 +50,33 @@ func NewReport(config *config.Config, opts ReportOptions) *Report {
// * Uploads the graphs to the project
// * Creates an issue using the issue template and returns the link
func (r *Report) CreateReportIssue(title string) string {
graphs := r.downloadGraphs()
weeklyOpsGraphs := r.uploadGraphs(graphs)
// Not downloading performance graphs for now
// graphs := r.downloadGraphs()
// weeklyOpsGraphs := r.uploadGraphs(graphs)
weeklyOpsGraphs := []WeeklyOpsGraph{}
desc := r.generateTemplate(weeklyOpsGraphs)
issue := r.Helpers.GitLab.CreateIssue(title, desc, r.Config.Projects.ReportProject.ID)
return issue.WebURL
}
// Upload graphs to the gitlab project
func (r *Report) uploadGraphs(graphs <-chan GraphData) (opsGraphs []WeeklyOpsGraph) {
log.Println("Uploading graphs to GitLab project ...")
start := time.Now()
for graph := range graphs {
// defer os.Remove(graph.Fname)
opsGraph := WeeklyOpsGraph{}
log.Println(" uploading", graph.Name)
pf := r.Helpers.GitLab.UploadFile(r.Config.Projects.ReportProject.ID, graph.Fname)
opsGraph.URL = pf.URL
opsGraph.Name = graph.Name
opsGraphs = append(opsGraphs, opsGraph)
}
secs := time.Since(start).Seconds()
log.Printf(" Completed upload in %.2f seconds\n", secs)
return opsGraphs
}
// Downloads graphs from Grafana
func (r *Report) downloadGraphs() (graphs <-chan GraphData) {
log.Println("Downloading graphs from performance ...")
start := time.Now()
var wg sync.WaitGroup
numGraphs := len(weeklyOpsGraphs(r.Config))
ch := make(chan GraphData, numGraphs)
wg.Add(numGraphs)
for _, graph := range weeklyOpsGraphs(r.Config) {
go makeRequest(graph.URL, graph.Name, ch, &wg, r.Config.APITokens.Grafana)
}
wg.Wait()
close(ch)
secs := time.Since(start).Seconds()
log.Printf(" %d graphs took %.2f seconds to download\n", numGraphs, secs)
return ch
// issue := r.Helpers.GitLab.CreateIssue(title, desc, r.Config.Projects.ReportProject.ID)
// return issue.WebURL
return desc
}
// Generate an issue using a template
func (r *Report) generateTemplate(weeklyOpsGraphs []WeeklyOpsGraph) string {
// Fetch all issues opened during shift in both trackers
// Infrastructure tracker
log.Println("Fetching infrastructure issues")
shiftIssues := r.Helpers.GitLab.GetIssuesOpenedDuringShift()
allOpenIssues := r.Helpers.GitLab.GetIssuesOpenAll()
// Production tracker
log.Println("Fetching production issues")
shiftIssuesProd := r.Helpers.GitLab.GetIssuesOpenedDuringShiftProd()
allOpenIssuesProd := r.Helpers.GitLab.GetIssuesOpenAllProd()
templateData := TemplateData{}
// PagerDuty team members
for _, schd := range r.Config.PagerDuty.Schedules {
onCallUsers := getOncallPersons(r.Config, schd.ID)
for _, u := range onCallUsers {
......@@ -110,9 +87,9 @@ func (r *Report) generateTemplate(weeklyOpsGraphs []WeeklyOpsGraph) string {
}
}
// PagerDuty incidents
incidents := getOncallIncidents(r.Config)
templateData.IncidentCount = len(incidents)
// Populate list of incidents for the template
for _, p := range incidents {
inc := Incident{}
inc.Summary = strings.Replace(p.Summary, "#", "", -1)
......@@ -120,62 +97,45 @@ func (r *Report) generateTemplate(weeklyOpsGraphs []WeeklyOpsGraph) string {
inc.CreatedAt = p.CreatedAt
templateData.Incidents = append(templateData.Incidents, inc)
}
shiftIssues := r.Helpers.GitLab.GetIssuesOpenedDuringShift()
allOpenIssues := r.Helpers.GitLab.GetIssuesOpenAll()
shiftIssuesProd := r.Helpers.GitLab.GetIssuesOpenedDuringShiftProd()
allOpenIssuesProd := r.Helpers.GitLab.GetIssuesOpenAllProd()
/*
WeeklyOpsGraphs []WeeklyOpsGraph
TeamMembers []TeamMember
Incidents []Incident
OnCallIssues []Issue
OpenChangeIssues []Issue
OpenIncidentIssues []Issue
CriticalIssues []Issue
ChangeIssues []Issue
IncidentIssues []Issue
OutageIssues []Issue
CorrectiveActionIssues []Issue
IncidentCount int
IssuesOpenAll IssueStats
IssuesOpenedDuringShift IssueStats
*/
// Open issue stats
// Issue stats
templateData.IssuesOpenedDuringShift.Count = len(shiftIssues)
templateData.IssuesOpenedDuringShift.OnCall = len(filterIssuesByLabel("oncall", shiftIssues))
templateData.IssuesOpenedDuringShift.AccessRequest = len(filterIssuesByLabel("access request", shiftIssues))
templateData.IssuesOpenedDuringShift.Critical = len(filterIssuesByLabel("critical", shiftIssues))
templateData.IssuesOpenedDuringShift.CorrectiveAction = len(filterIssuesByLabel("corrective action", shiftIssues))
templateData.IssuesOpenedDuringShift.Change = len(filterIssuesByLabel("change", shiftIssuesProd))
templateData.IssuesOpenedDuringShift.Incident = len(filterIssuesByLabel("incident", shiftIssuesProd))
templateData.IssuesOpenAll.Count = len(allOpenIssues) + len(allOpenIssuesProd)
templateData.IssuesOpenAll.OnCall = len(filterIssuesByLabel("oncall", allOpenIssues))
templateData.IssuesOpenAll.Change = len(filterIssuesByLabel("change", allOpenIssuesProd))
templateData.IssuesOpenAll.Incident = len(filterIssuesByLabel("incident", allOpenIssuesProd))
templateData.IssuesOpenAll.AccessRequest = len(filterIssuesByLabel("access request", allOpenIssues))
templateData.IssuesOpenAll.Critical = len(filterIssuesByLabel("critical", allOpenIssues))
templateData.IssuesOpenAll.CorrectiveAction = len(filterIssuesByLabel("corrective action", allOpenIssues))
templateData.IssuesOpenAll.Change = len(filterIssuesByLabel("corrective action", allOpenIssues))
templateData.IssuesOpenAll.Incident = len(filterIssuesByLabel("corrective action", allOpenIssues))
// Populate list of issues for the template - oncall issues
// Open incident issues
for _, p := range filterIssuesByLabel("incident", allOpenIssuesProd) {
templateData.OpenIncidentIssues = append(templateData.OpenIncidentIssues, *genIssueInfo(p))
}
// Open change issues
for _, p := range filterIssuesByLabel("change", allOpenIssuesProd) {
templateData.OpenChangeIssues = append(templateData.OpenChangeIssues, *genIssueInfo(p))
}
// Open oncall issues
for _, p := range filterIssuesByLabel("oncall", allOpenIssues) {
issue := Issue{}
created := p.CreatedAt.Format(time.RFC822)
if p.Assignee.Username == "" {
p.Assignee.Username = "unassigned"
}
issue.CreatedAt = created
issue.Summary = p.Title
issue.URL = p.WebURL
issue.Assignee = p.Assignee.Username
templateData.OnCallIssues = append(templateData.OnCallIssues, issue)
templateData.OpenOnCallIssues = append(templateData.OpenOnCallIssues, *genIssueInfo(p))
}
// 7 day issue stats
templateData.IssuesOpenedDuringShift.OnCall = len(filterIssuesByLabel("oncall", shiftIssues))
templateData.IssuesOpenedDuringShift.AccessRequest = len(filterIssuesByLabel("access request", shiftIssues))
templateData.IssuesOpenedDuringShift.Change = len(filterIssuesByLabel("change", shiftIssuesProd))
templateData.IssuesOpenedDuringShift.Incident = len(filterIssuesByLabel("incident", shiftIssuesProd))
// Shift incident issues
for _, p := range filterIssuesByLabel("incident", shiftIssuesProd) {
templateData.IncidentIssues = append(templateData.IncidentIssues, *genIssueInfo(p))
}
// Populate weekly ops graphs for template using uploaded files
for _, graph := range weeklyOpsGraphs {
templateData.WeeklyOpsGraphs = append(templateData.WeeklyOpsGraphs, graph)
// Shift change issues
for _, p := range filterIssuesByLabel("change", shiftIssuesProd) {
templateData.ChangeIssues = append(templateData.ChangeIssues, *genIssueInfo(p))
}
templateBox, err := rice.FindBox("../templates")
......@@ -197,16 +157,32 @@ func (r *Report) generateTemplate(weeklyOpsGraphs []WeeklyOpsGraph) string {
return desc.String()
}
func genIssueInfo(p *gitlab.Issue) *Issue {
issue := Issue{}
created := p.CreatedAt.Format(time.RFC822)
if p.Assignee.Username == "" {
p.Assignee.Username = "unassigned"
}
issue.CreatedAt = created
issue.Summary = p.Title
issue.URL = p.WebURL
issue.Assignee = p.Assignee.Username
issue.SeverityLabels = filterSeverityLabels(p.Labels)
return &issue
}
func newReportHelpers(config *config.Config) *ReportHelpers {
optsGitLab := GitLabHelperOptions{
APIToken: config.APITokens.GitLab,
ProjectID: config.Projects.Infrastructure.ID,
DayOffset: config.DayOffset,
APIToken: config.APITokens.GitLab,
ProjectID: config.Projects.Infrastructure.ID,
ProjectIDProd: config.Projects.Production.ID,
DayOffset: config.DayOffset,
}
optsGitLabDev := GitLabHelperOptions{
APIToken: config.APITokens.GitLabDev,
ProjectID: config.Projects.Infrastructure.ID,
DayOffset: config.DayOffset,
APIToken: config.APITokens.GitLabDev,
ProjectID: config.Projects.Infrastructure.ID,
ProjectIDProd: config.Projects.Production.ID,
DayOffset: config.DayOffset,
}
tc := ReportHelpers{
GitLab: NewGitLabHelper(optsGitLab),
......@@ -215,6 +191,43 @@ func newReportHelpers(config *config.Config) *ReportHelpers {
return &tc
}
/********* NOT USED UNTIL WE ADD PERFORMANCE GRAPHS BACK TO REPORT
// Upload graphs to the gitlab project
func (r *Report) uploadGraphs(graphs <-chan GraphData) (opsGraphs []WeeklyOpsGraph) {
log.Println("Uploading graphs to GitLab project ...")
start := time.Now()
for graph := range graphs {
// defer os.Remove(graph.Fname)
opsGraph := WeeklyOpsGraph{}
log.Println(" uploading", graph.Name)
pf := r.Helpers.GitLab.UploadFile(r.Config.Projects.ReportProject.ID, graph.Fname)
opsGraph.URL = pf.URL
opsGraph.Name = graph.Name
opsGraphs = append(opsGraphs, opsGraph)
}
secs := time.Since(start).Seconds()
log.Printf(" Completed upload in %.2f seconds\n", secs)
return opsGraphs
}
// Downloads graphs from Grafana
func (r *Report) downloadGraphs() (graphs <-chan GraphData) {
log.Println("Downloading graphs from performance ...")
start := time.Now()
var wg sync.WaitGroup
numGraphs := len(weeklyOpsGraphs(r.Config))
ch := make(chan GraphData, numGraphs)
wg.Add(numGraphs)
for _, graph := range weeklyOpsGraphs(r.Config) {
go makeRequest(graph.URL, graph.Name, ch, &wg, r.Config.APITokens.Grafana)
}
wg.Wait()
close(ch)
secs := time.Since(start).Seconds()
log.Printf(" %d graphs took %.2f seconds to download\n", numGraphs, secs)
return ch
}
func makeRequest(url string, name string, ch chan<- GraphData, wg *sync.WaitGroup, grafanaKey string) {
defer wg.Done()
start := time.Now()
......@@ -252,3 +265,4 @@ func weeklyOpsGraphs(config *config.Config) (graphs []WeeklyOpsGraph) {
}
return graphs
}
**********************/
......@@ -26,7 +26,9 @@ func getOncallIncidents(config *config.Config) []pagerduty.Incident {
var opts pagerduty.ListIncidentsOptions
opts.ServiceIDs = []string{config.PagerDuty.ServiceID}
opts.Since = nowPdDateWithOffset(-config.DayOffset)
var list pagerduty.APIListObject
list.Limit = 100
opts.APIListObject = list
if incs, err := client.ListIncidents(opts); err != nil {
panic(err)
} else {
......
......@@ -26,10 +26,11 @@ type IssueStats struct {
// Issue for oncall labeled issues
type Issue struct {
Summary string
URL string
CreatedAt string
Assignee string
Summary string
URL string
CreatedAt string
Assignee string
SeverityLabels []string
}
// Incident for PD incident
......@@ -44,7 +45,7 @@ type TemplateData struct {
WeeklyOpsGraphs []WeeklyOpsGraph
TeamMembers []TeamMember
Incidents []Incident
OnCallIssues []Issue
OpenOnCallIssues []Issue
OpenChangeIssues []Issue
OpenIncidentIssues []Issue
CriticalIssues []Issue
......
......@@ -20,15 +20,15 @@
### Open Issue Stats
* [Oncall issues](https://gitlab.com/gitlab-com/infrastructure/issues?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=oncall) : **{{.IssuesOpenAll.OnCall}}**
* [Change issues](https://gitlab.com/gitlab-com/infrastructure/issues?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=oncall) : **{{.IssuesOpenAll.Change}}**
* [Incident issues](https://gitlab.com/gitlab-com/infrastructure/issues?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=oncall) : **{{.IssuesOpenAll.Incident}}**
* [Change issues](https://gitlab.com/gitlab-com/production/issues?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=change) : **{{.IssuesOpenAll.Change}}**
* [Incident issues](https://gitlab.com/gitlab-com/production/issues?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=incident) : **{{.IssuesOpenAll.Incident}}**
* [Access Request](https://gitlab.com/gitlab-com/infrastructure/issues?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=access%20request) : **{{.IssuesOpenAll.AccessRequest}}**
#### Open Change Issues
| Created | Assignee | Summary |
| ------ | ---------| ------- |
| ------ | -------- | ------- |
{{- range .OpenChangeIssues }}
| [{{.CreatedAt}}]({{.URL}}) | {{.Assignee }} | {{.Summary}} |
......@@ -38,7 +38,7 @@
#### Open Incident Issues
| Created | Assignee | Summary |
| ------ | ---------| ------- |
| ------ | -------- | ------- |
{{- range .OpenIncidentIssues }}
| [{{.CreatedAt}}]({{.URL}}) | {{.Assignee }} | {{.Summary}} |
......@@ -47,9 +47,9 @@
#### Open Oncall Issues
| Created | Assignee | Summary |
| ------ | ---------| ------- |
| ------ | -------- | ------- |
{{- range .OnCallIssues }}
{{- range .OpenOnCallIssues }}
| [{{.CreatedAt}}]({{.URL}}) | {{.Assignee }} | {{.Summary}} |
{{- end }}
......@@ -63,7 +63,7 @@
#### Change Issues
| Created | Assignee | Summary |
| ------ | ---------| ------- |
| ------ | -------- | ------- |
{{- range .ChangeIssues }}
| [{{.CreatedAt}}]({{.URL}}) | {{.Assignee }} | {{.Summary}} |
......@@ -72,11 +72,11 @@
#### Incident Issues
| Created | Assignee | Summary |
| ------ | ---------| ------- |
| Created | Assignee | Summary | Severity |
| ------ | -------- | ------- | ------- |
{{- range .IncidentIssues }}
| [{{.CreatedAt}}]({{.URL}}) | {{.Assignee }} | {{.Summary}} |
| [{{.CreatedAt}}]({{.URL}}) | {{.Assignee }} | {{.Summary}} | {{range .SeverityLabels}}~{{.}}{{ end }} |
{{- end }}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment