From d5c243142ae20c7409d40d7e439bc4588f1745d7 Mon Sep 17 00:00:00 2001 From: Dmitry Savelev Date: Tue, 29 Oct 2024 16:45:27 +0100 Subject: [PATCH] Add Cobra & Viper --- cmd/export.go | 71 ++++++++++++ cmd/root.go | 83 ++++++++++++++ go.mod | 25 ++++- go.sum | 75 ++++++++++++- internal/db/config.go | 6 + internal/db/export.go | 192 ++++++++++++++++++++++++++++++++ internal/db/schema.go | 65 +++++++++++ internal/db/types.go | 21 ++++ internal/exporter/exporter.go | 108 ++++++++++++++++++ internal/githubclient/client.go | 150 +++++++++++++++++++++++++ internal/githubclient/config.go | 7 ++ internal/githubclient/types.go | 20 ++++ main.go | 10 ++ 13 files changed, 829 insertions(+), 4 deletions(-) create mode 100644 cmd/export.go create mode 100644 cmd/root.go create mode 100644 internal/db/config.go create mode 100644 internal/db/export.go create mode 100644 internal/db/schema.go create mode 100644 internal/db/types.go create mode 100644 internal/exporter/exporter.go create mode 100644 internal/githubclient/client.go create mode 100644 internal/githubclient/config.go create mode 100644 internal/githubclient/types.go create mode 100644 main.go diff --git a/cmd/export.go b/cmd/export.go new file mode 100644 index 0000000..fcc24f0 --- /dev/null +++ b/cmd/export.go @@ -0,0 +1,71 @@ +package cmd + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + + "github.com/neondatabase/gh-workflow-stats-action/internal/db" + "github.com/neondatabase/gh-workflow-stats-action/internal/exporter" + "github.com/neondatabase/gh-workflow-stats-action/internal/githubclient" +) + +func NewExportCommand() *cobra.Command { + dbCfg := &db.Config{} + ghCfg := &githubclient.Config{} + ghRunID := int64(0) + ghRunAttempt := int64(0) + + exportCmd := &cobra.Command{ + Use: "export", + Short: "A brief description of your command", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + Run: buildRunCommand(dbCfg, ghCfg, ghRunID, ghRunAttempt), + } + + exportCmd.Flags().StringVar(&dbCfg.DSN, "db-uri", "", "DB DSN") + exportCmd.Flags().StringVar(&dbCfg.TableName, "db-table", "", "DB Table") + exportCmd.Flags().StringVar(&ghCfg.Repository, "github-repository", "", "Github repository") + exportCmd.Flags().StringVar(&ghCfg.Token, "github-token", "", "Github token") + exportCmd.Flags().Int64Var(&ghRunID, "github-run-id", 0, "Github run ID") + exportCmd.Flags().Int64Var(&ghRunAttempt, "github-run-attempt", 0, "Github run ID") + + exportCmd.MarkFlagsRequiredTogether("github-repository", "github-run-id", "github-token", "github-run-attempt") + + exportCmd.MarkFlagRequired("db-uri") + exportCmd.MarkFlagRequired("github-repository") + + return exportCmd +} + +func init() { + rootCmd.AddCommand(NewExportCommand()) +} + +func buildRunCommand(dbCfg *db.Config, ghCfg *githubclient.Config, ghRunID int64, ghRunAttempt int64) func(*cobra.Command, []string) { + return func(cmd *cobra.Command, args []string) { + repo, err := db.NewDatabase(dbCfg) + if err != nil { + fmt.Println(err) + return + } + + ghClient, err := githubclient.NewClient(ghCfg) + if err != nil { + fmt.Println(err) + return + } + + exp := exporter.New(repo, ghClient) + if err := exp.ExportByRunAttempt(context.Background(), ghRunID, ghRunAttempt); err != nil { + fmt.Println(err) + return + } + } +} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..4926f74 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,83 @@ +package cmd + +import ( + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "gh-workflow-stats-action", + Short: "A brief description of your application", + Long: `A longer description that spans multiple lines and likely contains +examples and usage of using your application. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + return initializeConfig(cmd) + }, + // Uncomment the following line if your bare application + // has an action associated with it: + Run: func(cmd *cobra.Command, args []string) { + fmt.Println(args) + }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + // Here you will define your flags and configuration settings. + // Cobra supports persistent flags, which, if defined here, + // will be global for your application. + + // rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.gh-workflow-stats-action.yaml)") + + // Cobra also supports local flags, which will only run + // when this action is called directly. + rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} + +func initializeConfig(cmd *cobra.Command) error { + v := viper.New() + // Environment variables can't have dashes in them, so bind them to their equivalent + // keys with underscores, e.g. --favorite-color to STING_FAVORITE_COLOR + v.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + + // Bind to environment variables + // Works great for simple config names, but needs help for names + // like --favorite-color which we fix in the bindFlags function + v.AutomaticEnv() + + // Bind the current command's flags to viper + bindFlags(cmd, v) + + return nil +} + +// Bind each cobra flag to its associated viper configuration (config file and environment variable) +func bindFlags(cmd *cobra.Command, v *viper.Viper) { + cmd.Flags().VisitAll(func(f *pflag.Flag) { + // Determine the naming convention of the flags when represented in the config file + configName := f.Name + + // Apply the viper config value to the flag when the flag is not set and viper has a value + if !f.Changed && v.IsSet(configName) { + val := v.Get(configName) + cmd.Flags().Set(f.Name, fmt.Sprintf("%v", val)) + } + }) +} diff --git a/go.mod b/go.mod index 9f2e582..ea25c32 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,36 @@ module github.com/neondatabase/gh-workflow-stats-action go 1.23.1 require ( + github.com/gofri/go-github-ratelimit v1.1.0 github.com/google/go-github/v65 v65.0.0 github.com/hashicorp/go-retryablehttp v0.7.7 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.10.9 - golang.org/x/oauth2 v0.23.0 + github.com/spf13/cobra v1.8.1 + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.19.0 ) require ( - github.com/gofri/go-github-ratelimit v1.1.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.14.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 9854813..394175b 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,16 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/gofri/go-github-ratelimit v1.1.0 h1:ijQ2bcv5pjZXNil5FiwglCg8wc9s8EgjTmNkqjw8nuk= @@ -19,18 +28,80 @@ github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB1 github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= -golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/db/config.go b/internal/db/config.go new file mode 100644 index 0000000..70c71ab --- /dev/null +++ b/internal/db/config.go @@ -0,0 +1,6 @@ +package db + +type Config struct { + DSN string + TableName string +} diff --git a/internal/db/export.go b/internal/db/export.go new file mode 100644 index 0000000..7be2138 --- /dev/null +++ b/internal/db/export.go @@ -0,0 +1,192 @@ +package db + +import ( + "context" + "fmt" + "strings" + + "github.com/google/go-github/v65/github" + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" + + "github.com/neondatabase/gh-workflow-stats-action/internal/githubclient" + "github.com/neondatabase/gh-workflow-stats-action/pkg/data" +) + +var _ Repository = &Postgres{} + +type Postgres struct { + cfg *Config + conn *sqlx.DB +} + +func NewDatabase(cfg *Config) (*Postgres, error) { + conn, err := sqlx.Connect("postgres", cfg.DSN) + if err != nil { + return nil, err + } + + return &Postgres{ + cfg: cfg, + conn: conn, + }, nil +} + +func (p *Postgres) Init() error { + _, err := p.conn.Exec(fmt.Sprintf(schemeWorkflowRunsStats, p.cfg.TableName)) + if err != nil { + return err + } + + _, err = p.conn.Exec(fmt.Sprintf(schemeWorkflowRunAttempts, p.cfg.TableName+"_attempts")) + if err != nil { + return err + } + + _, err = p.conn.Exec(fmt.Sprintf(schemeWorkflowJobs, p.cfg.TableName+"_jobs")) + if err != nil { + return err + } + + _, err = p.conn.Exec(fmt.Sprintf(schemeWorkflowJobsSteps, p.cfg.TableName+"_steps")) + if err != nil { + return err + } + return nil +} + +func (p *Postgres) SaveWorkflowRun(record *data.WorkflowRunRec) error { + query := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", p.cfg.TableName, + "workflowid, name, status, conclusion, runid, runattempt, startedAt, updatedAt, repoName, event", + ":workflowid, :name, :status, :conclusion, :runid, :runattempt, :startedat, :updatedat, :reponame, :event", + ) + + _, err := p.conn.NamedExec(query, *record) + + if err != nil { + return err + } + return nil +} + +func (p *Postgres) SaveWorkflowRunAttempt(workflowRun *github.WorkflowRun) error { + query := fmt.Sprintf("INSERT INTO %s_attempts (%s) VALUES (%s)", p.cfg.TableName, + "workflowid, name, status, conclusion, runid, runattempt, startedAt, updatedAt, repoName, event", + ":workflowid, :name, :status, :conclusion, :runid, :runattempt, :startedat, :updatedat, :reponame, :event", + ) + + _, err := p.conn.NamedExec(query, data.GhWorkflowRunRec(workflowRun)) + return err +} + +func (p *Postgres) WithTransaction(ctx context.Context, fn func(tx *sqlx.Tx) error) error { + tx, err := p.conn.BeginTxx(ctx, nil) + if err != nil { + return err + } + + if err = fn(tx); err != nil { + if err := tx.Rollback(); err != nil { + fmt.Println(err) + } + + return err + } + + return tx.Commit() +} + +func (p *Postgres) InsertJob(tx *sqlx.Tx, workflowJob *github.WorkflowJob) error { + query := fmt.Sprintf("INSERT INTO %s_jobs (%s) VALUES (%s)", p.cfg.TableName, + "jobid, runid, nodeid, headbranch, headsha, status, conclusion, createdat, startedat, completedat, name, runnername, runnergroupname, runattempt, workflowname", + ":jobid, :runid, :nodeid, :headbranch, :headsha, :status, :conclusion, :createdat, :startedat, :completedat, :name, :runnername, :runnergroupname, :runattempt, :workflowname", + ) + stmt, err := tx.PrepareNamed(query) + if err != nil { + return err + } + + _, err = stmt.Exec(data.GhWorkflowJobRec(workflowJob)) + if err != nil { + return err + } + + return nil +} + +func (p *Postgres) InsertSteps(tx *sqlx.Tx, job *github.WorkflowJob) error { + query := fmt.Sprintf("INSERT INTO %s_steps (%s) VALUES (%s)", p.cfg.TableName, + "jobid, runid, runattempt, name, status, conclusion, number, startedat, completedat", + ":jobid, :runid, :runattempt, :name, :status, :conclusion, :number, :startedat, :completedat", + ) + stmt, err := tx.PrepareNamed(query) + if err != nil { + return err + } + + for _, step := range job.Steps { + _, err = stmt.Exec(data.GhWorkflowJobStepRec(job, step)) + if err != nil { + return err + } + } + + return nil +} + +func (p *Postgres) QueryWorkflowRunAttempts(runId int64) map[int64]struct{} { + result := make(map[int64]struct{}) + + query := fmt.Sprintf("SELECT runAttempt from %s_attempts WHERE runId=$1", p.cfg.TableName) + rows, err := p.conn.Query(query, runId) + if err != nil { + return result + } + var attempt int64 + for rows.Next() { + err = rows.Scan(&attempt) + if err != nil { + fmt.Println(err) + } else { + result[attempt] = struct{}{} + } + } + return result +} + +func (p *Postgres) QueryWorkflowRunsNotInDb(workflowRuns []githubclient.WorkflowRunAttemptKey) []githubclient.WorkflowRunAttemptKey { + result := make([]githubclient.WorkflowRunAttemptKey, 0) + + if len(workflowRuns) == 0 { + return result + } + // TODO: I have to find out how to use https://jmoiron.github.io/sqlx/#namedParams with sqlx.In() + // For now just generate query with strings.Builder + var valuesStr strings.Builder + for i, v := range workflowRuns { + if i > 0 { + valuesStr.WriteString(", ") + } + valuesStr.WriteString(fmt.Sprintf("(%d :: bigint, %d :: bigint)", v.RunId, v.RunAttempt)) + } + queryStr := fmt.Sprintf("SELECT runid, runattempt FROM (VALUES %s) as q (runid, runattempt) LEFT JOIN %s_attempts db "+ + "USING (runid, runattempt) WHERE db.runid is null", + valuesStr.String(), + p.cfg.TableName, + ) + rows, err := p.conn.Queryx(queryStr) + if err != nil { + fmt.Printf("Failed to Query: %s\n", err) + return result + } + var rec githubclient.WorkflowRunAttemptKey + for rows.Next() { + err = rows.StructScan(&rec) + if err != nil { + fmt.Println(err) + } else { + result = append(result, rec) + } + } + return result +} diff --git a/internal/db/schema.go b/internal/db/schema.go new file mode 100644 index 0000000..50cb8d5 --- /dev/null +++ b/internal/db/schema.go @@ -0,0 +1,65 @@ +package db + +var ( + schemeWorkflowRunsStats = ` + CREATE TABLE IF NOT EXISTS %s ( + workflowid BIGINT, + name TEXT, + status TEXT, + conclusion TEXT, + runid BIGINT, + runattempt INT, + startedat TIMESTAMP, + updatedat TIMESTAMP, + reponame TEXT, + event TEXT, + PRIMARY KEY(workflowid, runid, runattempt) + ) + ` + schemeWorkflowRunAttempts = ` + CREATE TABLE IF NOT EXISTS %s ( + workflowid BIGINT, + name TEXT, + status TEXT, + conclusion TEXT, + runid BIGINT, + runattempt INT, + startedat TIMESTAMP, + updatedat TIMESTAMP, + reponame TEXT, + event TEXT, + PRIMARY KEY(workflowid, runid, runattempt) + ) + ` + schemeWorkflowJobs = ` + CREATE TABLE IF NOT EXISTS %s ( + JobId BIGINT, + RunID BIGINT, + NodeID TEXT, + HeadBranch TEXT, + HeadSHA TEXT, + Status TEXT, + Conclusion TEXT, + CreatedAt TIMESTAMP, + StartedAt TIMESTAMP, + CompletedAt TIMESTAMP, + Name TEXT, + RunnerName TEXT, + RunnerGroupName TEXT, + RunAttempt BIGINT, + WorkflowName TEXT + )` + schemeWorkflowJobsSteps = ` + CREATE TABLE IF NOT EXISTS %s ( + JobId BIGINT, + RunId BIGINT, + RunAttempt BIGINT, + Name TEXT, + Status TEXT, + Conclusion TEXT, + Number BIGINT, + StartedAt TIMESTAMP, + CompletedAt TIMESTAMP + ) + ` +) diff --git a/internal/db/types.go b/internal/db/types.go new file mode 100644 index 0000000..69e77a8 --- /dev/null +++ b/internal/db/types.go @@ -0,0 +1,21 @@ +package db + +import ( + "context" + + "github.com/google/go-github/v65/github" + "github.com/jmoiron/sqlx" + + "github.com/neondatabase/gh-workflow-stats-action/internal/githubclient" + "github.com/neondatabase/gh-workflow-stats-action/pkg/data" +) + +type Repository interface { + SaveWorkflowRun(record *data.WorkflowRunRec) error + SaveWorkflowRunAttempt(workflowRun *github.WorkflowRun) error + InsertJob(tx *sqlx.Tx, workflowJob *github.WorkflowJob) error + InsertSteps(tx *sqlx.Tx, workflowJob *github.WorkflowJob) error + WithTransaction(ctx context.Context, fn func(tx *sqlx.Tx) error) error + QueryWorkflowRunAttempts(runId int64) map[int64]struct{} + QueryWorkflowRunsNotInDb(workflowRuns []githubclient.WorkflowRunAttemptKey) []githubclient.WorkflowRunAttemptKey +} diff --git a/internal/exporter/exporter.go b/internal/exporter/exporter.go new file mode 100644 index 0000000..d685a8a --- /dev/null +++ b/internal/exporter/exporter.go @@ -0,0 +1,108 @@ +package exporter + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/google/go-github/v65/github" + "github.com/jmoiron/sqlx" + + "github.com/neondatabase/gh-workflow-stats-action/internal/db" + "github.com/neondatabase/gh-workflow-stats-action/internal/githubclient" +) + +type Export struct { + repo db.Repository + ghClient githubclient.Client +} + +func New(repo db.Repository, ghClient githubclient.Client) *Export { + return &Export{ + repo: repo, + ghClient: ghClient, + } +} + +func (e *Export) ExportByInterval(ctx context.Context, startDate, endDate time.Time) error { + durations := []time.Duration{ + 6 * time.Hour, // 18:00 - 24:00 + 3 * time.Hour, // 15:00 - 18:00 + 1 * time.Hour, // 14:00 - 15:00 + 1 * time.Hour, // 13:00 - 14:00 + 1 * time.Hour, // 12:00 - 13:00 + 2 * time.Hour, // 10:00 - 12:00 + 4 * time.Hour, // 06:00 - 10:00 + 6 * time.Hour, // 00:00 - 06:00 + } + curDurIdx := 0 + for date := endDate.Add(-durations[curDurIdx]); date.Compare(startDate) >= 0; date = date.Add(-durations[curDurIdx]) { + runs, rate, _ := e.ghClient.ListWorkflowRuns(ctx, date, date.Add(durations[curDurIdx])) + fmt.Println("\n", date, len(runs)) + if len(runs) >= 1000 { + fmt.Printf("\n\n+++\n+ PAGINATION LIMIT: %v\n+++\n", date) + } + fetchedRunsKeys := make([]githubclient.WorkflowRunAttemptKey, len(runs)) + for key := range runs { + fetchedRunsKeys = append(fetchedRunsKeys, key) + } + notInDb := e.repo.QueryWorkflowRunsNotInDb(fetchedRunsKeys) + fmt.Printf("Time range: %v - %v, fetched: %d, notInDb: %d.\n", + date, date.Add(durations[curDurIdx]), + len(runs), len(notInDb), + ) + if rate.Remaining < 30 { + fmt.Printf("Close to rate limit, remaining: %d", rate.Remaining) + fmt.Printf("Sleep till %v (%v seconds)\n", rate.Reset, time.Until(rate.Reset.Time)) + time.Sleep(time.Until(rate.Reset.Time) + 10*time.Second) + } else { + fmt.Printf("Rate: %+v\n", rate) + } + for _, key := range notInDb { + fmt.Printf("Saving runId %d Attempt %d. ", key.RunId, key.RunAttempt) + var attemptRun *github.WorkflowRun + var ok bool + if attemptRun, ok = runs[githubclient.WorkflowRunAttemptKey{RunId: key.RunId, RunAttempt: key.RunAttempt}]; ok { + fmt.Printf("Got it from ListWorkflowRuns results. ") + } else { + fmt.Printf("Fetching it from GH API. ") + attemptRun, _ = e.ghClient.GetWorkflowAttempt(ctx, key.RunId, key.RunAttempt) + } + e.repo.SaveWorkflowRunAttempt(attemptRun) + e.ExportByRunAttempt(ctx, key.RunId, key.RunAttempt) + } + curDurIdx = (curDurIdx + 1) % len(durations) + } + + return nil +} + +func (e *Export) ExportByRunAttempt(ctx context.Context, runID int64, runAttempt int64) error { + jobsInfo, rate, err := e.ghClient.GetWorkflowAttemptJobs(ctx, runID, runAttempt) + if err != nil { + log.Fatal(err) + } + if rate.Remaining < 20 { + fmt.Printf("Close to rate limit, remaining: %d", rate.Remaining) + fmt.Printf("Sleep till %v (%v seconds)\n", rate.Reset, time.Until(rate.Reset.Time)) + time.Sleep(time.Until(rate.Reset.Time)) + } + err = e.repo.WithTransaction(ctx, func(tx *sqlx.Tx) error { + for _, jobInfo := range jobsInfo { + err = e.repo.InsertJob(tx, jobInfo) + if err != nil { + return err + } + + err = e.repo.InsertSteps(tx, jobInfo) + if err != nil { + return err + } + } + + return nil + }) + + return err +} diff --git a/internal/githubclient/client.go b/internal/githubclient/client.go new file mode 100644 index 0000000..90ec2c1 --- /dev/null +++ b/internal/githubclient/client.go @@ -0,0 +1,150 @@ +package githubclient + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/gofri/go-github-ratelimit/github_ratelimit" + "github.com/google/go-github/v65/github" + "github.com/hashicorp/go-retryablehttp" + + "github.com/neondatabase/gh-workflow-stats-action/pkg/data" +) + +type ClientImpl struct { + cfg *Config + owner string + repo string + + client *github.Client +} + +func NewClient(cfg *Config) (*ClientImpl, error) { + repoDetails := strings.Split(cfg.Repository, "/") + if len(repoDetails) != 2 { + return nil, fmt.Errorf("invalid config: GITHUB_REPOSITORY") + } + + retryClient := retryablehttp.NewClient() + retryClient.RetryMax = 5 + if cfg.MaxRetry != 0 { + retryClient.RetryMax = cfg.MaxRetry + } + + rl, err := github_ratelimit.NewRateLimitWaiterClient(retryClient.StandardClient().Transport) + if err != nil { + return nil, fmt.Errorf("failed to create rate limit waiter: %v", err) + } + + return &ClientImpl{ + cfg: cfg, + owner: repoDetails[0], + repo: repoDetails[1], + client: github.NewClient(rl).WithAuthToken(cfg.Token), + }, nil +} + +func (c *ClientImpl) GetWorkflowStat(ctx context.Context, runID int64) (*data.WorkflowRunRec, error) { + fmt.Printf("Getting data for %s/%s, runID %d\n", c.owner, c.repo, runID) + workflowRunData, _, err := c.client.Actions.GetWorkflowRunByID(ctx, c.owner, c.repo, runID) + if err != nil { + return nil, err + } + + if workflowRunData == nil { + fmt.Printf("Got nil\n") + return &data.WorkflowRunRec{RepoName: c.repo}, nil + } + + return data.GhWorkflowRunRec(workflowRunData), nil +} + +func (c *ClientImpl) GetWorkflowAttempt(ctx context.Context, runID int64, attempt int64) (*github.WorkflowRun, error) { + workflowRunData, _, err := c.client.Actions.GetWorkflowRunAttempt( + ctx, + c.owner, c.repo, + runID, + int(attempt), + nil, + ) + if err != nil { + return nil, err + } + return workflowRunData, nil +} + +func (c *ClientImpl) GetWorkflowAttemptJobs(ctx context.Context, runID int64, attempt int64) ([]*github.WorkflowJob, github.Rate, error) { + var result []*github.WorkflowJob + finalRate := github.Rate{} + + opts := &github.ListOptions{PerPage: 100} + for { + jobsData, resp, err := c.client.Actions.ListWorkflowJobsAttempt( + ctx, + c.owner, c.repo, + runID, + attempt, + opts, + ) + if resp != nil { + finalRate = resp.Rate + } + if err != nil { + return nil, finalRate, err + } + result = append(result, jobsData.Jobs...) + if resp.NextPage == 0 { + break + } + + opts.Page = resp.NextPage + } + return result, finalRate, nil +} + +type WorkflowRunAttemptKey struct { + RunId int64 + RunAttempt int64 +} + +func (c *ClientImpl) ListWorkflowRuns( + ctx context.Context, + start time.Time, end time.Time, +) (map[WorkflowRunAttemptKey]*github.WorkflowRun, github.Rate, error) { + result := make(map[WorkflowRunAttemptKey]*github.WorkflowRun) + finalRate := github.Rate{} + + opts := &github.ListOptions{PerPage: 100} + for { + workflowRuns, resp, err := c.client.Actions.ListRepositoryWorkflowRuns( + ctx, + c.owner, c.repo, + &github.ListWorkflowRunsOptions{ + Created: fmt.Sprintf("%s..%s", start.Format(time.RFC3339), end.Format(time.RFC3339)), + Status: "completed", + ListOptions: *opts, + }, + ) + if resp != nil { + finalRate = resp.Rate + } + if err != nil { + return nil, finalRate, err + } + for _, rec := range workflowRuns.WorkflowRuns { + key := WorkflowRunAttemptKey{RunId: rec.GetID(), RunAttempt: int64(rec.GetRunAttempt())} + if v, ok := result[key]; ok { + fmt.Printf("Strange, record is already stored for %v (%+v), updating with %+v\n", key, v, rec) + } + result[key] = rec + } + if resp.NextPage == 0 { + finalRate = resp.Rate + break + } + opts.Page = resp.NextPage + } + return result, finalRate, nil +} diff --git a/internal/githubclient/config.go b/internal/githubclient/config.go new file mode 100644 index 0000000..a1a881c --- /dev/null +++ b/internal/githubclient/config.go @@ -0,0 +1,7 @@ +package githubclient + +type Config struct { + Repository string + Token string + MaxRetry int +} diff --git a/internal/githubclient/types.go b/internal/githubclient/types.go new file mode 100644 index 0000000..6e37552 --- /dev/null +++ b/internal/githubclient/types.go @@ -0,0 +1,20 @@ +package githubclient + +import ( + "context" + "time" + + "github.com/google/go-github/v65/github" + + "github.com/neondatabase/gh-workflow-stats-action/pkg/data" +) + +type Client interface { + GetWorkflowStat(ctx context.Context, runID int64) (*data.WorkflowRunRec, error) + GetWorkflowAttempt(ctx context.Context, runID int64, attempt int64) (*github.WorkflowRun, error) + GetWorkflowAttemptJobs(ctx context.Context, runID int64, attempt int64) ([]*github.WorkflowJob, github.Rate, error) + ListWorkflowRuns( + ctx context.Context, + start time.Time, end time.Time, + ) (map[WorkflowRunAttemptKey]*github.WorkflowRun, github.Rate, error) +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..59620f3 --- /dev/null +++ b/main.go @@ -0,0 +1,10 @@ +/* +Copyright © 2024 NAME HERE +*/ +package main + +import "github.com/neondatabase/gh-workflow-stats-action/cmd" + +func main() { + cmd.Execute() +}