From ee7fee47e07caaf8a1992b955b7a3051d8f983b4 Mon Sep 17 00:00:00 2001 From: vijeyash Date: Wed, 19 Jul 2023 11:31:20 +0530 Subject: [PATCH] issuse #94 obj to table --- agent/container/cfg.yaml | 2 +- .../pkg/handler/docker_event_dockerhub.go | 26 +++- .../pkg/handler/docker_event_github.go | 29 +++- client/pkg/clickhouse/db_client.go | 30 ++-- client/pkg/clickhouse/statements.go | 12 ++ client/pkg/clients/container_client.go | 81 ++++------- model/dockerhub.go | 37 +++++ model/github_docker.go | 133 ++++++++++++++++++ 8 files changed, 269 insertions(+), 81 deletions(-) create mode 100644 model/dockerhub.go create mode 100644 model/github_docker.go diff --git a/agent/container/cfg.yaml b/agent/container/cfg.yaml index 4e55092c..de91ea73 100644 --- a/agent/container/cfg.yaml +++ b/agent/container/cfg.yaml @@ -3,4 +3,4 @@ generate: chi-server: true models: true embedded-spec: true -output: /Users/vijeshdeepan/Desktop/kubviz/agent/container/api/agent.gen.go +output: agent/container/api/agent.gen.go diff --git a/agent/container/pkg/handler/docker_event_dockerhub.go b/agent/container/pkg/handler/docker_event_dockerhub.go index a213f3de..0d6d0e08 100644 --- a/agent/container/pkg/handler/docker_event_dockerhub.go +++ b/agent/container/pkg/handler/docker_event_dockerhub.go @@ -1,20 +1,32 @@ package handler import ( + "errors" "io" "log" "net/http" ) +// parse errors +var ( + ErrReadingBody = errors.New("error reading the request body") + ErrPublishToNats = errors.New("error while publishing to nats") +) + func (ah *APIHandler) PostEventDockerHub(w http.ResponseWriter, r *http.Request) { - event, err := io.ReadAll(r.Body) - if err != nil { - log.Printf("Event body read failed: %v", err) + defer func() { + _, _ = io.Copy(io.Discard, r.Body) + _ = r.Body.Close() + }() + payload, err := io.ReadAll(r.Body) + if err != nil || len(payload) == 0 { + log.Printf("%v: %v", ErrReadingBody, err) + return } - - log.Printf("Received event from docker artifactory: %v", string(event)) - err = ah.conn.Publish(event, "docker registry") + log.Printf("Received event from docker artifactory: %v", string(payload)) + err = ah.conn.Publish(payload, "Dockerhub_Registry") if err != nil { - log.Printf("Publish failed for event: %v, reason: %v", string(event), err) + log.Printf("%v: %v", ErrPublishToNats, err) + return } } diff --git a/agent/container/pkg/handler/docker_event_github.go b/agent/container/pkg/handler/docker_event_github.go index ea768119..1dda3843 100644 --- a/agent/container/pkg/handler/docker_event_github.go +++ b/agent/container/pkg/handler/docker_event_github.go @@ -1,20 +1,37 @@ package handler import ( + "errors" "io" "log" "net/http" ) +var ( + ErrMissingGithubEventHeader = errors.New("missing X-GitHub-Event Header") +) + func (ah *APIHandler) PostEventDockerGithub(w http.ResponseWriter, r *http.Request) { - event, err := io.ReadAll(r.Body) - if err != nil { - log.Printf("Event body read failed: %v", err) + defer func() { + _, _ = io.Copy(io.Discard, r.Body) + _ = r.Body.Close() + }() + event := r.Header.Get("X-GitHub-Event") + if event == "" { + log.Printf("%v", ErrMissingGithubEventHeader) + return + } + + payload, err := io.ReadAll(r.Body) + if err != nil || len(payload) == 0 { + log.Printf("%v: %v", ErrReadingBody, err) + return } - log.Printf("Received docker event from github artifactory: %v", string(event)) - err = ah.conn.Publish(event, "Github_Registory") + log.Printf("Received docker event from github artifactory: %v", string(payload)) + err = ah.conn.Publish(payload, "Github_Registry") if err != nil { - log.Printf("Publish failed for event: %v, reason: %v", string(event), err) + log.Printf("%v: %v", ErrPublishToNats, err) + return } } diff --git a/client/pkg/clickhouse/db_client.go b/client/pkg/clickhouse/db_client.go index 0b6dc3a8..213facd3 100644 --- a/client/pkg/clickhouse/db_client.go +++ b/client/pkg/clickhouse/db_client.go @@ -33,7 +33,7 @@ type DBInterface interface { RetriveOutdatedEvent() ([]model.CheckResultfinal, error) RetriveKubepugEvent() ([]model.Result, error) RetrieveKubvizEvent() ([]model.DbEvent, error) - InsertContainerEventDockerHub(string) + InsertContainerEventDockerHub(model.DockerHubBuild) InsertContainerEventGithub(string) Close() } @@ -63,7 +63,7 @@ func NewDBClient(conf *config.Config) (DBInterface, error) { } return nil, err } - tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, gitTable, kubescoreTable} + tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, gitTable, kubescoreTable, dockerHubBuildTable} for _, table := range tables { if err = splconn.Exec(context.Background(), string(table)); err != nil { return nil, err @@ -379,21 +379,27 @@ func (c *DBClient) RetrieveKubvizEvent() ([]model.DbEvent, error) { return events, nil } -func (c *DBClient) InsertContainerEventDockerHub(event string) { - ctx := context.Background() - batch, err := c.splconn.PrepareBatch(ctx, "INSERT INTO container_dockerhub") - if err != nil { - log.Fatal(err) - } - - if err = batch.Append(event); err != nil { +func (c *DBClient) InsertContainerEventDockerHub(build model.DockerHubBuild) { + var ( + tx, _ = c.conn.Begin() + stmt, _ = tx.Prepare(string(InsertDockerHubBuild)) + ) + defer stmt.Close() + if _, err := stmt.Exec( + build.PushedBy, + build.ImageTag, + build.RepositoryName, + build.DateCreated, + build.Owner, + build.Event, + ); err != nil { log.Fatal(err) } - - if err = batch.Send(); err != nil { + if err := tx.Commit(); err != nil { log.Fatal(err) } } + func (c *DBClient) InsertContainerEventGithub(event string) { ctx := context.Background() batch, err := c.splconn.PrepareBatch(ctx, "INSERT INTO container_github") diff --git a/client/pkg/clickhouse/statements.go b/client/pkg/clickhouse/statements.go index 3e92b1ea..37783859 100644 --- a/client/pkg/clickhouse/statements.go +++ b/client/pkg/clickhouse/statements.go @@ -79,6 +79,18 @@ const kubescoreTable DBStatement = ` recommendations String ) engine=File(TabSeparated) ` + +const dockerHubBuildTable DBStatement = ` + CREATE TABLE IF NOT EXISTS dockerhubbuild ( + PushedBy String, + ImageTag String, + RepositoryName String, + DateCreated String, + Owner String, + Event String + ) engine=File(TabSeparated) + ` +const InsertDockerHubBuild DBStatement = "INSERT INTO dockerhubbuild (PushedBy, ImageTag, RepositoryName, DateCreated, Owner, Event) VALUES (?, ?, ?, ?, ?, ?)" const InsertRakees DBStatement = "INSERT INTO rakkess (ClusterName, Name, Create, Delete, List, Update) VALUES (?, ?, ?, ?, ?, ?)" const InsertKetall DBStatement = "INSERT INTO getall_resources (ClusterName, Namespace, Kind, Resource, Age) VALUES (?, ?, ?, ?, ?)" const InsertOutdated DBStatement = "INSERT INTO outdated_images (ClusterName, Namespace, Pod, CurrentImage, CurrentTag, LatestVersion, VersionsBehind) VALUES (?, ?, ?, ?, ?, ?, ?)" diff --git a/client/pkg/clients/container_client.go b/client/pkg/clients/container_client.go index 93a3a849..59d511ea 100644 --- a/client/pkg/clients/container_client.go +++ b/client/pkg/clients/container_client.go @@ -2,12 +2,19 @@ package clients import ( "encoding/json" + "errors" "log" + "time" "github.com/intelops/kubviz/client/pkg/clickhouse" + "github.com/intelops/kubviz/model" "github.com/nats-io/nats.go" ) +var ( + ErrUnmarshalBuildPayload = errors.New("error while unmarshal the dockerhub build payload") +) + type Container string // constant variables to use with nats stream and @@ -18,66 +25,30 @@ const ( containerConsumer Container = "container-event-consumer" ) -// func (n *NATSContext) SubscribeContainerNats(conn clickhouse.DBInterface) { -// n.stream.Subscribe(string(containerSubject), func(msg *nats.Msg) { -// type events struct { -// Events []json.RawMessage `json:"events"` -// } - -// eventDocker := &events{} -// err := json.Unmarshal(msg.Data, &eventDocker) -// if err == nil { -// log.Println(eventDocker) -// msg.Ack() -// repoName := msg.Header.Get("REPO_NAME") -// type newEvent struct { -// RepoName string `json:"repoName"` -// Event json.RawMessage `json:"event"` -// } - -// for _, event := range eventDocker.Events { -// event := &newEvent{ -// RepoName: repoName, -// Event: event, -// } - -// eventsJSON, err := json.Marshal(event) -// if err != nil { -// log.Printf("Failed to marshall with repo name going ahead with only event, %v", err) -// eventsJSON = msg.Data -// } -// conn.InsertContainerEvent(string(eventsJSON)) -// } -// } else { -// log.Printf("Failed to unmarshal event, %v", err) -// conn.InsertContainerEvent(string(msg.Data)) -// } - -// log.Println("Inserted metrics:", string(msg.Data)) -// }, nats.Durable(string(containerConsumer)), nats.ManualAck()) -// } func (n *NATSContext) SubscribeContainerNats(conn clickhouse.DBInterface) { n.stream.Subscribe(string(containerSubject), func(msg *nats.Msg) { - type pubData struct { - Metrics json.RawMessage `json:"event"` - Repo string `json:"repoName"` - } msg.Ack() repoName := msg.Header.Get("REPO_NAME") - metrics := &pubData{ - Metrics: json.RawMessage(msg.Data), - Repo: repoName, - } - data, err := json.Marshal(metrics) - if err != nil { - log.Fatal(err) - } - if repoName == "Github_Registory" { - conn.InsertContainerEventGithub(string(data)) - log.Println("Inserted Github Container metrics:", string(msg.Data)) - } else if repoName == "Dockerhub_Registry" { - conn.InsertContainerEventDockerHub(string(data)) + if repoName == "Dockerhub_Registry" { + var pl model.BuildPayload + err := json.Unmarshal(msg.Data, &pl) + if err != nil { + log.Printf("%v", ErrUnmarshalBuildPayload) + return + } + var hub model.DockerHubBuild + t := time.Unix(int64(pl.Repository.DateCreated), 0) + hub.DateCreated = t.Format("2006-01-02 15:04:05") + hub.PushedBy = pl.PushData.Pusher + hub.ImageTag = pl.PushData.Tag + hub.RepositoryName = pl.Repository.Name + hub.Owner = pl.Repository.Owner + hub.Event = string(msg.Data) + conn.InsertContainerEventDockerHub(hub) log.Println("Inserted DockerHub Container metrics:", string(msg.Data)) + } else if repoName == "Github_Registry" { + conn.InsertContainerEventGithub(string(msg.Data)) + log.Println("Inserted Github Container metrics:", string(msg.Data)) } }, nats.Durable(string(containerConsumer)), nats.ManualAck()) } diff --git a/model/dockerhub.go b/model/dockerhub.go new file mode 100644 index 00000000..e407b135 --- /dev/null +++ b/model/dockerhub.go @@ -0,0 +1,37 @@ +package model + +type BuildPayload struct { + CallbackURL string `json:"callback_url"` + PushData struct { + Images []string `json:"images"` + PushedAt float32 `json:"pushed_at"` + Pusher string `json:"pusher"` + Tag string `json:"tag"` + } `json:"push_data"` + Repository struct { + CommentCount int `json:"comment_count"` + DateCreated float32 `json:"date_created"` + Description string `json:"description"` + Dockerfile string `json:"dockerfile"` + FullDescription string `json:"full_description"` + IsOfficial bool `json:"is_official"` + IsPrivate bool `json:"is_private"` + IsTrusted bool `json:"is_trusted"` + Name string `json:"name"` + Namespace string `json:"namespace"` + Owner string `json:"owner"` + RepoName string `json:"repo_name"` + RepoURL string `json:"repo_url"` + StarCount int `json:"star_count"` + Status string `json:"status"` + } `json:"repository"` +} + +type DockerHubBuild struct { + PushedBy string + ImageTag string + RepositoryName string + DateCreated string + Owner string + Event string +} diff --git a/model/github_docker.go b/model/github_docker.go new file mode 100644 index 00000000..e3ee2513 --- /dev/null +++ b/model/github_docker.go @@ -0,0 +1,133 @@ +package model + +import "time" + +type PingPayload struct { + HookID int `json:"hook_id"` + Hook struct { + Type string `json:"type"` + ID int64 `json:"id"` + NodeID string `json:"node_id"` + Name string `json:"name"` + Active bool `json:"active"` + Events []string `json:"events"` + AppID int `json:"app_id"` + Config struct { + ContentType string `json:"content_type"` + InsecureSSL string `json:"insecure_ssl"` + Secret string `json:"secret"` + URL string `json:"url"` + } `json:"config"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + } `json:"hook"` + Repository struct { + ID int64 `json:"id"` + NodeID string `json:"node_id"` + Name string `json:"name"` + FullName string `json:"full_name"` + Owner struct { + Login string `json:"login"` + ID int64 `json:"id"` + NodeID string `json:"node_id"` + AvatarURL string `json:"avatar_url"` + GravatarID string `json:"gravatar_id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + FollowersURL string `json:"followers_url"` + FollowingURL string `json:"following_url"` + GistsURL string `json:"gists_url"` + StarredURL string `json:"starred_url"` + SubscriptionsURL string `json:"subscriptions_url"` + OrganizationsURL string `json:"organizations_url"` + ReposURL string `json:"repos_url"` + EventsURL string `json:"events_url"` + ReceivedEventsURL string `json:"received_events_url"` + Type string `json:"type"` + SiteAdmin bool `json:"site_admin"` + } `json:"owner"` + Private bool `json:"private"` + HTMLURL string `json:"html_url"` + Description string `json:"description"` + Fork bool `json:"fork"` + URL string `json:"url"` + ForksURL string `json:"forks_url"` + KeysURL string `json:"keys_url"` + CollaboratorsURL string `json:"collaborators_url"` + TeamsURL string `json:"teams_url"` + HooksURL string `json:"hooks_url"` + IssueEventsURL string `json:"issue_events_url"` + EventsURL string `json:"events_url"` + AssigneesURL string `json:"assignees_url"` + BranchesURL string `json:"branches_url"` + TagsURL string `json:"tags_url"` + BlobsURL string `json:"blobs_url"` + GitTagsURL string `json:"git_tags_url"` + GitRefsURL string `json:"git_refs_url"` + TreesURL string `json:"trees_url"` + StatusesURL string `json:"statuses_url"` + LanguagesURL string `json:"languages_url"` + StargazersURL string `json:"stargazers_url"` + ContributorsURL string `json:"contributors_url"` + SubscribersURL string `json:"subscribers_url"` + SubscriptionURL string `json:"subscription_url"` + CommitsURL string `json:"commits_url"` + GitCommitsURL string `json:"git_commits_url"` + CommentsURL string `json:"comments_url"` + IssueCommentURL string `json:"issue_comment_url"` + ContentsURL string `json:"contents_url"` + CompareURL string `json:"compare_url"` + MergesURL string `json:"merges_url"` + ArchiveURL string `json:"archive_url"` + DownloadsURL string `json:"downloads_url"` + IssuesURL string `json:"issues_url"` + PullsURL string `json:"pulls_url"` + MilestonesURL string `json:"milestones_url"` + NotificationsURL string `json:"notifications_url"` + LabelsURL string `json:"labels_url"` + ReleasesURL string `json:"releases_url"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + PushedAt time.Time `json:"pushed_at"` + GitURL string `json:"git_url"` + SSHURL string `json:"ssh_url"` + CloneURL string `json:"clone_url"` + SvnURL string `json:"svn_url"` + Homepage *string `json:"homepage"` + Size int64 `json:"size"` + StargazersCount int64 `json:"stargazers_count"` + WatchersCount int64 `json:"watchers_count"` + Language *string `json:"language"` + HasIssues bool `json:"has_issues"` + HasDownloads bool `json:"has_downloads"` + HasWiki bool `json:"has_wiki"` + HasPages bool `json:"has_pages"` + ForksCount int64 `json:"forks_count"` + MirrorURL *string `json:"mirror_url"` + OpenIssuesCount int64 `json:"open_issues_count"` + Forks int64 `json:"forks"` + OpenIssues int64 `json:"open_issues"` + Watchers int64 `json:"watchers"` + DefaultBranch string `json:"default_branch"` + } `json:"repository"` + Sender struct { + Login string `json:"login"` + ID int64 `json:"id"` + NodeID string `json:"node_id"` + AvatarURL string `json:"avatar_url"` + GravatarID string `json:"gravatar_id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + FollowersURL string `json:"followers_url"` + FollowingURL string `json:"following_url"` + GistsURL string `json:"gists_url"` + StarredURL string `json:"starred_url"` + SubscriptionsURL string `json:"subscriptions_url"` + OrganizationsURL string `json:"organizations_url"` + ReposURL string `json:"repos_url"` + EventsURL string `json:"events_url"` + ReceivedEventsURL string `json:"received_events_url"` + Type string `json:"type"` + SiteAdmin bool `json:"site_admin"` + } `json:"sender"` +}