Skip to content

Commit

Permalink
Merge pull request #510 from Gui-Yue/bpf_visual
Browse files Browse the repository at this point in the history
Save data to sqlite3&Start service with one command.
  • Loading branch information
chenamy2017 authored Sep 5, 2023
2 parents 3723ea1 + a3342ba commit 7f85382
Show file tree
Hide file tree
Showing 11 changed files with 213 additions and 23 deletions.
1 change: 1 addition & 0 deletions .github/workflows/ebpf_prometheus.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,5 +29,6 @@ jobs:
run: |
cd eBPF_Visualization/eBPF_prometheus
make
make start_service
sudo -E timeout 120 ./data-visual c example/vfsstat.py || if [[ $? != 124 && $? != 0 ]];then exit $?;fi
timeout-minutes: 5
4 changes: 4 additions & 0 deletions eBPF_Visualization/eBPF_prometheus/Makefile
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
Prometheus_image = prom/prometheus
all:
go mod tidy
go build -o data-visual ./main.go

start_service:
./runimages.sh
29 changes: 12 additions & 17 deletions eBPF_Visualization/eBPF_prometheus/Readme.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
## BPF数据可视化

​ 该项目是eBPF_Visualization项目的一部分,目前是一个命令行工具,实现数据加载到符合Prometheus规范的metrics中,此后,利用prometheus-server方便地对收集地数据进行管理,利用grafana进行数据的可视化。

## 环境要求:
golang 1.19+
Docker version 24.0.4+
(docker安装方式参考:https://docs.docker.com/desktop/install/linux-install/)
## 输出的格式要求

第一行输出各项指标的名称,之后每一行输出数据,以空格分隔。
Expand All @@ -15,6 +18,10 @@ TIME READ/s WRITE/s FSYNC/s OPEN/s CREATE/s
```

## 使用方式
```bash
make start_service
```
上述命令会自动配置下载docker镜像并启动grafana和prometheus服务。建议一直使用该命令一键式启动所需要的容器。

python封装的bpf程序:
```bash
Expand Down Expand Up @@ -51,27 +58,15 @@ $ ./data-visual proc_image

### 使用prometheus-server的docker镜像监控metrics

编辑prom_core/prometheus.yaml中targets参数,使其符合用户ip地址,默认127.0.0.1。之后使用下述操作。

```bash
docker run \
-p 9090:9090 \
-v ./prom_core/promehteus.yaml:/etc/prometheus/prometheus.yml \
prom/prometheus
```

以上命令会自动下载prometheus-server的docker镜像,并以/prom_core/promehteus.yaml为配置文件启动prometheus-server。
编辑prom_core/prometheus.yaml中targets参数,使其符合用户ip地址,默认127.0.0.1。

通过访问http://127.0.0.1:9090 可对prometheus-server进行管理。进入Status-Targets,即可看到metrics的状态。
![捕获1](https://github.com/Gui-Yue/lmp/assets/78520005/0ed9e69f-d477-4f7e-91e0-3e9d240f31d3)


### 使用grafana的docker镜像绘图

```bash
docker run -d -p 3000:3000 --name=grafana grafana/grafana-enterprise
```

以上命令会自动下载grafana的docker镜像,并启动,可通过http://127.0.0.1:3000 访问。在grafana中进行可视化设置,选择prometheus服务器,即可实现metrics可视化。效果如下:
通过http://127.0.0.1:3000 访问。在grafana中进行可视化设置,选择prometheus服务器,即可实现metrics可视化。效果如下:
![捕获2](https://github.com/Gui-Yue/lmp/assets/78520005/b7bb8668-b3cb-496a-bbfc-ba74ea3ef1b7)

### 通过sqlite3查看收集到的数据
进入/dao目录可以看到data.db数据库文件,通过sqlite3访问查看收集到的数据。
22 changes: 19 additions & 3 deletions eBPF_Visualization/eBPF_prometheus/collector/collect_output.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ package collector
import (
"bufio"
"ebpf_prometheus/checker"
"ebpf_prometheus/dao"
"ebpf_prometheus/prom_core"
"fmt"
"github.com/urfave/cli/v2"
Expand Down Expand Up @@ -109,12 +110,19 @@ func newProcCmd(ctx *cli.Context, opts ...interface{}) (interface{}, error) {
return proc_imageCommand, nil
}

type BPF_name struct {
Name string
}

func simpleCollect(ctx *cli.Context) error {
filePath, err := checker.CollectCheck(ctx)
if err != nil {
return err
}
return Run(filePath)
path := strings.Fields(filePath)[0]
pathlist := strings.Split(path, "/")
n := BPF_name{Name: strings.ReplaceAll(pathlist[len(pathlist)-1], ".py", "")}
return n.Run(filePath)
}

func CheckFileType(filePath string) (specificcommand string) {
Expand All @@ -137,7 +145,7 @@ func CheckFileType(filePath string) (specificcommand string) {
}
}

func Run(filePath string) error {
func (b *BPF_name) Run(filePath string) error {
cmdStr := CheckFileType(filePath)
cmd := exec.Command("sh", "-c", cmdStr)

Expand All @@ -161,7 +169,9 @@ func Run(filePath string) error {
log.Println("I am normal")
}

metricsobj := &prom_core.MyMetrics{}
metricsobj := &prom_core.MyMetrics{BPFName: b.Name, Sqlinited: false}
sqlobj := &dao.Sqlobj{Tablename: b.Name}
metricsobj.Sqlobj = sqlobj

go metricsobj.StartService()
// process chan from redirect Stdout
Expand All @@ -170,7 +180,13 @@ func Run(filePath string) error {
select {
case <-mapchan:
metricsobj.Maplist = <-mapchan
log.Println(metricsobj.Maplist)
metricsobj.UpdateData()
if metricsobj.Sqlinited {
metricsobj.UpdataSql()
} else {
metricsobj.Initsql()
}
<-mapchan
default:
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ package collector
import (
"bufio"
"ebpf_prometheus/checker"
"ebpf_prometheus/dao"
"ebpf_prometheus/prom_core"
"github.com/urfave/cli/v2"
"gopkg.in/yaml.v2"
Expand Down Expand Up @@ -93,7 +94,10 @@ func ProcRun(command string) error {
loc, _ := time.LoadLocation("Asia/Shanghai")
currenttime := float64(time.Now().In(loc).UnixNano()) / 1e9
go redirectProc(stdout, mapchan)

procdata := prom_core.ProcMetrics{Max_records: maxrecords, NowTime: currenttime}
sqlobj := &dao.Sqlobj{Tablename: "proc_image_data"}
procdata.Sqlobj = sqlobj
// process chan from redirectProc Stdout
go procdata.BootProcService()

Expand All @@ -102,6 +106,11 @@ func ProcRun(command string) error {
select {
case <-mapchan:
procdata.Getorigindata(mapchan)
if procdata.Sqlinted {
procdata.UpdateSql()
} else {
procdata.Initsql()
}
procdata.UpdateRecords()
<-mapchan
default:
Expand Down
88 changes: 88 additions & 0 deletions eBPF_Visualization/eBPF_prometheus/dao/data_to_sqlite.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
// Copyright 2023 The LMP Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://github.com/linuxkerneltravel/lmp/blob/develop/LICENSE
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// author: Gui-Yue
//
// 数据持久化的方式:通过将数据写入数据库实现数据持久化。

package dao

import (
"fmt"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"log"
"os"
"strconv"
)

type Sqlobj struct {
Tablename string
db *gorm.DB
Data map[string]interface{}
}

type Basicdata struct {
ID int `gorm:"primaryKey;unique;column:ID"`
}

// Connectsql 连接数据库
func (s *Sqlobj) Connectsql() {
currentdir, _ := os.Getwd()
path := currentdir + "/dao/data.db"
db, _ := gorm.Open(sqlite.Open(path), &gorm.Config{})
log.Println("connected.")
s.db = db
}

func (s *Sqlobj) Tableexist(name string) bool {
return s.db.Migrator().HasTable(name)
}

// CreateTable 建表
func (s *Sqlobj) OperateTable(name string) {
if !s.Tableexist(name) {
deletetable := fmt.Sprintf("drop table if exists %s;", s.Tablename)
if err := s.db.Exec(deletetable).Error; err != nil {
log.Fatalf("drop exist table failed.")
}
if err := s.db.Table(s.Tablename).AutoMigrate(&Basicdata{}); err != nil {
log.Fatalf("create table failed.")
}
s.AppendTable()
s.CreateRow()
} else {
s.CreateRow()
}
}

// AppendTable 扩展表
func (s *Sqlobj) AppendTable() {
for key, value := range s.Data {
datatype := "text"
if strvalue, is_string := value.(string); is_string {
// shift numerical data to float64
if _, err := strconv.ParseFloat(strvalue, 64); err == nil {
datatype = "real"
}
}
addcolumn := fmt.Sprintf("alter table %s add column \"%s\" %s", s.Tablename, key, datatype)
s.db.Exec(addcolumn)
}
}

// CreateRow 写入数据
func (s *Sqlobj) CreateRow() {
s.db.Table(s.Tablename).Create(s.Data)
}
5 changes: 5 additions & 0 deletions eBPF_Visualization/eBPF_prometheus/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,19 @@ require (
github.com/prometheus/client_golang v1.16.0
github.com/urfave/cli/v2 v2.25.7
gopkg.in/yaml.v2 v2.4.0
gorm.io/driver/sqlite v1.5.3
gorm.io/gorm v1.25.4
)

require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/mattn/go-sqlite3 v1.14.17 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
Expand Down
10 changes: 10 additions & 0 deletions eBPF_Visualization/eBPF_prometheus/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,15 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
Expand Down Expand Up @@ -46,3 +52,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gorm.io/driver/sqlite v1.5.3 h1:7/0dUgX28KAcopdfbRWWl68Rflh6osa4rDh+m51KL2g=
gorm.io/driver/sqlite v1.5.3/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4=
gorm.io/gorm v1.25.4 h1:iyNd8fNAe8W9dvtlgeRI5zSVZPsq3OpcTu37cYcpCmw=
gorm.io/gorm v1.25.4/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
22 changes: 19 additions & 3 deletions eBPF_Visualization/eBPF_prometheus/prom_core/processer.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package prom_core

import (
"ebpf_prometheus/checker"
"ebpf_prometheus/dao"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"log"
Expand All @@ -30,9 +31,12 @@ import (
)

type MyMetrics struct {
mu sync.Mutex
Maps map[string]interface{}
Maplist []map[string]interface{}
BPFName string
mu sync.Mutex
Maps map[string]interface{}
Maplist []map[string]interface{}
Sqlobj *dao.Sqlobj
Sqlinited bool
}

func (m *MyMetrics) Describe(ch chan<- *prometheus.Desc) {}
Expand All @@ -48,6 +52,18 @@ func (m *MyMetrics) UpdateData() {
m.Maps = new_Dict
}

func (m *MyMetrics) UpdataSql() {
m.Sqlobj.Data = m.Maps
m.Sqlobj.CreateRow()
}

func (m *MyMetrics) Initsql() {
m.Sqlobj.Data = m.Maps
m.Sqlobj.Connectsql()
m.Sqlobj.OperateTable(m.BPFName)
m.Sqlinited = true
}

// Format_Dict format dict.
func Format_Dict(dict map[string]interface{}) (map[string]float64, map[string]string) {
measurable_dict := map[string]float64{}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package prom_core

import (
"ebpf_prometheus/dao"
"encoding/json"
"log"
"net/http"
Expand All @@ -33,6 +34,8 @@ type ProcMetrics struct {
mu sync.Mutex
OriginalValue map[string]interface{}
Records []OneRecord
Sqlinted bool
Sqlobj *dao.Sqlobj
}

// 原始数据处理后所保留的基本数据
Expand All @@ -47,6 +50,18 @@ func (p *ProcMetrics) Getorigindata(originalvalue chan map[string]interface{}) {
p.OriginalValue = <-originalvalue
}

func (p *ProcMetrics) UpdateSql() {
p.Sqlobj.Data = p.OriginalValue
p.Sqlobj.CreateRow()
}

func (p *ProcMetrics) Initsql() {
p.Sqlobj.Data = p.OriginalValue
p.Sqlobj.Connectsql()
p.Sqlobj.OperateTable("proc_image")
p.Sqlinted = true
}

// processJson 实现将原始数据进行处理,获取展示所需要的基本数据
func (p *ProcMetrics) processJson() OneRecord {
timestamp := float64(0)
Expand Down
Loading

0 comments on commit 7f85382

Please sign in to comment.