Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cdn qps default #287

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Cdn qps default #287

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7f76522
add default value
sxci Oct 28, 2020
07ac89a
add default value
sxci Oct 28, 2020
de9f03b
remove a debug
sxci Oct 28, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ _testmain.go
.idea

.vscode
.DS_Store

# go test coverage file
coverage.txt
27 changes: 19 additions & 8 deletions cmd/cdn.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,18 +55,17 @@ func init() {

cdnRefreshCmd.Flags().BoolVarP(&isDir, "dirs", "r", false, "refresh directory")
cdnRefreshCmd.Flags().StringVarP(&prefetchFile, "input-file", "i", "", "input file")
cdnRefreshCmd.Flags().IntVar(&qpsLimit, "qps", 0, "qps limit for http call")
cdnRefreshCmd.Flags().IntVarP(&itemsLimit, "size", "s", 0, "max item-size pre commit")
cdnRefreshCmd.Flags().IntVar(&qpsLimit, "qps", 1, "qps limit for http call")
cdnRefreshCmd.Flags().IntVarP(&itemsLimit, "size", "s", -1, "max item-size pre commit")

cdnPreCmd.Flags().StringVarP(&prefetchFile, "input-file", "i", "", "input file")
cdnPreCmd.Flags().IntVar(&qpsLimit, "qps", 0, "qps limit for http call")
cdnPreCmd.Flags().IntVarP(&itemsLimit, "size", "s", 0, "max item-size pre commit")
cdnPreCmd.Flags().IntVar(&qpsLimit, "qps", 1, "qps limit for http call")
cdnPreCmd.Flags().IntVarP(&itemsLimit, "size", "s", -1, "max item-size pre commit")

RootCmd.AddCommand(cdnPreCmd, cdnRefreshCmd)
}

func initOnInitialize() {
logs.Debug("qps limit: %d, max item-size: %d", qpsLimit, itemsLimit)
if qpsLimit > 0 {
d := time.Second / time.Duration(qpsLimit)
timeTicker = time.NewTicker(d)
Expand All @@ -81,6 +80,14 @@ func acquire() {

// 【cdnrefresh】刷新所有CDN节点
func CdnRefresh(cmd *cobra.Command, params []string) {
if itemsLimit == -1 {
if isDir {
itemsLimit = 9
} else {
itemsLimit = 59
}
}
logs.Debug("dir: %v, qps limit: %d, max item-size: %d", isDir, qpsLimit, itemsLimit)
var urlListFile string

if prefetchFile != "" {
Expand Down Expand Up @@ -132,7 +139,7 @@ func CdnRefresh(cmd *cobra.Command, params []string) {
if len(itemsToRefresh) == BATCH_CDN_REFRESH_URLS_ALLOW_MAX ||
(itemsLimit > 0 && len(itemsToRefresh) >= itemsLimit) {
cdnRefresh(cm, itemsToRefresh, nil)
itemsToRefresh = make([]string, 0, 50)
itemsToRefresh = make([]string, 0, 60)
}
}
}
Expand Down Expand Up @@ -162,6 +169,10 @@ func cdnRefresh(cm *cdn.CdnManager, urls []string, dirs []string) {

// 【cdnprefetch】CDN 文件预取
func CdnPrefetch(cmd *cobra.Command, params []string) {
if itemsLimit == -1 {
itemsLimit = 59
}
logs.Debug("qps limit: %d, max item-size: %d", qpsLimit, itemsLimit)
var urlListFile string

if prefetchFile != "" {
Expand All @@ -186,7 +197,7 @@ func CdnPrefetch(cmd *cobra.Command, params []string) {
cm := iqshell.GetCdnManager()
scanner := bufio.NewScanner(fp)

urlsToPrefetch := make([]string, 0, 10)
urlsToPrefetch := make([]string, 0, 60)
for scanner.Scan() {
url := strings.TrimSpace(scanner.Text())
if url == "" {
Expand All @@ -197,7 +208,7 @@ func CdnPrefetch(cmd *cobra.Command, params []string) {
if len(urlsToPrefetch) == BATCH_CDN_PREFETCH_ALLOW_MAX ||
(itemsLimit > 0 && len(urlsToPrefetch) >= itemsLimit) {
cdnPrefetch(cm, urlsToPrefetch)
urlsToPrefetch = make([]string, 0, 10)
urlsToPrefetch = make([]string, 0, 60)
}
}

Expand Down