Google split (#521)

* updated splitting algorithm for google cloud output operator

* updatewd benchmark

* changed append expression to exclude beginning 0

operator/builtin/output/googlecloud/request.go

@@ -53,13 +53,26 @@ func (g *GoogleRequestBuilder) buildRequests(entries []*logging.LogEntry) []*log
 	}
 
 	totalEntries := len(request.Entries)
-	midPoint := totalEntries / 2
-	leftEntries := request.Entries[0:midPoint]
-	rightEntries := request.Entries[midPoint:totalEntries]
+	firstRequest := g.buildRequest([]*logging.LogEntry{})
+	firstSize := 0
+	index := 0
 
-	leftRequests := g.buildRequests(leftEntries)
-	rightRequests := g.buildRequests(rightEntries)
-	return append(leftRequests, rightRequests...)
+	for i, entry := range request.Entries {
+
+		firstRequest.Entries = append(firstRequest.Entries, entry)
+		firstSize = proto.Size(firstRequest)
+
+		if firstSize > g.MaxRequestSize {
+			index = i
+			firstRequest.Entries = firstRequest.Entries[:index]
+			break
+		}
+	}
+
+	secondEntries := request.Entries[index:totalEntries]
+	secondRequests := g.buildRequests(secondEntries)
+
+	return append([]*logging.WriteLogEntriesRequest{firstRequest}, secondRequests...)
 }
 
 // buildRequest builds a request from the supplied entries

operator/builtin/output/googlecloud/request_test.go

@@ -2,6 +2,7 @@ package googlecloud
 
 import (
 	"errors"
+	"fmt"
 	"testing"
 
 	"github.com/observiq/stanza/entry"
@@ -11,6 +12,29 @@ import (
 	"google.golang.org/genproto/googleapis/logging/v2"
 )
 
+func BenchmarkBuildRequest(b *testing.B) {
+	entryBuilder := &GoogleEntryBuilder{
+		MaxEntrySize: defaultMaxEntrySize,
+		ProjectID:    "project",
+	}
+	entries := []*entry.Entry{}
+
+	for i := 0; i < 1000; i++ {
+		entry, _ := createEntry(i)
+		entries = append(entries, entry)
+	}
+
+	requestBuilder := GoogleRequestBuilder{
+		MaxRequestSize: 10000,
+		ProjectID:      "test_project",
+		EntryBuilder:   entryBuilder,
+		SugaredLogger:  zap.NewNop().Sugar(),
+	}
+
+	requests := requestBuilder.Build(entries)
+	require.Len(b, requests, 3)
+}
+
 func TestBuildRequest(t *testing.T) {
 	entryOne := &entry.Entry{Record: "request 1"}
 	entryTwo := &entry.Entry{Record: "request 2"}
@@ -90,3 +114,9 @@ func (_m *MockEntryBuilder) Build(_a0 *entry.Entry) (*logging.LogEntry, error) {
 
 	return r0, r1
 }
+
+func createEntry(num int) (*entry.Entry, *logging.LogEntry) {
+	entry := &entry.Entry{Record: fmt.Sprintf("request %d", num)}
+	result := &logging.LogEntry{Payload: &logging.LogEntry_TextPayload{TextPayload: fmt.Sprintf("request %d", num)}}
+	return entry, result
+}