add redis cluster to test setup

circle.yml

@@ -6,27 +6,32 @@ machine:
     - /usr/bin/redis-server --port 6380:
           background: true
   environment:
-    SRC_LOCATION:     "/home/ubuntu/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME"
-    COVERAGE_PROFILE: "/home/ubuntu/coverage.out"
-    GO_LDFLAGS:       '-extldflags "-static" -X main.VERSION=$CIRCLE_TAG -X main.COMMIT_SHA1=$CIRCLE_SHA1 -X main.BUILD_DATE=$(date +%F-%T)'
-    MY_GO_VERSION:    "1.9.2"
+    SRC_LOCATION:       "/home/ubuntu/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME"
+    COVERAGE_PROFILE:   "/home/ubuntu/coverage.out"
+    GO_LDFLAGS:         '-extldflags "-static" -X main.VERSION=$CIRCLE_TAG -X main.COMMIT_SHA1=$CIRCLE_SHA1 -X main.BUILD_DATE=$(date +%F-%T)'
+    MY_GO_VERSION:      "1.9.2"
+    REDIS_TEST_VERSION: "3.2.11"
 
 dependencies:
   pre:
     - rm -rf /home/ubuntu/.go_workspace
     - rm -rf /home/ubuntu/.go_project
     - sudo service redis-server stop
     - >
-      cd ~ && if [ ! -d "redis-3.2.10" ]; then
-        wget http://download.redis.io/releases/redis-3.2.10.tar.gz
-        tar xzf redis-3.2.10.tar.gz
-        cd redis-3.2.10 && make;
+      cd ~ && if [ ! -d "redis-$REDIS_TEST_VERSION" ]; then
+        wget http://download.redis.io/releases/redis-$REDIS_TEST_VERSION.tar.gz
+        tar xzf redis-$REDIS_TEST_VERSION.tar.gz
+        cd redis-$REDIS_TEST_VERSION && make;
       fi
-    - cd ~/redis-3.2.10 && sudo make install
+    - cd ~/redis-$REDIS_TEST_VERSION && sudo make install
     - sudo sed -i 's/bin/local\/bin/g' /etc/init/redis-server.conf
     - sudo service redis-server start
+    #
+    # the next line will bring up a cluster of redis instances with slaves
+    # for addtl tests
+    - docker run -p 7000-7006:7000-7006 --name redis_cluster_test -d oliver006/redis_cluster_test
   cache_directories:
-    - ~/redis-3.2.10
+    - ~/redis-$REDIS_TEST_VERSION
   override:
     - mkdir -p "/home/ubuntu/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME"
     - ln -s $HOME/$CIRCLE_PROJECT_REPONAME $SRC_LOCATION

exporter/redis.go

@@ -570,6 +570,16 @@ func (e *Exporter) extractInfoMetrics(info, addr string, alias string, scrapes c
 	return nil
 }
 
+func doRedisCmd(c redis.Conn, cmd string, args ...interface{}) (reply interface{}, err error) {
+	log.Debugf("c.Do() - running command: %s %s", cmd, args)
+	defer log.Debugf("c.Do() - done")
+	res, err := c.Do(cmd, args...)
+	if err != nil {
+		log.Debugf("c.Do() - err: %s", err)
+	}
+	return res, err
+}
+
 func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx int) error {
 	options := []redis.DialOption{
 		redis.DialConnectTimeout(5 * time.Second),
@@ -596,7 +606,7 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx
 	}
 
 	if err != nil {
-		log.Printf("redis err: %s", err)
+		log.Debugf("aborting for addr: %s - redis err: %s", addr, err)
 		return err
 	}
 
@@ -615,24 +625,24 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx
 		log.Debugf("Redis CONFIG err: %s", err)
 	}
 
-	info, err := redis.String(c.Do("INFO", "ALL"))
+	info, err := redis.String(doRedisCmd(c, "INFO", "ALL"))
 	if err == nil {
 		e.extractInfoMetrics(info, addr, e.redis.Aliases[idx], scrapes, dbCount, true)
 	} else {
 		log.Errorf("Redis INFO err: %s", err)
 		return err
 	}
 
-	if strings.Index(info, "cluster_enabled:1") != -1 {
-		info, err = redis.String(c.Do("CLUSTER", "INFO"))
+	if strings.Contains(info, "cluster_enabled:1") {
+		info, err = redis.String(doRedisCmd(c, "CLUSTER", "INFO"))
 		if err != nil {
 			log.Errorf("redis err: %s", err)
 		} else {
 			e.extractInfoMetrics(info, addr, e.redis.Aliases[idx], scrapes, dbCount, false)
 		}
 	}
 
-	if reply, err := c.Do("LATENCY", "LATEST"); err == nil {
+	if reply, err := doRedisCmd(c, "LATENCY", "LATEST"); err == nil {
 		var eventName string
 		var spikeLast, milliseconds, max int64
 		if tempVal, _ := reply.([]interface{}); len(tempVal) > 0 {
@@ -646,13 +656,14 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx
 		}
 	}
 
+	log.Debugf("e.keys: %#v", e.keys)
 	for _, k := range e.keys {
-		if _, err := c.Do("SELECT", k.db); err != nil {
+		if _, err := doRedisCmd(c, "SELECT", k.db); err != nil {
 			continue
 		}
 
 		obtainedKeys := []string{}
-		if tempVal, err := redis.Strings(c.Do("KEYS", k.key)); err == nil && tempVal != nil {
+		if tempVal, err := redis.Strings(doRedisCmd(c, "KEYS", k.key)); err == nil && tempVal != nil {
 			for _, tempKey := range tempVal {
 				log.Debugf("Append result: %s", tempKey)
 				obtainedKeys = append(obtainedKeys, tempKey)
@@ -662,7 +673,7 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx
 		for _, key := range obtainedKeys {
 			dbLabel := "db" + k.db
 			keyLabel := key
-			if tempVal, err := c.Do("GET", key); err == nil && tempVal != nil {
+			if tempVal, err := doRedisCmd(c, "GET", key); err == nil && tempVal != nil {
 				if val, err := strconv.ParseFloat(fmt.Sprintf("%s", tempVal), 64); err == nil {
 					e.keyValues.WithLabelValues(addr, e.redis.Aliases[idx], dbLabel, keyLabel).Set(val)
 				}
@@ -676,26 +687,26 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx
 				"PFCOUNT",
 				"STRLEN",
 			} {
-				if tempVal, err := c.Do(op, key); err == nil && tempVal != nil {
+				if tempVal, err := doRedisCmd(c, op, key); err == nil && tempVal != nil {
 					e.keySizes.WithLabelValues(addr, e.redis.Aliases[idx], dbLabel, keyLabel).Set(float64(tempVal.(int64)))
 					break
 				}
 			}
 		}
 	}
 
+	log.Debugf("scrapeRedisHost() done")
 	return nil
 }
 
 func (e *Exporter) scrape(scrapes chan<- scrapeResult) {
-
 	defer close(scrapes)
+
 	now := time.Now().UnixNano()
 	e.totalScrapes.Inc()
 
 	errorCount := 0
 	for idx, addr := range e.redis.Addrs {
-
 		var up float64 = 1
 		if err := e.scrapeRedisHost(scrapes, addr, idx); err != nil {
 			errorCount++