fixes issues 26 & 46. Added support for pagination

github.go

@@ -106,31 +106,41 @@ func FetchTags(githubRepoUrl string, githubToken string, instance GitHubInstance
 		return tagsString, wrapError(err)
 	}
 
-	url := createGitHubRepoUrlForPath(repo, "tags")
-	resp, err := callGitHubApi(repo, url, map[string]string{})
-	if err != nil {
-		return tagsString, err
-	}
+	//per_page is max to reduce network calls
+	for currPath := "tags?per_page=100"; currPath != ""; {
+		url := createGitHubRepoUrlForPath(repo, currPath)
+		resp, err := callGitHubApi(repo, url, map[string]string{})
+		if err != nil {
+			return tagsString, err
+		}
 
-	// Convert the response body to a byte array
-	buf := new(bytes.Buffer)
-	_, goErr := buf.ReadFrom(resp.Body)
-	if goErr != nil {
-		return tagsString, wrapError(goErr)
-	}
-	jsonResp := buf.Bytes()
+		// Convert the response body to a byte array
+		buf := new(bytes.Buffer)
+		_, goErr := buf.ReadFrom(resp.Body)
+		if goErr != nil {
+			return tagsString, wrapError(goErr)
+		}
+		jsonResp := buf.Bytes()
 
-	// Extract the JSON into our array of gitHubTagsCommitApiResponse's
-	var tags []GitHubTagsApiResponse
-	if err := json.Unmarshal(jsonResp, &tags); err != nil {
-		return tagsString, wrapError(err)
-	}
+		// Extract the JSON into our array of gitHubTagsCommitApiResponse's
+		var tags []GitHubTagsApiResponse
+		if err := json.Unmarshal(jsonResp, &tags); err != nil {
+			return tagsString, wrapError(err)
+		}
 
-	for _, tag := range tags {
-		// Skip tags that are not semantically versioned so that they don't cause errors. (issue #75)
-		if _, err := version.NewVersion(tag.Name); err == nil {
-			tagsString = append(tagsString, tag.Name)
+		for _, tag := range tags {
+			// Skip tags that are not semantically versioned so that they don't cause errors. (issue #75)
+			if _, err := version.NewVersion(tag.Name); err == nil {
+				tagsString = append(tagsString, tag.Name)
+			}
 		}
+
+		//Get paginated tags (issue #26 and #46)
+		currPath, err = getNextPath(resp.Header.Get("link"))
+		if err != nil {
+			return tagsString, err
+		}
+
 	}
 
 	return tagsString, nil
@@ -202,6 +212,39 @@ func createGitHubRepoUrlForPath(repo GitHubRepo, path string) string {
 	return fmt.Sprintf("repos/%s/%s/%s", repo.Owner, repo.Name, path)
 }
 
+// Get the Next paginated path from the link url api.github.com/repos/:owner/:repo/:path
+// If there is no next page, return an empty string
+// Links are formatted: "<url>; rel=next, <url>; rel=last"
+func getNextPath(links string) (string, *FetchError) {
+	if len(links) == 0 {
+		return "", nil
+	}
+
+	nextLinkRegex, regexErr := regexp.Compile(`<(.+?)>;\s+rel="next"`)
+	if regexErr != nil {
+		return "", wrapError(regexErr)
+	}
+	pathRegex, regexErr := regexp.Compile(`([^\/]+$)`)
+	if regexErr != nil {
+		return "", wrapError(regexErr)
+	}
+
+	for _, link := range strings.Split(links, ",") {
+
+		urlMatches := nextLinkRegex.FindStringSubmatch(link)
+		if len(urlMatches) == 2 {
+			// Get only the next link path
+			pathMatches := pathRegex.FindStringSubmatch(urlMatches[1])
+			if len(pathMatches) != 2 {
+				return "", newError(githubRepoUrlMalformedOrNotParseable, fmt.Sprintf("Path parsed incorrectly for url: %s", urlMatches[1]))
+			}
+			return pathMatches[1], nil
+		}
+	}
+
+	return "", nil
+}
+
 // Call the GitHub API at the given path and return the HTTP response
 func callGitHubApi(repo GitHubRepo, path string, customHeaders map[string]string) (*http.Response, *FetchError) {
 	httpClient := &http.Client{}

github_test.go

@@ -57,6 +57,33 @@ func TestGetListOfReleasesFromGitHubRepo(t *testing.T) {
 	}
 }
 
+func TestGetNextPath(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		links            string
+		expectedNextPath string
+	}{
+		{`<https://api.github.com/search/code?q=addClass+user%3Amozilla&page=15>; rel="next", <https://api.github.com/search/code?q=addClass+user%3Amozilla&page=34>; rel="last"`, "code?q=addClass+user%3Amozilla&page=15"},
+		{`<https://api.github.com/search/code?q=addClass+user%3Amozilla&page=15>; rel="first", <https://api.github.com/search/code?q=addClass+user%3Amozilla&page=34>; rel="last"`, ""},
+		{`<https://api.github.com/temp/proj/tags?page=15>; rel="next", <https://api.github.com/temp/proj/tags?page=15>; rel="last"`, "tags?page=15"},
+		{`<https://api.github.com/temp/proj/tags?per_page=100&page=15>;  	rel="next", <https://api.github.com/temp/proj/tags?per_page=100&page=15>;  rel="last"`, "tags?per_page=100&page=15"},
+	}
+
+	for _, tc := range cases {
+		nextPath, err := getNextPath(tc.links)
+
+		if err != nil {
+			t.Fatalf("error getting next path: %s", err)
+		}
+
+		if nextPath != tc.expectedNextPath {
+			t.Fatalf("Expected next path %s, but got %s", tc.expectedNextPath, nextPath)
+		}
+	}
+
+}
+
 func TestParseUrlIntoGithubInstance(t *testing.T) {
 	t.Parallel()