diff --git a/github.go b/github.go index 0669639..2bba850 100644 --- a/github.go +++ b/github.go @@ -106,31 +106,41 @@ func FetchTags(githubRepoUrl string, githubToken string, instance GitHubInstance return tagsString, wrapError(err) } - url := createGitHubRepoUrlForPath(repo, "tags") - resp, err := callGitHubApi(repo, url, map[string]string{}) - if err != nil { - return tagsString, err - } + //per_page is max to reduce network calls + for currPath := "tags?per_page=100"; currPath != ""; { + url := createGitHubRepoUrlForPath(repo, currPath) + resp, err := callGitHubApi(repo, url, map[string]string{}) + if err != nil { + return tagsString, err + } - // Convert the response body to a byte array - buf := new(bytes.Buffer) - _, goErr := buf.ReadFrom(resp.Body) - if goErr != nil { - return tagsString, wrapError(goErr) - } - jsonResp := buf.Bytes() + // Convert the response body to a byte array + buf := new(bytes.Buffer) + _, goErr := buf.ReadFrom(resp.Body) + if goErr != nil { + return tagsString, wrapError(goErr) + } + jsonResp := buf.Bytes() - // Extract the JSON into our array of gitHubTagsCommitApiResponse's - var tags []GitHubTagsApiResponse - if err := json.Unmarshal(jsonResp, &tags); err != nil { - return tagsString, wrapError(err) - } + // Extract the JSON into our array of gitHubTagsCommitApiResponse's + var tags []GitHubTagsApiResponse + if err := json.Unmarshal(jsonResp, &tags); err != nil { + return tagsString, wrapError(err) + } - for _, tag := range tags { - // Skip tags that are not semantically versioned so that they don't cause errors. (issue #75) - if _, err := version.NewVersion(tag.Name); err == nil { - tagsString = append(tagsString, tag.Name) + for _, tag := range tags { + // Skip tags that are not semantically versioned so that they don't cause errors. (issue #75) + if _, err := version.NewVersion(tag.Name); err == nil { + tagsString = append(tagsString, tag.Name) + } } + + //Get paginated tags (issue #26 and #46) + currPath, err = getNextPath(resp.Header.Get("link")) + if err != nil { + return tagsString, err + } + } return tagsString, nil @@ -202,6 +212,39 @@ func createGitHubRepoUrlForPath(repo GitHubRepo, path string) string { return fmt.Sprintf("repos/%s/%s/%s", repo.Owner, repo.Name, path) } +// Get the Next paginated path from the link url api.github.com/repos/:owner/:repo/:path +// If there is no next page, return an empty string +// Links are formatted: "; rel=next, ; rel=last" +func getNextPath(links string) (string, *FetchError) { + if len(links) == 0 { + return "", nil + } + + nextLinkRegex, regexErr := regexp.Compile(`<(.+?)>;\s+rel="next"`) + if regexErr != nil { + return "", wrapError(regexErr) + } + pathRegex, regexErr := regexp.Compile(`([^\/]+$)`) + if regexErr != nil { + return "", wrapError(regexErr) + } + + for _, link := range strings.Split(links, ",") { + + urlMatches := nextLinkRegex.FindStringSubmatch(link) + if len(urlMatches) == 2 { + // Get only the next link path + pathMatches := pathRegex.FindStringSubmatch(urlMatches[1]) + if len(pathMatches) != 2 { + return "", newError(githubRepoUrlMalformedOrNotParseable, fmt.Sprintf("Path parsed incorrectly for url: %s", urlMatches[1])) + } + return pathMatches[1], nil + } + } + + return "", nil +} + // Call the GitHub API at the given path and return the HTTP response func callGitHubApi(repo GitHubRepo, path string, customHeaders map[string]string) (*http.Response, *FetchError) { httpClient := &http.Client{} diff --git a/github_test.go b/github_test.go index 5c68e33..fa73709 100644 --- a/github_test.go +++ b/github_test.go @@ -57,6 +57,33 @@ func TestGetListOfReleasesFromGitHubRepo(t *testing.T) { } } +func TestGetNextPath(t *testing.T) { + t.Parallel() + + cases := []struct { + links string + expectedNextPath string + }{ + {`; rel="next", ; rel="last"`, "code?q=addClass+user%3Amozilla&page=15"}, + {`; rel="first", ; rel="last"`, ""}, + {`; rel="next", ; rel="last"`, "tags?page=15"}, + {`; rel="next", ; rel="last"`, "tags?per_page=100&page=15"}, + } + + for _, tc := range cases { + nextPath, err := getNextPath(tc.links) + + if err != nil { + t.Fatalf("error getting next path: %s", err) + } + + if nextPath != tc.expectedNextPath { + t.Fatalf("Expected next path %s, but got %s", tc.expectedNextPath, nextPath) + } + } + +} + func TestParseUrlIntoGithubInstance(t *testing.T) { t.Parallel()