added pagination for github api call

.github/workflows/contributors/update_contributors.py

@@ -20,92 +20,94 @@ def main(_):
         "Authorization": f"token {token}"
     }
 
-    web_address = f'https://api.github.com/repos/{OWNER}/{REPO}/commits?sha={BRANCH}&per_page=100'
-    res = requests.get(web_address, headers=headers)
+    data = []
+    next_page = f'https://api.github.com/repos/{OWNER}/{REPO}/commits?sha={BRANCH}&per_page=100'
+    last_page = None
+    while next_page != last_page:
+        print(f'Fetching page: {next_page}')
+        res = requests.get(next_page, headers=headers)
+        data.extend(res.json())
+        next_page = res.links.get('next', {}).get('url', None)
+        last_page = res.links.get('last', {}).get('url', None)
 
-    print(web_address)
     user_to_name_dict = dict()
     users_from_api = []
 
-    if res.status_code == 200:
-        data = res.json()
-
-        for node in data:
-            user_full_name = None
-            username = None
-            if node['commit']:
-                commit = node['commit']
-                if commit['author']:
-                    author = commit['author']
-                    user_full_name = author['name']
-                elif commit['committer']:
-                    committer = commit['committer']
-                    user_full_name = committer['name']
-            if node['committer']:
-                committer = node['committer']
-                username = committer['login']
-            if node['author']:
-                author = node['author']
-                if author['login']:
-                    username = author['login']
-
-            assert user_full_name is not None, 'User full name should not be None'
-            assert username is not None, 'Username should not be None'
-
-            user_to_name_dict[username] = user_full_name
-            users_from_api.append(username)
-
-        users_from_api = set(users_from_api)
-        print('Users pulled from API: ', users_from_api)
-
-        with open(CONTRIBUTORS_FILE, 'r') as contrib_file:
-            existing_contributor_data = json.load(contrib_file)
-            existing_contributors = existing_contributor_data['contributors']
-
-            existing_contributor_logins = []
-            for existing_contributor in existing_contributors:
-                user_to_name_dict[existing_contributor['login']] = existing_contributor['name']
-                existing_contributor_logins.append(existing_contributor['login'])
-            existing_contributor_logins_set = set(existing_contributor_logins)
-            print('Existing contributors: ', existing_contributor_logins_set)
-            existing_contributor_logins_set -= EXCLUDED_USERS
-            # All contributors in the file should be in the API
-            assert existing_contributor_logins_set.issubset(
-                users_from_api), 'All contributors in the .all-contributorsrc file should be pulled using the API'
-
-            new_contributor_logins = users_from_api - existing_contributor_logins_set
-            print('New contributors: ', new_contributor_logins)
-
-            result = users_from_api - EXCLUDED_USERS
-
-            final_result = dict(
-                projectName=REPO,
-                projectOwner=OWNER,
-                files=["contributors.qmd", "README.md"],
-                contributors=[dict(login=user,
-                                   name=user_to_name_dict[user],
-                                   avatar_url=f'https://avatars.githubusercontent.com/{user}',
-                                   profile=f'https://github.com/{user}',
-                                   contributions=['doc'], ) for
-                              user in result],
-
-                repoType='github',
-                contributorsPerLine=7,
-                repoHost="https=//github.com",
-                commitConvention='angular',
-                skipCi=True,
-                commitType="docs"
-            )
-
-            print(final_result)
-            json_string = json.dumps(final_result,
-                                     indent=4)  # The indent parameter is optional, but it formats the output to be more readable
-            print(json_string)
-
-        with open(CONTRIBUTORS_FILE, 'w') as contrib_file:
-            contrib_file.write(json_string)
-    else:
-        print(f"Failed with status code: {res.status_code}")
+    for node in data:
+        commit_info = node.get('commit', None)
+        commit_author_info = commit_info.get('author', None)
+        commit_commiter_info = commit_info.get('committer', None)
+        author_info = node.get('author', None)
+        committer_info = node.get('committer', None)
+        committer_login_info = committer_info.get('login', None)
+        user_full_name = None
+        username = None
+
+        if commit_author_info:
+            user_full_name = commit_author_info['name']
+        elif commit_commiter_info:
+            user_full_name = commit_commiter_info['name']
+
+        if author_info:
+            username = author_info['login']
+        elif committer_login_info:
+            username = committer_login_info['login']
+
+        assert user_full_name is not None, 'User full name should not be None'
+        assert username is not None, 'Username should not be None'
+
+        user_to_name_dict[username] = user_full_name
+        users_from_api.append(username)
+
+    users_from_api = set(users_from_api)
+    print('Users pulled from API: ', users_from_api)
+
+    with open(CONTRIBUTORS_FILE, 'r') as contrib_file:
+        existing_contributor_data = json.load(contrib_file)
+        existing_contributors = existing_contributor_data['contributors']
+
+        existing_contributor_logins = []
+        for existing_contributor in existing_contributors:
+            user_to_name_dict[existing_contributor['login']] = existing_contributor['name']
+            existing_contributor_logins.append(existing_contributor['login'])
+        existing_contributor_logins_set = set(existing_contributor_logins)
+        print('Existing contributors: ', existing_contributor_logins_set)
+        existing_contributor_logins_set -= EXCLUDED_USERS
+        # All contributors in the file should be in the API
+        assert existing_contributor_logins_set.issubset(
+            users_from_api), 'All contributors in the .all-contributorsrc file should be pulled using the API'
+
+        new_contributor_logins = users_from_api - existing_contributor_logins_set
+        print('New contributors: ', new_contributor_logins)
+
+        result = users_from_api - EXCLUDED_USERS
+
+        final_result = dict(
+            projectName=REPO,
+            projectOwner=OWNER,
+            files=["contributors.qmd", "README.md"],
+            contributors=[dict(login=user,
+                               name=user_to_name_dict[user],
+                               avatar_url=f'https://avatars.githubusercontent.com/{user}',
+                               profile=f'https://github.com/{user}',
+                               contributions=['doc'], ) for
+                          user in result],
+
+            repoType='github',
+            contributorsPerLine=7,
+            repoHost="https=//github.com",
+            commitConvention='angular',
+            skipCi=True,
+            commitType="docs"
+        )
+
+        print(final_result)
+        json_string = json.dumps(final_result,
+                                 indent=4)  # The indent parameter is optional, but it formats the output to be more readable
+        print(json_string)
+
+    with open(CONTRIBUTORS_FILE, 'w') as contrib_file:
+        contrib_file.write(json_string)
 
 
 if __name__ == '__main__':