github_search_api.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-02T20:09:14.509720Z",
     "start_time": "2018-06-02T20:09:14.399008Z"
    }
   },
   "outputs": [],
   "source": [
    "# Import required modules\n",
    "import requests\n",
    "import time\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-02T20:09:15.866468Z",
     "start_time": "2018-06-02T20:09:15.863341Z"
    }
   },
   "outputs": [],
   "source": [
    "# Paste your Access token here\n",
    "# To create an access token - https://github.com/settings/tokens\n",
    "token = \"access_token=\" + \"access_token_here\"\n",
    "\n",
    "# Base API Endpoint\n",
    "base_api_url = 'https://api.github.com/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-02T20:09:20.264582Z",
     "start_time": "2018-06-02T20:09:16.999410Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter the Search Query to get the Data \n",
      "tensorflow\n",
      "\n",
      " Query entered is tensorflow \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Enter multiple word queries with a '+' sign\n",
    "# Ex: machine+learning to search for Machine Learning\n",
    "\n",
    "print('Enter the Search Query to get the Data ')\n",
    "\n",
    "query = input()\n",
    "print('\\n Query entered is', query, '\\n')\n",
    "\n",
    "search_final_url = base_api_url + 'search/repositories?q=' + query + '&' + token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-02T20:09:22.904005Z",
     "start_time": "2018-06-02T20:09:22.900182Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# A CSV file containting the data would be saved with the name as the query\n",
    "# Ex: machine+learning.csv\n",
    "filename = query + '.csv'\n",
    "\n",
    "# Create a CSV file or clear the existing one with the same name\n",
    "with open(filename, 'w', newline='') as csvfile:\n",
    "    write_to_csv = csv.writer(csvfile, delimiter='|')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-02T20:15:31.673898Z",
     "start_time": "2018-06-02T20:14:52.565406Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Repo Name =  tensorflow \tDescription Computation using data flow graphs for scalable machine learning \tStars =  101561 \tWatchers =  101561 \tForks =  63763 \tOpen Issues =  1626 \tPrimary Language =  C++ \tRepo Languages = {'C++': 48.09, 'Python': 40.83, 'HTML': 5.21, 'Jupyter Notebook': 2.46, 'Go': 1.28, 'Java': 0.84, 'Shell': 0.48, 'C': 0.34, 'CMake': 0.23, 'Objective-C++': 0.11, 'Makefile': 0.05, 'PureBasic': 0.03, 'Objective-C': 0.01, 'Batchfile': 0.01, 'Perl': 0.01, 'Smarty': 0.01, 'LLVM': 0.01, 'PHP': 0.0, 'Ruby': 0.0} \tRepo Score 153.34117\n",
      "==========\n",
      "Repo Name =  TensorFlow-Examples \tDescription TensorFlow Tutorial and Examples for Beginners with Latest APIs \tStars =  22670 \tWatchers =  22670 \tForks =  8262 \tOpen Issues =  126 \tPrimary Language =  Jupyter Notebook \tRepo Languages = {'Jupyter Notebook': 99.28, 'Python': 0.72} \tRepo Score 104.73484\n",
      "==========\n",
      "Repo Name =  tensorflow-zh \tDescription 谷歌全新开源人工智能系统TensorFlow官方文档中文版 \tStars =  8744 \tWatchers =  8744 \tForks =  3563 \tOpen Issues =  26 \tPrimary Language =  TeX \tRepo Languages = {'TeX': 95.01, 'Python': 4.37, 'C++': 0.59, 'HTML': 0.04} \tRepo Score 101.07045\n",
      "==========\n",
      "Repo Name =  models \tDescription Models and examples built with TensorFlow \tStars =  36053 \tWatchers =  36053 \tForks =  20834 \tOpen Issues =  962 \tPrimary Language =  Python \tRepo Languages = {'Python': 69.2, 'Jupyter Notebook': 18.39, 'C++': 10.29, 'HTML': 0.97, 'Shell': 0.88, 'JavaScript': 0.22, 'Makefile': 0.03, 'GLSL': 0.01} \tRepo Score 99.58476\n",
      "==========\n",
      "Repo Name =  awesome-tensorflow \tDescription TensorFlow - A curated list of dedicated resources http://tensorflow.org \tStars =  11752 \tWatchers =  11752 \tForks =  1974 \tOpen Issues =  22 \tPrimary Language =  None \tRepo Languages = {} \tRepo Score 97.68051\n",
      "==========\n",
      "Repo Name =  stanford-tensorflow-tutorials \tDescription This repository contains code examples for the Stanford's course: TensorFlow for Deep Learning Research.  \tStars =  6409 \tWatchers =  6409 \tForks =  2903 \tOpen Issues =  69 \tPrimary Language =  Python \tRepo Languages = {'Python': 100.0} \tRepo Score 92.835175\n",
      "==========\n",
      "Repo Name =  tensorflow \tDescription 图解tensorflow 源码 \tStars =  1491 \tWatchers =  1491 \tForks =  467 \tOpen Issues =  1 \tPrimary Language =  None \tRepo Languages = {} \tRepo Score 92.66157\n",
      "==========\n",
      "Repo Name =  DCGAN-tensorflow \tDescription A tensorflow implementation of \"Deep Convolutional Generative Adversarial Networks\" \tStars =  4209 \tWatchers =  4209 \tForks =  1685 \tOpen Issues =  123 \tPrimary Language =  JavaScript \tRepo Languages = {'JavaScript': 50.95, 'Python': 23.02, 'HTML': 15.43, 'CSS': 10.6} \tRepo Score 92.60694\n",
      "==========\n",
      "Repo Name =  facenet \tDescription Face recognition using Tensorflow \tStars =  4686 \tWatchers =  4686 \tForks =  2006 \tOpen Issues =  189 \tPrimary Language =  Python \tRepo Languages = {'Python': 92.81, 'Matlab': 7.19} \tRepo Score 91.77118\n",
      "==========\n",
      "Repo Name =  tensorflow_cookbook \tDescription Code for Tensorflow Machine Learning Cookbook \tStars =  3199 \tWatchers =  3199 \tForks =  1481 \tOpen Issues =  14 \tPrimary Language =  Jupyter Notebook \tRepo Languages = {'Jupyter Notebook': 86.63, 'Python': 13.37} \tRepo Score 90.918785\n",
      "==========\n",
      "Repo Name =  tensorflow_tutorials \tDescription From the basics to slightly more interesting applications of Tensorflow \tStars =  4915 \tWatchers =  4915 \tForks =  1028 \tOpen Issues =  12 \tPrimary Language =  Jupyter Notebook \tRepo Languages = {'Jupyter Notebook': 86.69, 'Python': 13.31} \tRepo Score 89.67762\n",
      "==========\n",
      "Repo Name =  TensorFlow-Tutorials \tDescription TensorFlow Tutorials with YouTube Videos \tStars =  4343 \tWatchers =  4343 \tForks =  2064 \tOpen Issues =  0 \tPrimary Language =  Jupyter Notebook \tRepo Languages = {'Jupyter Notebook': 99.64, 'Python': 0.36} \tRepo Score 89.224335\n",
      "==========\n",
      "Repo Name =  EffectiveTensorflow \tDescription TensorFlow tutorials and best practices. \tStars =  7288 \tWatchers =  7288 \tForks =  716 \tOpen Issues =  0 \tPrimary Language =  None \tRepo Languages = {} \tRepo Score 87.96589\n",
      "==========\n",
      "Repo Name =  tensorflow-wavenet \tDescription A TensorFlow implementation of DeepMind's WaveNet paper \tStars =  3672 \tWatchers =  3672 \tForks =  858 \tOpen Issues =  122 \tPrimary Language =  Python \tRepo Languages = {'Python': 99.55, 'Shell': 0.45} \tRepo Score 87.39114\n",
      "==========\n",
      "Repo Name =  TensorFlow-Tutorials \tDescription Simple tutorials using Google's TensorFlow Framework \tStars =  5412 \tWatchers =  5412 \tForks =  1433 \tOpen Issues =  15 \tPrimary Language =  Jupyter Notebook \tRepo Languages = {'Jupyter Notebook': 94.0, 'Python': 5.94, 'Shell': 0.06} \tRepo Score 87.175964\n",
      "==========\n",
      "Repo Name =  tensorflow \tDescription TensorFlow for R \tStars =  1025 \tWatchers =  1025 \tForks =  278 \tOpen Issues =  31 \tPrimary Language =  R \tRepo Languages = {'R': 100.0} \tRepo Score 86.65774\n",
      "==========\n",
      "Repo Name =  DeepLearningZeroToAll \tDescription TensorFlow Basic Tutorial Labs \tStars =  2418 \tWatchers =  2418 \tForks =  1336 \tOpen Issues =  15 \tPrimary Language =  Jupyter Notebook \tRepo Languages = {'Jupyter Notebook': 65.31, 'Python': 34.69} \tRepo Score 86.577705\n",
      "==========\n",
      "Repo Name =  tensorflow-tutorial \tDescription Example TensorFlow codes and Caicloud TensorFlow as a Service dev environment. \tStars =  1312 \tWatchers =  1312 \tForks =  1131 \tOpen Issues =  35 \tPrimary Language =  Jupyter Notebook \tRepo Languages = {'Jupyter Notebook': 99.15, 'Python': 0.82, 'Shell': 0.03} \tRepo Score 85.97754\n",
      "==========\n",
      "Repo Name =  tensorflow \tDescription 使用谷歌开源的TensorFlow进行一系列的训练实践 \tStars =  649 \tWatchers =  649 \tForks =  300 \tOpen Issues =  4 \tPrimary Language =  Python \tRepo Languages = {'Python': 96.32, 'Protocol Buffer': 3.09, 'Jupyter Notebook': 0.59} \tRepo Score 85.38757\n",
      "==========\n",
      "Repo Name =  caffe-tensorflow \tDescription Caffe models in TensorFlow \tStars =  2167 \tWatchers =  2167 \tForks =  826 \tOpen Issues =  102 \tPrimary Language =  Python \tRepo Languages = {'Python': 100.0} \tRepo Score 85.234604\n",
      "==========\n",
      "Repo Name =  SSD-Tensorflow \tDescription Single Shot MultiBox Detector in TensorFlow \tStars =  1839 \tWatchers =  1839 \tForks =  846 \tOpen Issues =  180 \tPrimary Language =  Jupyter Notebook \tRepo Languages = {'Jupyter Notebook': 74.42, 'Python': 25.58} \tRepo Score 84.84915\n",
      "==========\n",
      "Repo Name =  Tensorflow-101 \tDescription TensorFlow Tutorials \tStars =  1630 \tWatchers =  1630 \tForks =  523 \tOpen Issues =  13 \tPrimary Language =  Jupyter Notebook \tRepo Languages = {'Jupyter Notebook': 99.96, 'Python': 0.04} \tRepo Score 84.600174\n",
      "==========\n",
      "Repo Name =  Tensorflow-Tutorial \tDescription Tensorflow tutorial from basic to hard \tStars =  1708 \tWatchers =  1708 \tForks =  815 \tOpen Issues =  1 \tPrimary Language =  Python \tRepo Languages = {'Python': 100.0} \tRepo Score 84.36726\n",
      "==========\n",
      "Repo Name =  TensorFlow-Book \tDescription Accompanying source code for Machine Learning with TensorFlow. Refer to the book for step-by-step explanations. \tStars =  3872 \tWatchers =  3872 \tForks =  954 \tOpen Issues =  12 \tPrimary Language =  Jupyter Notebook \tRepo Languages = {'Jupyter Notebook': 96.38, 'Python': 3.62} \tRepo Score 83.35463\n",
      "==========\n",
      "Repo Name =  FastMaskRCNN \tDescription Mask RCNN in TensorFlow  \tStars =  2299 \tWatchers =  2299 \tForks =  880 \tOpen Issues =  136 \tPrimary Language =  Python \tRepo Languages = {'Python': 93.46, 'C++': 2.97, 'C': 2.35, 'Cuda': 1.14, 'Makefile': 0.06, 'Shell': 0.02} \tRepo Score 83.29858\n",
      "==========\n",
      "Repo Name =  CapsNet-Tensorflow \tDescription A Tensorflow implementation of CapsNet(Capsules Net) in Hinton's paper Dynamic Routing Between Capsules \tStars =  2829 \tWatchers =  2829 \tForks =  932 \tOpen Issues =  20 \tPrimary Language =  Python \tRepo Languages = {'Python': 98.17, 'R': 1.83} \tRepo Score 82.00451\n",
      "==========\n",
      "Repo Name =  neural-style \tDescription Neural style in TensorFlow! :art: \tStars =  3858 \tWatchers =  3858 \tForks =  1135 \tOpen Issues =  0 \tPrimary Language =  Python \tRepo Languages = {'Python': 100.0} \tRepo Score 81.76342\n",
      "==========\n",
      "Repo Name =  sonnet \tDescription TensorFlow-based neural network library \tStars =  6511 \tWatchers =  6511 \tForks =  869 \tOpen Issues =  5 \tPrimary Language =  Python \tRepo Languages = {'Python': 93.02, 'Jupyter Notebook': 6.86, 'Shell': 0.12} \tRepo Score 81.48367\n",
      "==========\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Repo Name =  Reinforcement-learning-with-tensorflow \tDescription Simple Reinforcement learning tutorials \tStars =  1612 \tWatchers =  1612 \tForks =  1068 \tOpen Issues =  3 \tPrimary Language =  Python \tRepo Languages = {'Python': 100.0} \tRepo Score 81.32195\n",
      "==========\n",
      "Repo Name =  deep-learning-keras-tensorflow \tDescription Introduction to Deep Neural Networks with Keras and Tensorflow \tStars =  2041 \tWatchers =  2041 \tForks =  889 \tOpen Issues =  11 \tPrimary Language =  Jupyter Notebook \tRepo Languages = {'Jupyter Notebook': 99.71, 'Python': 0.29} \tRepo Score 80.80793\n",
      "==========\n"
     ]
    }
   ],
   "source": [
    "# GitHub returns information of only 30 repositories with every request\n",
    "# The Search API Endpoint only allows upto 1000 results, hence the range has been set to 35\n",
    "for page in range(1, 35):\n",
    "\n",
    "    # Building the Search API URL\n",
    "    search_final_url = base_api_url + 'search/repositories?q=' + \\\n",
    "        query + '&page=' + str(page) + '&' + token\n",
    "\n",
    "    # try-except block just incase you set up the range in the above for loop beyond 35\n",
    "    try:\n",
    "        response = requests.get(search_final_url).json()\n",
    "    except:\n",
    "        print(\"Issue with GitHub API, Check your token\")\n",
    "\n",
    "    # Parsing through the response of the search query\n",
    "    for item in response['items']:\n",
    "        # Append to the CSV file\n",
    "        with open(filename, 'a', newline='') as csvfile:\n",
    "            write_to_csv = csv.writer(csvfile, delimiter='|')\n",
    "\n",
    "            repo_name = item['name']\n",
    "            repo_description = item['description']\n",
    "            repo_stars = item['stargazers_count']\n",
    "            repo_watchers = item['watchers_count']\n",
    "            repo_forks = item['forks_count']\n",
    "            repo_issues_count = item['open_issues_count']\n",
    "            repo_main_language = item['language']\n",
    "            repo_license = None\n",
    "            # repo_score is the relevancy score of a repository to the search query\n",
    "            # Reference - https://developer.github.com/v3/search/#ranking-search-results\n",
    "            repo_score = item['score']\n",
    "\n",
    "            # Many Repositories don't have a license, this is to filter them out\n",
    "            if item['license']:\n",
    "                repo_license = item['license']['name']\n",
    "            else:\n",
    "                repo_license = \"NO LICENSE\"\n",
    "\n",
    "            # Just incase, you face any issue with GitHub API Rate Limiting, use the sleep function as a workaround\n",
    "            # Reference - https://developer.github.com/v3/search/#rate-limit\n",
    "\n",
    "            # time.sleep(10)\n",
    "\n",
    "            # Languages URL to access all the languages present in the repository\n",
    "            language_url = item['url'] + '/languages?' + token\n",
    "            language_response = requests.get(language_url).json()\n",
    "\n",
    "            repo_languages = {}\n",
    "\n",
    "            # Calculation for the percentage of all the languages present in the repository\n",
    "            count_value = sum([value for value in language_response.values()])\n",
    "            for key, value in language_response.items():\n",
    "                key_value = round((value / count_value) * 100, 2)\n",
    "                repo_languages[key] = key_value\n",
    "            print(\"Repo Name = \", repo_name, \"\\tDescription\", repo_description, \"\\tStars = \", repo_stars, \"\\tWatchers = \", repo_watchers, \"\\tForks = \", repo_forks,\n",
    "                  \"\\tOpen Issues = \", repo_issues_count, \"\\tPrimary Language = \", repo_main_language, \"\\tRepo Languages =\", repo_languages, '\\tRepo Score', repo_score)\n",
    "\n",
    "            # Write as a row to the CSV file\n",
    "            write_to_csv.writerow([repo_name, repo_description, repo_stars, repo_watchers, repo_forks,\n",
    "                                   repo_license, repo_issues_count, repo_score, repo_main_language, repo_languages])\n",
    "\n",
    "            print('==========')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-02T18:05:43.112805Z",
     "start_time": "2018-06-02T18:05:41.272097Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-02T18:05:45.087277Z",
     "start_time": "2018-06-02T18:05:45.079985Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "notify_time": "5"
 },
 "nbformat": 4,
 "nbformat_minor": 2
}