web_crawler.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "kbzBARQgOd0A",
        "outputId": "61baeabf-c032-4b2e-ad3d-20a5569e2340"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['http://news.iitgn.ac.in', []]\n",
            "['https://iitgn.ac.in/research', []]\n",
            "['https://twitter.com/iitgn', [{'name': 'guest_id', 'value': 'v1%3A166960667628082819', 'domain': '.twitter.com', 'path': '/'}, {'name': 'guest_id_ads', 'value': 'v1%3A166960667628082819', 'domain': '.twitter.com', 'path': '/'}, {'name': 'guest_id_marketing', 'value': 'v1%3A166960667628082819', 'domain': '.twitter.com', 'path': '/'}, {'name': 'personalization_id', 'value': '\"v1_mIZlrzDEN1CLZZYsY9QxBA==\"', 'domain': '.twitter.com', 'path': '/'}]]\n",
            "['https://www.facebook.com/iitgn.official', [{'name': 'datr', 'value': 'FC2EY_cfQjx5z-eaQSCHjBgw', 'domain': '.facebook.com', 'path': '/'}, {'name': 'fr', 'value': '0mf3OMeCqqvG2Fmwq..BjhC0U.Pf.AAA.0.0.BjhC0U.AWUbHEqTjc4', 'domain': '.facebook.com', 'path': '/'}, {'name': 'sb', 'value': 'FC2EY3NdkR0imDsSrmjUWBLd', 'domain': '.facebook.com', 'path': '/'}]]\n",
            "['https://www.youtube.com/user/IITGN1', [{'name': 'GPS', 'value': '1', 'domain': '.youtube.com', 'path': '/'}, {'name': 'VISITOR_INFO1_LIVE', 'value': 'IPUrIFl38TQ', 'domain': '.youtube.com', 'path': '/'}, {'name': 'YSC', 'value': 'Cm2WWsBjLNs', 'domain': '.youtube.com', 'path': '/'}]]\n",
            "['https://legacy.iitgn.ac.in/index_hindi.htm', []]\n",
            "['https://iitgn.ac.in/contact', []]\n",
            "['https://iitgn.ac.in/research', []]\n",
            "['https://www.google.com/a/iitgn.ac.in/ServiceLogin?service=mail&passive=true&rm=false&continue=https://mail.google.com/a/iitgn.ac.in/&ss=1&ltmpl=default&ltmplcache=2', []]\n",
            "['https://sites.google.com/iitgn.ac.in/iws/', []]\n",
            "['http://ims.iitgn.ac.in/', []]\n",
            "['http://campus.iitgn.ac.in/', []]\n",
            "['https://sites.google.com/a/iitgn.ac.in/greeniitgn/about', []]\n",
            "['http://initiatives.iitgn.ac.in/uba/', []]\n",
            "['https://iitgn.ac.in/holiday/2022', []]\n",
            "['https://iitgn.ac.in/about/nirf/nirf-2022', []]\n",
            "['https://iitgn.ac.in/about/ariia', []]\n",
            "['https://iitgn.ac.in/tender', []]\n",
            "['https://campus.iitgn.ac.in/facility/', []]\n",
            "['http://www.iitgn.ac.in/student/lifeoncampus/', []]\n",
            "['https://library.iitgn.ac.in/', []]\n",
            "['https://students.iitgn.ac.in/student_affairs', []]\n",
            "['https://alumni.iitgn.ac.in/', [{'name': 'tz', 'value': 'Asia%2FKolkata', 'domain': 'alumni.iitgn.ac.in', 'path': '/'}]]\n",
            "['https://iitgn.ac.in/students/scholarships', []]\n",
            "['http://sites.iitgn.ac.in/cds/?q=cds/', []]\n",
            "['https://iitgn.ac.in/icc', []]\n",
            "['http://sites.iitgn.ac.in/counselling', []]\n",
            "['https://initiatives.iitgn.ac.in/ldi', []]\n",
            "['https://iitgn.ac.in/rti', []]\n",
            "['http://news.iitgn.ac.in', []]\n",
            "['https://iitgn.ac.in/research', []]\n",
            "['https://twitter.com/iitgn', [{'name': 'guest_id', 'value': 'v1%3A166960670362655071', 'domain': '.twitter.com', 'path': '/'}, {'name': 'guest_id_ads', 'value': 'v1%3A166960670362655071', 'domain': '.twitter.com', 'path': '/'}, {'name': 'guest_id_marketing', 'value': 'v1%3A166960670362655071', 'domain': '.twitter.com', 'path': '/'}, {'name': 'personalization_id', 'value': '\"v1_XG8/9IWZ+VBruzwPugLTNQ==\"', 'domain': '.twitter.com', 'path': '/'}]]\n",
            "['https://www.facebook.com/iitgn.official', [{'name': 'datr', 'value': 'Ly2EYwY4-qde5DXPDjkoVBFC', 'domain': '.facebook.com', 'path': '/'}, {'name': 'fr', 'value': '0Pem0xCfutufCQWQm..BjhC0v.S6.AAA.0.0.BjhC0w.AWWE9LY54uU', 'domain': '.facebook.com', 'path': '/'}, {'name': 'sb', 'value': 'MC2EY7UqDeT5lB1Ct2swIMpR', 'domain': '.facebook.com', 'path': '/'}]]\n",
            "['https://www.youtube.com/user/IITGN1', [{'name': 'GPS', 'value': '1', 'domain': '.youtube.com', 'path': '/'}, {'name': 'VISITOR_INFO1_LIVE', 'value': 'IyA3_-XbQwo', 'domain': '.youtube.com', 'path': '/'}, {'name': 'YSC', 'value': 'YCAWndfGOFA', 'domain': '.youtube.com', 'path': '/'}]]\n",
            "['https://legacy.iitgn.ac.in/index_hindi.htm', []]\n",
            "['https://iitgn.ac.in/contact', []]\n",
            "['https://iitgn.ac.in/research', []]\n",
            "['https://www.google.com/a/iitgn.ac.in/ServiceLogin?service=mail&passive=true&rm=false&continue=https://mail.google.com/a/iitgn.ac.in/&ss=1&ltmpl=default&ltmplcache=2', []]\n",
            "['https://sites.google.com/iitgn.ac.in/iws/', []]\n",
            "['http://ims.iitgn.ac.in/', []]\n",
            "['http://campus.iitgn.ac.in/', []]\n",
            "['https://sites.google.com/a/iitgn.ac.in/greeniitgn/about', []]\n",
            "['http://initiatives.iitgn.ac.in/uba/', []]\n",
            "['https://iitgn.ac.in/holiday/2022', []]\n",
            "['https://iitgn.ac.in/about/nirf/nirf-2022', []]\n",
            "['https://iitgn.ac.in/about/ariia', []]\n",
            "['https://iitgn.ac.in/tender', []]\n",
            "['https://campus.iitgn.ac.in/facility/', []]\n",
            "['http://www.iitgn.ac.in/student/lifeoncampus/', []]\n",
            "['https://library.iitgn.ac.in/', []]\n",
            "['https://students.iitgn.ac.in/student_affairs', []]\n",
            "['https://alumni.iitgn.ac.in/', [{'name': 'tz', 'value': 'Asia%2FKolkata', 'domain': 'alumni.iitgn.ac.in', 'path': '/'}]]\n",
            "['https://iitgn.ac.in/students/scholarships', []]\n",
            "['http://sites.iitgn.ac.in/cds/?q=cds/', []]\n",
            "['https://iitgn.ac.in/icc', []]\n",
            "['http://sites.iitgn.ac.in/counselling', []]\n",
            "['https://initiatives.iitgn.ac.in/ldi', []]\n",
            "['https://iitgn.ac.in/rti', []]\n",
            "['http://news.iitgn.ac.in', []]\n",
            "['https://iitgn.ac.in/research', []]\n",
            "['https://twitter.com/iitgn', [{'name': 'guest_id', 'value': 'v1%3A166960672886604575', 'domain': '.twitter.com', 'path': '/'}, {'name': 'guest_id_ads', 'value': 'v1%3A166960672886604575', 'domain': '.twitter.com', 'path': '/'}, {'name': 'guest_id_marketing', 'value': 'v1%3A166960672886604575', 'domain': '.twitter.com', 'path': '/'}, {'name': 'personalization_id', 'value': '\"v1_hNBzY2VqukwwisjAgnttpQ==\"', 'domain': '.twitter.com', 'path': '/'}]]\n",
            "['https://www.facebook.com/iitgn.official', [{'name': 'datr', 'value': 'SS2EY6VZqc7gJaKvWJsFqAqc', 'domain': '.facebook.com', 'path': '/'}, {'name': 'fr', 'value': '0dZNlTsuJWv1DZ0DD..BjhC1J.Kk.AAA.0.0.BjhC1J.AWUl9x_YbkA', 'domain': '.facebook.com', 'path': '/'}, {'name': 'sb', 'value': 'SS2EY2NMM42t3Q0u6_m_VUtu', 'domain': '.facebook.com', 'path': '/'}]]\n",
            "['https://www.youtube.com/user/IITGN1', [{'name': 'GPS', 'value': '1', 'domain': '.youtube.com', 'path': '/'}, {'name': 'VISITOR_INFO1_LIVE', 'value': 'rtodsvLvLWo', 'domain': '.youtube.com', 'path': '/'}, {'name': 'YSC', 'value': '2rLa15AGpec', 'domain': '.youtube.com', 'path': '/'}]]\n",
            "['https://legacy.iitgn.ac.in/index_hindi.htm', []]\n",
            "['https://iitgn.ac.in/contact', []]\n",
            "['https://iitgn.ac.in/research', []]\n",
            "['https://www.google.com/a/iitgn.ac.in/ServiceLogin?service=mail&passive=true&rm=false&continue=https://mail.google.com/a/iitgn.ac.in/&ss=1&ltmpl=default&ltmplcache=2', []]\n",
            "['https://sites.google.com/iitgn.ac.in/iws/', []]\n",
            "['http://ims.iitgn.ac.in/', []]\n",
            "['http://campus.iitgn.ac.in/', []]\n",
            "['https://sites.google.com/a/iitgn.ac.in/greeniitgn/about', []]\n",
            "['http://initiatives.iitgn.ac.in/uba/', []]\n",
            "['https://iitgn.ac.in/holiday/2022', []]\n",
            "['https://iitgn.ac.in/about/nirf/nirf-2022', []]\n",
            "['https://iitgn.ac.in/about/ariia', []]\n",
            "['https://iitgn.ac.in/tender', []]\n",
            "['https://campus.iitgn.ac.in/facility/', []]\n",
            "['http://www.iitgn.ac.in/student/lifeoncampus/', []]\n",
            "['https://library.iitgn.ac.in/', []]\n",
            "['https://students.iitgn.ac.in/student_affairs', []]\n",
            "['https://alumni.iitgn.ac.in/', [{'name': 'tz', 'value': 'Asia%2FKolkata', 'domain': 'alumni.iitgn.ac.in', 'path': '/'}]]\n",
            "['https://iitgn.ac.in/students/scholarships', []]\n",
            "['http://sites.iitgn.ac.in/cds/?q=cds/', []]\n",
            "['https://iitgn.ac.in/icc', []]\n",
            "['http://sites.iitgn.ac.in/counselling', []]\n",
            "['https://initiatives.iitgn.ac.in/ldi', []]\n",
            "['https://iitgn.ac.in/rti', []]\n",
            "['http://news.iitgn.ac.in', []]\n",
            "['https://iitgn.ac.in/research', []]\n",
            "['https://twitter.com/iitgn', [{'name': 'guest_id', 'value': 'v1%3A166960675602311015', 'domain': '.twitter.com', 'path': '/'}, {'name': 'guest_id_ads', 'value': 'v1%3A166960675602311015', 'domain': '.twitter.com', 'path': '/'}, {'name': 'guest_id_marketing', 'value': 'v1%3A166960675602311015', 'domain': '.twitter.com', 'path': '/'}, {'name': 'personalization_id', 'value': '\"v1_8dVSV4nRzAwoKF6N0aL+AQ==\"', 'domain': '.twitter.com', 'path': '/'}]]\n",
            "['https://www.facebook.com/iitgn.official', [{'name': 'datr', 'value': 'ZC2EY3LXmX-2ZczfkquM2tci', 'domain': '.facebook.com', 'path': '/'}, {'name': 'fr', 'value': '0fCjiZ1qYQYONzl4R..BjhC1k.un.AAA.0.0.BjhC1k.AWVcdtBnUeM', 'domain': '.facebook.com', 'path': '/'}, {'name': 'sb', 'value': 'ZC2EY8SnJ-Gmw_Ws2Q7YGycg', 'domain': '.facebook.com', 'path': '/'}]]\n",
            "['https://www.youtube.com/user/IITGN1', [{'name': 'GPS', 'value': '1', 'domain': '.youtube.com', 'path': '/'}, {'name': 'VISITOR_INFO1_LIVE', 'value': '-88-2fb2W0U', 'domain': '.youtube.com', 'path': '/'}, {'name': 'YSC', 'value': '6ZTBbTao6lk', 'domain': '.youtube.com', 'path': '/'}]]\n",
            "['https://legacy.iitgn.ac.in/index_hindi.htm', []]\n",
            "['https://iitgn.ac.in/contact', []]\n",
            "['https://iitgn.ac.in/research', []]\n",
            "['https://www.google.com/a/iitgn.ac.in/ServiceLogin?service=mail&passive=true&rm=false&continue=https://mail.google.com/a/iitgn.ac.in/&ss=1&ltmpl=default&ltmplcache=2', []]\n",
            "['https://sites.google.com/iitgn.ac.in/iws/', []]\n",
            "['http://ims.iitgn.ac.in/', []]\n",
            "['http://campus.iitgn.ac.in/', []]\n",
            "['https://sites.google.com/a/iitgn.ac.in/greeniitgn/about', []]\n",
            "['http://initiatives.iitgn.ac.in/uba/', []]\n",
            "['https://iitgn.ac.in/holiday/2022', []]\n",
            "['https://iitgn.ac.in/about/nirf/nirf-2022', []]\n",
            "['https://iitgn.ac.in/about/ariia', []]\n",
            "['https://iitgn.ac.in/tender', []]\n",
            "['https://campus.iitgn.ac.in/facility/', []]\n",
            "['http://www.iitgn.ac.in/student/lifeoncampus/', []]\n",
            "['https://library.iitgn.ac.in/', []]\n",
            "['https://students.iitgn.ac.in/student_affairs', []]\n",
            "['https://alumni.iitgn.ac.in/', [{'name': 'tz', 'value': 'Asia%2FKolkata', 'domain': 'alumni.iitgn.ac.in', 'path': '/'}]]\n",
            "['https://iitgn.ac.in/students/scholarships', []]\n",
            "['http://sites.iitgn.ac.in/cds/?q=cds/', []]\n",
            "['https://iitgn.ac.in/icc', []]\n",
            "['http://sites.iitgn.ac.in/counselling', []]\n",
            "['https://initiatives.iitgn.ac.in/ldi', []]\n",
            "['https://iitgn.ac.in/rti', []]\n",
            "['http://news.iitgn.ac.in', []]\n",
            "['https://iitgn.ac.in/research', []]\n",
            "['https://twitter.com/iitgn', [{'name': 'guest_id', 'value': 'v1%3A166960678235653984', 'domain': '.twitter.com', 'path': '/'}, {'name': 'guest_id_ads', 'value': 'v1%3A166960678235653984', 'domain': '.twitter.com', 'path': '/'}, {'name': 'guest_id_marketing', 'value': 'v1%3A166960678235653984', 'domain': '.twitter.com', 'path': '/'}, {'name': 'personalization_id', 'value': '\"v1_pU5rEZiuUmrK9fok9HvSbA==\"', 'domain': '.twitter.com', 'path': '/'}]]\n",
            "['https://www.facebook.com/iitgn.official', [{'name': 'datr', 'value': 'fi2EY5_YoPyaxXEFoHoUQfju', 'domain': '.facebook.com', 'path': '/'}, {'name': 'fr', 'value': '06ehSrYEqpZqVKwdX..BjhC1-.Vf.AAA.0.0.BjhC1-.AWVPIu6K9o8', 'domain': '.facebook.com', 'path': '/'}, {'name': 'sb', 'value': 'fi2EY86W3WM2_q2apIPTVSA7', 'domain': '.facebook.com', 'path': '/'}]]\n",
            "['https://www.youtube.com/user/IITGN1', [{'name': 'GPS', 'value': '1', 'domain': '.youtube.com', 'path': '/'}, {'name': 'VISITOR_INFO1_LIVE', 'value': 'RCnhuymGgNY', 'domain': '.youtube.com', 'path': '/'}, {'name': 'YSC', 'value': 'UWlo8olSv68', 'domain': '.youtube.com', 'path': '/'}]]\n",
            "['https://legacy.iitgn.ac.in/index_hindi.htm', []]\n",
            "['https://iitgn.ac.in/contact', []]\n",
            "['https://iitgn.ac.in/research', []]\n",
            "['https://www.google.com/a/iitgn.ac.in/ServiceLogin?service=mail&passive=true&rm=false&continue=https://mail.google.com/a/iitgn.ac.in/&ss=1&ltmpl=default&ltmplcache=2', []]\n",
            "['https://sites.google.com/iitgn.ac.in/iws/', []]\n",
            "['http://ims.iitgn.ac.in/', []]\n",
            "['http://campus.iitgn.ac.in/', []]\n",
            "['https://sites.google.com/a/iitgn.ac.in/greeniitgn/about', []]\n",
            "['http://initiatives.iitgn.ac.in/uba/', []]\n",
            "['https://iitgn.ac.in/holiday/2022', []]\n",
            "['https://iitgn.ac.in/about/nirf/nirf-2022', []]\n",
            "['https://iitgn.ac.in/about/ariia', []]\n",
            "['https://iitgn.ac.in/tender', []]\n",
            "['https://campus.iitgn.ac.in/facility/', []]\n",
            "['http://www.iitgn.ac.in/student/lifeoncampus/', []]\n",
            "['https://library.iitgn.ac.in/', []]\n",
            "['https://students.iitgn.ac.in/student_affairs', []]\n",
            "['https://alumni.iitgn.ac.in/', [{'name': 'tz', 'value': 'Asia%2FKolkata', 'domain': 'alumni.iitgn.ac.in', 'path': '/'}]]\n",
            "['https://iitgn.ac.in/students/scholarships', []]\n",
            "['http://sites.iitgn.ac.in/cds/?q=cds/', []]\n",
            "['https://iitgn.ac.in/icc', []]\n",
            "['http://sites.iitgn.ac.in/counselling', []]\n",
            "['https://initiatives.iitgn.ac.in/ldi', []]\n",
            "['https://iitgn.ac.in/rti', []]\n",
            "['http://news.iitgn.ac.in', []]\n",
            "['https://iitgn.ac.in/research', []]\n",
            "['https://twitter.com/iitgn', [{'name': 'guest_id', 'value': 'v1%3A166960680729432651', 'domain': '.twitter.com', 'path': '/'}, {'name': 'guest_id_ads', 'value': 'v1%3A166960680729432651', 'domain': '.twitter.com', 'path': '/'}, {'name': 'guest_id_marketing', 'value': 'v1%3A166960680729432651', 'domain': '.twitter.com', 'path': '/'}, {'name': 'personalization_id', 'value': '\"v1_5b/lj9oxLl84N6iyQWxWrQ==\"', 'domain': '.twitter.com', 'path': '/'}]]\n",
            "['https://www.facebook.com/iitgn.official', [{'name': 'datr', 'value': 'ly2EY_gcw5s7ldcjUlOB2Lne', 'domain': '.facebook.com', 'path': '/'}, {'name': 'fr', 'value': '0s2Lb9yW1TCw1VUgw..BjhC2X.Rf.AAA.0.0.BjhC2X.AWWEpbvRwMw', 'domain': '.facebook.com', 'path': '/'}, {'name': 'sb', 'value': 'ly2EY6T7JMD6SMaUdnnfHPsb', 'domain': '.facebook.com', 'path': '/'}]]\n",
            "['https://www.youtube.com/user/IITGN1', [{'name': 'GPS', 'value': '1', 'domain': '.youtube.com', 'path': '/'}, {'name': 'VISITOR_INFO1_LIVE', 'value': '9VXI6lSkBvk', 'domain': '.youtube.com', 'path': '/'}, {'name': 'YSC', 'value': 'lWQTLQVovDc', 'domain': '.youtube.com', 'path': '/'}]]\n",
            "['https://legacy.iitgn.ac.in/index_hindi.htm', []]\n",
            "['https://iitgn.ac.in/contact', []]\n",
            "['https://iitgn.ac.in/research', []]\n",
            "['https://www.google.com/a/iitgn.ac.in/ServiceLogin?service=mail&passive=true&rm=false&continue=https://mail.google.com/a/iitgn.ac.in/&ss=1&ltmpl=default&ltmplcache=2', []]\n",
            "['https://sites.google.com/iitgn.ac.in/iws/', []]\n",
            "['http://ims.iitgn.ac.in/', []]\n",
            "['http://campus.iitgn.ac.in/', []]\n",
            "['https://sites.google.com/a/iitgn.ac.in/greeniitgn/about', []]\n",
            "['http://initiatives.iitgn.ac.in/uba/', []]\n",
            "['https://iitgn.ac.in/holiday/2022', []]\n",
            "['https://iitgn.ac.in/about/nirf/nirf-2022', []]\n",
            "['https://iitgn.ac.in/about/ariia', []]\n",
            "['https://iitgn.ac.in/tender', []]\n",
            "['https://campus.iitgn.ac.in/facility/', []]\n",
            "['http://www.iitgn.ac.in/student/lifeoncampus/', []]\n",
            "['https://library.iitgn.ac.in/', []]\n",
            "['https://students.iitgn.ac.in/student_affairs', []]\n",
            "['https://alumni.iitgn.ac.in/', [{'name': 'tz', 'value': 'Asia%2FKolkata', 'domain': 'alumni.iitgn.ac.in', 'path': '/'}]]\n",
            "['https://iitgn.ac.in/students/scholarships', []]\n",
            "['http://sites.iitgn.ac.in/cds/?q=cds/', []]\n",
            "['https://iitgn.ac.in/icc', []]\n",
            "['http://sites.iitgn.ac.in/counselling', []]\n",
            "['https://initiatives.iitgn.ac.in/ldi', []]\n",
            "['https://iitgn.ac.in/rti', []]\n",
            "['http://news.iitgn.ac.in', []]\n",
            "['https://iitgn.ac.in/research', []]\n",
            "['https://twitter.com/iitgn', [{'name': 'guest_id', 'value': 'v1%3A166960683219162617', 'domain': '.twitter.com', 'path': '/'}, {'name': 'guest_id_ads', 'value': 'v1%3A166960683219162617', 'domain': '.twitter.com', 'path': '/'}, {'name': 'guest_id_marketing', 'value': 'v1%3A166960683219162617', 'domain': '.twitter.com', 'path': '/'}, {'name': 'personalization_id', 'value': '\"v1_3W8Ih1DcLvLJabtXT7+49Q==\"', 'domain': '.twitter.com', 'path': '/'}]]\n",
            "['https://www.facebook.com/iitgn.official', [{'name': 'datr', 'value': 'sC2EY3DTK0FM-WqifUvVfYS1', 'domain': '.facebook.com', 'path': '/'}, {'name': 'fr', 'value': '0bhRphdCBH4WQcKvw..BjhC2w.n_.AAA.0.0.BjhC2w.AWUqThvIuRI', 'domain': '.facebook.com', 'path': '/'}, {'name': 'sb', 'value': 'sC2EYynWXHQwfVT2PbGw2G9e', 'domain': '.facebook.com', 'path': '/'}]]\n",
            "['https://www.youtube.com/user/IITGN1', [{'name': 'GPS', 'value': '1', 'domain': '.youtube.com', 'path': '/'}, {'name': 'VISITOR_INFO1_LIVE', 'value': 'jJul2Eu6TuE', 'domain': '.youtube.com', 'path': '/'}, {'name': 'YSC', 'value': 'BbnmRXvoK-I', 'domain': '.youtube.com', 'path': '/'}]]\n",
            "['https://legacy.iitgn.ac.in/index_hindi.htm', []]\n",
            "['https://iitgn.ac.in/contact', []]\n",
            "['https://iitgn.ac.in/research', []]\n",
            "['https://www.google.com/a/iitgn.ac.in/ServiceLogin?service=mail&passive=true&rm=false&continue=https://mail.google.com/a/iitgn.ac.in/&ss=1&ltmpl=default&ltmplcache=2', []]\n",
            "['https://sites.google.com/iitgn.ac.in/iws/', []]\n",
            "['http://ims.iitgn.ac.in/', []]\n",
            "['http://campus.iitgn.ac.in/', []]\n",
            "['https://sites.google.com/a/iitgn.ac.in/greeniitgn/about', []]\n",
            "['http://initiatives.iitgn.ac.in/uba/', []]\n",
            "['https://iitgn.ac.in/holiday/2022', []]\n",
            "['https://iitgn.ac.in/about/nirf/nirf-2022', []]\n",
            "['https://iitgn.ac.in/about/ariia', []]\n",
            "['https://iitgn.ac.in/tender', []]\n",
            "['https://campus.iitgn.ac.in/facility/', []]\n",
            "['http://www.iitgn.ac.in/student/lifeoncampus/', []]\n",
            "['https://library.iitgn.ac.in/', []]\n",
            "['https://students.iitgn.ac.in/student_affairs', []]\n",
            "['https://alumni.iitgn.ac.in/', [{'name': 'tz', 'value': 'Asia%2FKolkata', 'domain': 'alumni.iitgn.ac.in', 'path': '/'}]]\n",
            "['https://iitgn.ac.in/students/scholarships', []]\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    376\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Python 2.7, use buffering of HTTP responses\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 377\u001b[0;31m                 \u001b[0mhttplib_response\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuffering\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    378\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Python 3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: getresponse() got an unexpected keyword argument 'buffering'",
            "\nDuring handling of the above exception, another exception occurred:\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-2-06a6611d5309>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     30\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mlink\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlinks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m             \u001b[0mme\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlink\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m             \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlink\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     33\u001b[0m             \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'name'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'value'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'domain'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdomain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'path'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m}\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcookies\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m             \u001b[0mj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mlink\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/requests/api.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(url, params, **kwargs)\u001b[0m\n\u001b[1;32m     74\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m     \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetdefault\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'allow_redirects'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 76\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'get'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     77\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     78\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/requests/api.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m     59\u001b[0m     \u001b[0;31m# cases, and look like a memory leak in others.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0msessions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/requests/sessions.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    528\u001b[0m         }\n\u001b[1;32m    529\u001b[0m         \u001b[0msend_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msettings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 530\u001b[0;31m         \u001b[0mresp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0msend_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    531\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    532\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/requests/sessions.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    663\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    664\u001b[0m         \u001b[0;31m# Resolve redirects if allowed.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 665\u001b[0;31m         \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mresp\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mresp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgen\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mallow_redirects\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    666\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    667\u001b[0m         \u001b[0;31m# Shuffle things around if there's history.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/requests/sessions.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    663\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    664\u001b[0m         \u001b[0;31m# Resolve redirects if allowed.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 665\u001b[0;31m         \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mresp\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mresp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgen\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mallow_redirects\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    666\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    667\u001b[0m         \u001b[0;31m# Shuffle things around if there's history.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/requests/sessions.py\u001b[0m in \u001b[0;36mresolve_redirects\u001b[0;34m(self, resp, req, stream, timeout, verify, cert, proxies, yield_requests, **adapter_kwargs)\u001b[0m\n\u001b[1;32m    243\u001b[0m                     \u001b[0mproxies\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mproxies\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    244\u001b[0m                     \u001b[0mallow_redirects\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 245\u001b[0;31m                     \u001b[0;34m**\u001b[0m\u001b[0madapter_kwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    246\u001b[0m                 )\n\u001b[1;32m    247\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/requests/sessions.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    641\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    642\u001b[0m         \u001b[0;31m# Send the request\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 643\u001b[0;31m         \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0madapter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    644\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    645\u001b[0m         \u001b[0;31m# Total elapsed time of the request (approximately)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/requests/adapters.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    447\u001b[0m                     \u001b[0mdecode_content\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    448\u001b[0m                     \u001b[0mretries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_retries\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 449\u001b[0;31m                     \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    450\u001b[0m                 )\n\u001b[1;32m    451\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    598\u001b[0m                                                   \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout_obj\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    599\u001b[0m                                                   \u001b[0mbody\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 600\u001b[0;31m                                                   chunked=chunked)\n\u001b[0m\u001b[1;32m    601\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    602\u001b[0m             \u001b[0;31m# If we're going to release the connection in ``finally:``, then\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    378\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Python 3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    379\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 380\u001b[0;31m                     \u001b[0mhttplib_response\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    381\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    382\u001b[0m                     \u001b[0;31m# Remove the TypeError from the exception chain in Python 3;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/http/client.py\u001b[0m in \u001b[0;36mgetresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1371\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1372\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1373\u001b[0;31m                 \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbegin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1374\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mConnectionError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1375\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/http/client.py\u001b[0m in \u001b[0;36mbegin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    317\u001b[0m         \u001b[0;31m# read until we get a non-100 response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    318\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 319\u001b[0;31m             \u001b[0mversion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreason\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    320\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mCONTINUE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    321\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/http/client.py\u001b[0m in \u001b[0;36m_read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    278\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    279\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 280\u001b[0;31m         \u001b[0mline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_MAXLINE\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"iso-8859-1\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    281\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0m_MAXLINE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    282\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mLineTooLong\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"status line\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/socket.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    587\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    588\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 589\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    590\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    591\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_timeout_occurred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/ssl.py\u001b[0m in \u001b[0;36mrecv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m   1069\u001b[0m                   \u001b[0;34m\"non-zero flags not allowed in calls to recv_into() on %s\"\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1070\u001b[0m                   self.__class__)\n\u001b[0;32m-> 1071\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnbytes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1072\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1073\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnbytes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/ssl.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m    927\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    928\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mbuffer\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 929\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    930\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    931\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "import re\n",
        "import time\n",
        "from urllib3.exceptions import InsecureRequestWarning\n",
        "from urllib3 import disable_warnings\n",
        "\n",
        "\n",
        "headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}\n",
        "\n",
        "source_code = requests.get('https://iitgn.ac.in/')\n",
        "soup = BeautifulSoup(source_code.content, 'lxml')\n",
        "data = []\n",
        "links = []\n",
        "cookies = []\n",
        "me = 'gt'\n",
        "def remove_duplicates(l): # remove duplicates and unURL string\n",
        "    for item in l:\n",
        "        match = re.search(\"(?P<url>https?://[^\\s]+)\", item)\n",
        "        if match is not None:\n",
        "            links.append((match.group(\"url\")))\n",
        "\n",
        "\n",
        "for link in soup.find_all('a', href=True):\n",
        "    data.append(str(link.get('href')))\n",
        "flag = True\n",
        "remove_duplicates(data)\n",
        "while flag:\n",
        "    try:\n",
        "        for link in links:\n",
        "            me = link \n",
        "            response = requests.get(link, headers=headers)\n",
        "            h = [{'name': c.name, 'value': c.value, 'domain': c.domain, 'path': c.path} for c in response.cookies]\n",
        "            j = [link, h]\n",
        "            print(j)\n",
        "            cookies.append(j)\n",
        "            \n",
        "            for j in soup.find_all('a', href=True):\n",
        "                temp = []\n",
        "                disable_warnings(InsecureRequestWarning)\n",
        "                source_code = requests.get(link)\n",
        "                soup = BeautifulSoup(source_code.content, 'lxml')\n",
        "                temp.append(str(j.get('href')))\n",
        "                remove_duplicates(temp)\n",
        "\n",
        "                if len(links) > 500: # set limitation to number of URLs\n",
        "                    break\n",
        "            if len(links) > 500:\n",
        "                break\n",
        "        if len(links) > 500:\n",
        "            break\n",
        "    except Exception as e:\n",
        "        if len(links) > 500:\n",
        "            break\n",
        "\n",
        "for url in links:\n",
        "  print(url)\n",
        "print(len(links))\n",
        "print(cookies)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}