Lectures/L-NoSQL-ES/02-Load_data_ES.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T22:22:28.607644Z",
     "start_time": "2020-11-18T22:22:28.602362Z"
    }
   },
   "outputs": [],
   "source": [
    "# !pip install --user xmltodict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T11:07:53.172264Z",
     "start_time": "2021-05-03T11:07:48.999203Z"
    }
   },
   "outputs": [],
   "source": [
    "import random\n",
    "import pytz\n",
    "import shapely\n",
    "from shapely.geometry import Point\n",
    "import elasticsearch\n",
    "from datetime import datetime\n",
    "import time\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T11:07:53.942510Z",
     "start_time": "2021-05-03T11:07:53.934311Z"
    }
   },
   "outputs": [],
   "source": [
    "# define mapping\n",
    "mapping = {\n",
    "        \"mappings\": {\n",
    "            #\"mydocs\": {\n",
    "                \"properties\": {\n",
    "                    \"Date\": {\n",
    "                     \"type\": \"date\"\n",
    "                    },\n",
    "                    \"Destination\": {\n",
    "                     \"type\": \"geo_point\"\n",
    "                    },\n",
    "                    \"Origin\": {\n",
    "                      \"type\": \"geo_point\"\n",
    "                    },\n",
    "                    \"Title\": {\n",
    "                      \"type\": \"keyword\"\n",
    "                    },\n",
    "                }\n",
    "            #}\n",
    "       }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T12:52:56.207160Z",
     "start_time": "2021-05-03T12:52:56.199718Z"
    }
   },
   "outputs": [],
   "source": [
    "# load Elasticsearch\n",
    "try:\n",
    "    es = elasticsearch.Elasticsearch([\"kooplex-fiek.elte.hu\"],port=9100)\n",
    "except:\n",
    "    print( \"unable to es\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T12:56:36.116510Z",
     "start_time": "2021-05-03T12:56:35.348638Z"
    }
   },
   "outputs": [],
   "source": [
    "# create index mass_traffic\n",
    "try:\n",
    "    response = es.indices.create(index=\"random_point2\",body=mapping, ignore=400)\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T12:56:37.909179Z",
     "start_time": "2021-05-03T12:56:37.874554Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'acknowledged': True, 'shards_acknowledged': True, 'index': 'random_point2'}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T22:22:52.917398Z",
     "start_time": "2020-11-18T22:22:46.984677Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Date': '2020-11-18T22:22:46.992122Z', 'Destination': {'lat': 2.446666040740263, 'lon': 39.691025283524425}, 'Origin': {'lat': -40.997995434117186, 'lon': 33.4908122609994}, 'Title': 'flight1, flight2, flight3'}\n",
      "------------------------------------------\n",
      "{'_index': 'random_point', '_type': '_doc', '_id': '2', '_version': 4, 'result': 'updated', '_shards': {'total': 2, 'successful': 1, 'failed': 0}, '_seq_no': 245, '_primary_term': 1}\n",
      "all done, starting again soon\n",
      "{'Date': '2020-11-18T22:22:46.992122Z', 'Destination': {'lat': 15.635578954524437, 'lon': 40.177133412237296}, 'Origin': {'lat': -15.880408809584328, 'lon': 26.700175268293563}, 'Title': 'flight1, flight2, flight3'}\n",
      "------------------------------------------\n",
      "{'_index': 'random_point', '_type': '_doc', '_id': '3', '_version': 3, 'result': 'updated', '_shards': {'total': 2, 'successful': 1, 'failed': 0}, '_seq_no': 246, '_primary_term': 1}\n",
      "all done, starting again soon\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-4b1b012e76e4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     27\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m \u001b[0;34m\"------------------------------------------\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m \u001b[0;34m\"all done, starting again soon\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m     \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     30\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mid\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m         \u001b[0mid\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "now = pytz.utc.localize(datetime.utcnow())\n",
    "id=1\n",
    "latr1, latr2, lonr1, lonr2 = [random.uniform(-10,10) for i in range(4)]\n",
    "while True:\n",
    "    try:\n",
    "        new_body = {}\n",
    "        new_body[\"Date\"] =  datetime.strftime(now,'%Y-%m-%dT%H:%M:%S.%fZ')\n",
    "        p = Point((random.uniform(0.2+latr1,23+latr1), random.uniform(45.5+lonr1,49+lonr1))) \n",
    "        new_body[\"Destination\"] = { \"lat\": p.x, 'lon': p.y}                \n",
    "        p = Point((random.uniform(-43+latr2,-10+latr2), random.uniform(25.5+lonr2,39+lonr2))) \n",
    "        new_body[\"Origin\"] = {\"lat\": p.x, 'lon': p.y}              \n",
    "\n",
    "        new_body[\"Title\"] = random.choice(['flight1, flight2, flight3'])\n",
    "        id+=1\n",
    "\n",
    "\n",
    "        print(new_body)\n",
    "        res = es.index(index=\"random_point\", doc_type=\"_doc\", id=id, body=new_body)\n",
    "        print( \"------------------------------------------\")\n",
    "        print(res)\n",
    "    except Exception as e:\n",
    "        print( \"------------------------------------------\")\n",
    "        print(\"unable to index in elasticsearch\")\n",
    "        print( \"------------------------------------------\")\n",
    "        error = \"{0}: {1}\".format(id,e)\n",
    "        print( error)\n",
    "        print( \"------------------------------------------\")\n",
    "    print( \"all done, starting again soon\")\n",
    "    time.sleep(5)\n",
    "    if id==30:\n",
    "        id=1\n",
    "        latr1, latr2, lonr1, lonr2 = random(4)\n",
    "        latr1, latr2, lonr1, lonr2 = [random.uniform(-10,10) for i in range(4)]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Queries DSL\n",
    "\n",
    "GET random_point/_search\n",
    "{\n",
    "   \"query\": {\n",
    "    \"bool\": {\n",
    "      \"must\": {\n",
    "        \"match_all\": {}\n",
    "      },\n",
    "      \"filter\": {\n",
    "        \"geo_bounding_box\": {\n",
    "          \"Origin\": {\n",
    "            \"top_left\": {\n",
    "              \"lat\": -28.6603547,\n",
    "              \"lon\": 0.2268691\n",
    "            },\n",
    "            \"bottom_right\": {\n",
    "              \"lat\": -48.3658642,\n",
    "              \"lon\": -26.9556643\n",
    "            }\n",
    "          }\n",
    "        }\n",
    "      }\n",
    "    }\n",
    "  }\n",
    "}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T14:39:41.561136Z",
     "start_time": "2021-05-03T14:39:41.518878Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'took': 1,\n",
       " 'timed_out': False,\n",
       " '_shards': {'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0},\n",
       " 'hits': {'total': {'value': 30, 'relation': 'eq'},\n",
       "  'max_score': 1.0,\n",
       "  'hits': [{'_index': 'random_point',\n",
       "    '_type': '_doc',\n",
       "    '_id': '2',\n",
       "    '_score': 1.0,\n",
       "    '_source': {'Date': '2020-11-19T09:42:49.600762Z',\n",
       "     'Destination': {'lat': 14.128594028987003, 'lon': 44.258336663067276},\n",
       "     'Origin': {'lat': -18.99753070156272, 'lon': 18.649682090504445},\n",
       "     'Title': 'flight1, flight2, flight3'}},\n",
       "   {'_index': 'random_point',\n",
       "    '_type': '_doc',\n",
       "    '_id': '4',\n",
       "    '_score': 1.0,\n",
       "    '_source': {'Date': '2020-11-19T09:42:49.600762Z',\n",
       "     'Destination': {'lat': 29.771240013385356, 'lon': 45.80298806015347},\n",
       "     'Origin': {'lat': -25.890167699195864, 'lon': 25.649076820110572},\n",
       "     'Title': 'flight1, flight2, flight3'}},\n",
       "   {'_index': 'random_point',\n",
       "    '_type': '_doc',\n",
       "    '_id': '3',\n",
       "    '_score': 1.0,\n",
       "    '_source': {'Date': '2020-11-19T09:42:49.600762Z',\n",
       "     'Destination': {'lat': 19.418767604258065, 'lon': 45.79290066013341},\n",
       "     'Origin': {'lat': -31.534099968422627, 'lon': 28.48530423331058},\n",
       "     'Title': 'flight1, flight2, flight3'}},\n",
       "   {'_index': 'random_point',\n",
       "    '_type': '_doc',\n",
       "    '_id': '<built-in function id>',\n",
       "    '_score': 1.0,\n",
       "    '_source': {'Date': '2020-11-18T22:00:56.612098Z',\n",
       "     'Destination': {'lat': 13.962160129225085, 'lon': 48.107752316661966},\n",
       "     'Origin': {'lat': -12.042520287195355, 'lon': 35.38631072664103},\n",
       "     'Title': 'flight1, flight2, flight3'}},\n",
       "   {'_index': 'random_point',\n",
       "    '_type': '_doc',\n",
       "    '_id': '11',\n",
       "    '_score': 1.0,\n",
       "    '_source': {'Date': '2020-11-19T09:42:49.600762Z',\n",
       "     'Destination': {'lat': 14.929564253164926, 'lon': 43.789750761611536},\n",
       "     'Origin': {'lat': -25.617013067222583, 'lon': 18.197412748546938},\n",
       "     'Title': 'flight1, flight2, flight3'}},\n",
       "   {'_index': 'random_point',\n",
       "    '_type': '_doc',\n",
       "    '_id': '10',\n",
       "    '_score': 1.0,\n",
       "    '_source': {'Date': '2020-11-19T09:42:49.600762Z',\n",
       "     'Destination': {'lat': 28.316341013809787, 'lon': 43.80961845705821},\n",
       "     'Origin': {'lat': -38.662024908989295, 'lon': 25.137689978138546},\n",
       "     'Title': 'flight1, flight2, flight3'}},\n",
       "   {'_index': 'random_point',\n",
       "    '_type': '_doc',\n",
       "    '_id': '6',\n",
       "    '_score': 1.0,\n",
       "    '_source': {'Date': '2020-11-19T09:42:49.600762Z',\n",
       "     'Destination': {'lat': 28.596497488631766, 'lon': 43.96328569672819},\n",
       "     'Origin': {'lat': -39.10536433990131, 'lon': 27.628326268229195},\n",
       "     'Title': 'flight1, flight2, flight3'}},\n",
       "   {'_index': 'random_point',\n",
       "    '_type': '_doc',\n",
       "    '_id': '12',\n",
       "    '_score': 1.0,\n",
       "    '_source': {'Date': '2020-11-19T09:42:49.600762Z',\n",
       "     'Destination': {'lat': 11.05802440928464, 'lon': 45.27766794213287},\n",
       "     'Origin': {'lat': -31.029251243095803, 'lon': 24.110878169014615},\n",
       "     'Title': 'flight1, flight2, flight3'}},\n",
       "   {'_index': 'random_point',\n",
       "    '_type': '_doc',\n",
       "    '_id': '13',\n",
       "    '_score': 1.0,\n",
       "    '_source': {'Date': '2020-11-19T09:42:49.600762Z',\n",
       "     'Destination': {'lat': 18.92064849494497, 'lon': 45.27491510233818},\n",
       "     'Origin': {'lat': -19.937343659836234, 'lon': 26.915274675570323},\n",
       "     'Title': 'flight1, flight2, flight3'}},\n",
       "   {'_index': 'random_point',\n",
       "    '_type': '_doc',\n",
       "    '_id': '5',\n",
       "    '_score': 1.0,\n",
       "    '_source': {'Date': '2020-11-19T09:42:49.600762Z',\n",
       "     'Destination': {'lat': 18.028926658765872, 'lon': 42.704362954611156},\n",
       "     'Origin': {'lat': -28.78955944207125, 'lon': 20.43826772640305},\n",
       "     'Title': 'flight1, flight2, flight3'}}]}}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Search in random_point index\n",
    "res = es.search(index=\"random_point\")\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T14:40:06.707213Z",
     "start_time": "2021-05-03T14:40:06.684107Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'_index': 'random_point',\n",
       " '_type': '_doc',\n",
       " '_id': '2',\n",
       " '_version': 6,\n",
       " '_seq_no': 276,\n",
       " '_primary_term': 1,\n",
       " 'found': True,\n",
       " '_source': {'Date': '2020-11-19T09:42:49.600762Z',\n",
       "  'Destination': {'lat': 14.128594028987003, 'lon': 44.258336663067276},\n",
       "  'Origin': {'lat': -18.99753070156272, 'lon': 18.649682090504445},\n",
       "  'Title': 'flight1, flight2, flight3'}}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res=es.get(index='random_point',doc_type='_doc', id=2)\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}