01-Load-Data-ACogSearch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import requests\n",
    "from pprint import pprint\n",
    "from app.credentials import (\n",
    "    DATASOURCE_CONNECTION_STRING,\n",
    "    AZURE_SEARCH_API_VERSION,\n",
    "    AZURE_SEARCH_ENDPOINT,\n",
    "    AZURE_SEARCH_KEY,\n",
    "    COG_SERVICES_NAME,\n",
    "    COG_SERVICES_KEY\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the names for the data source, skillset, index and indexer\n",
    "blob_container_name = \"snowflake-data\"\n",
    "datasource_name = \"cogsrch-snowflake-datasource-files\"\n",
    "skillset_name = \"cogsrch-snowflake-skillset-files\"\n",
    "index_name = \"cogsrch-snowflake-index-files\"\n",
    "indexer_name = \"cogsrch-snowflake-indexer-files\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup the Payloads header\n",
    "headers = {'Content-Type': 'application/json','api-key': AZURE_SEARCH_KEY}\n",
    "params = {'api-version': AZURE_SEARCH_API_VERSION}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Data Source (Blob container with the Arxiv CS pdfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "204\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "# Create a data source\n",
    "# You should already have a blob container that contains the sample data, see app/credentials.py\n",
    "\n",
    "datasource_payload = {\n",
    "    \"name\": datasource_name,\n",
    "    \"description\": \"Demo files to demonstrate cognitive search capabilities.\",\n",
    "    \"type\": \"azureblob\",\n",
    "    \"credentials\": {\n",
    "        \"connectionString\": DATASOURCE_CONNECTION_STRING\n",
    "    },\n",
    "    \"container\": {\n",
    "        \"name\": blob_container_name\n",
    "    }\n",
    "}\n",
    "r = requests.put(AZURE_SEARCH_ENDPOINT + \"/datasources/\" + datasource_name,\n",
    "                 data=json.dumps(datasource_payload), headers=headers, params=params)\n",
    "print(r.status_code)\n",
    "print(r.ok)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If you have a 403 code, probably you have a wrong endpoint or key, you can debug by uncomment this\n",
    "# r.text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Skillset - OCR, Text Splitter, Language Detection, KeyPhrase extraction, Entity Recognition\n",
    "\n",
    "We need to create now the skillset. This is a set of steps in which we use many Cognitive Services to enrich the documents by extracting information, applying OCR, translating, etc.\n",
    "\n",
    "https://learn.microsoft.com/en-us/azure/search/cognitive-search-working-with-skillsets\n",
    "\n",
    "https://learn.microsoft.com/en-us/azure/search/cognitive-search-predefined-skills\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a skillset\n",
    "skillset_payload = {\n",
    "    \"name\": skillset_name,\n",
    "    \"description\": \"Extract entities, detect language and extract key-phrases\",\n",
    "    \"skills\":\n",
    "    [\n",
    "        {\n",
    "            \"@odata.type\": \"#Microsoft.Skills.Vision.OcrSkill\",\n",
    "            \"description\": \"Extract text (plain and structured) from image.\",\n",
    "            \"context\": \"/document/normalized_images/*\",\n",
    "            \"defaultLanguageCode\": \"en\",\n",
    "            \"detectOrientation\": True,\n",
    "            \"inputs\": [\n",
    "                {\n",
    "                  \"name\": \"image\",\n",
    "                  \"source\": \"/document/normalized_images/*\"\n",
    "                }\n",
    "            ],\n",
    "                \"outputs\": [\n",
    "                {\n",
    "                  \"name\": \"text\",\n",
    "                  \"targetName\" : \"images_text\"\n",
    "                }\n",
    "            ]\n",
    "        },\n",
    "        {\n",
    "            \"@odata.type\": \"#Microsoft.Skills.Text.MergeSkill\",\n",
    "            \"description\": \"Create merged_text, which includes all the textual representation of each image inserted at the right location in the content field. This is useful for PDF and other file formats that supported embedded images.\",\n",
    "            \"context\": \"/document\",\n",
    "            \"insertPreTag\": \" \",\n",
    "            \"insertPostTag\": \" \",\n",
    "            \"inputs\": [\n",
    "                {\n",
    "                  \"name\":\"text\", \"source\": \"/document/content\"\n",
    "                },\n",
    "                {\n",
    "                  \"name\": \"itemsToInsert\", \"source\": \"/document/normalized_images/*/images_text\"\n",
    "                },\n",
    "                {\n",
    "                  \"name\":\"offsets\", \"source\": \"/document/normalized_images/*/contentOffset\"\n",
    "                }\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\n",
    "                  \"name\": \"mergedText\", \n",
    "                  \"targetName\" : \"merged_text\"\n",
    "                }\n",
    "            ]\n",
    "        },\n",
    "        {\n",
    "            \"@odata.type\": \"#Microsoft.Skills.Text.LanguageDetectionSkill\",\n",
    "            \"context\": \"/document\",\n",
    "            \"description\": \"If you have multilingual content, adding a language code is useful for filtering\",\n",
    "            \"inputs\": [\n",
    "                {\n",
    "                  \"name\": \"text\",\n",
    "                  \"source\": \"/document/content\"\n",
    "                }\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\n",
    "                  \"name\": \"languageCode\",\n",
    "                  \"targetName\": \"language\"\n",
    "                }\n",
    "            ]\n",
    "        },\n",
    "        {\n",
    "            \"@odata.type\": \"#Microsoft.Skills.Text.SplitSkill\",\n",
    "            \"context\": \"/document\",\n",
    "            \"textSplitMode\": \"pages\",\n",
    "            \"maximumPageLength\": 5000, # 5000 is default\n",
    "            \"defaultLanguageCode\": \"en\",\n",
    "            \"inputs\": [\n",
    "                {\n",
    "                    \"name\": \"text\",\n",
    "                    \"source\": \"/document/content\"\n",
    "                },\n",
    "                {\n",
    "                    \"name\": \"languageCode\",\n",
    "                    \"source\": \"/document/language\"\n",
    "                }\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\n",
    "                    \"name\": \"textItems\",\n",
    "                    \"targetName\": \"pages\"\n",
    "                }\n",
    "            ]\n",
    "        },\n",
    "        {\n",
    "            \"@odata.type\": \"#Microsoft.Skills.Text.KeyPhraseExtractionSkill\",\n",
    "            \"context\": \"/document/pages/*\",\n",
    "            \"maxKeyPhraseCount\": 2,\n",
    "            \"defaultLanguageCode\": \"en\",\n",
    "            \"inputs\": [\n",
    "                {\n",
    "                    \"name\": \"text\", \n",
    "                    \"source\": \"/document/pages/*\"\n",
    "                },\n",
    "                {\n",
    "                    \"name\": \"languageCode\",\n",
    "                    \"source\": \"/document/language\"\n",
    "                }\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\n",
    "                    \"name\": \"keyPhrases\",\n",
    "                    \"targetName\": \"keyPhrases\"\n",
    "                }\n",
    "            ]\n",
    "        },\n",
    "        {\n",
    "            \"@odata.type\": \"#Microsoft.Skills.Text.V3.EntityRecognitionSkill\",\n",
    "            \"context\": \"/document/pages/*\",\n",
    "            \"categories\": [\"Person\", \"Location\", \"Organization\", \"DateTime\", \"URL\", \"Email\"],\n",
    "            \"minimumPrecision\": 0.5, \n",
    "            \"defaultLanguageCode\": \"en\",\n",
    "            \"inputs\": [\n",
    "                {\n",
    "                    \"name\": \"text\", \n",
    "                    \"source\":\"/document/pages/*\"\n",
    "                },\n",
    "                {\n",
    "                    \"name\": \"languageCode\",\n",
    "                    \"source\": \"/document/language\"\n",
    "                }\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\n",
    "                    \"name\": \"persons\", \n",
    "                    \"targetName\": \"persons\"\n",
    "                },\n",
    "                {\n",
    "                    \"name\": \"locations\", \n",
    "                    \"targetName\": \"locations\"\n",
    "                },\n",
    "                {\n",
    "                    \"name\": \"organizations\", \n",
    "                    \"targetName\": \"organizations\"\n",
    "                },\n",
    "                {\n",
    "                    \"name\": \"dateTimes\", \n",
    "                    \"targetName\": \"dateTimes\"\n",
    "                },\n",
    "                {\n",
    "                    \"name\": \"urls\", \n",
    "                    \"targetName\": \"urls\"\n",
    "                },\n",
    "                {\n",
    "                    \"name\": \"emails\", \n",
    "                    \"targetName\": \"emails\"\n",
    "                }\n",
    "            ]\n",
    "        }\n",
    "    ],\n",
    "    \"cognitiveServices\": {\n",
    "        \"@odata.type\": \"#Microsoft.Azure.Search.CognitiveServicesByKey\",\n",
    "        \"description\": COG_SERVICES_NAME,\n",
    "        \"key\": COG_SERVICES_KEY\n",
    "    }\n",
    "}\n",
    "\n",
    "r = requests.put(AZURE_SEARCH_ENDPOINT + \"/skillsets/\" + skillset_name,\n",
    "                 data=json.dumps(skillset_payload), headers=headers, params=params)\n",
    "print(r.status_code)\n",
    "print(r.ok)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In Azure Cognitive Search, a search index is your searchable content, available to the search engine for indexing, full text search, and filtered queries. An index is defined by a schema and saved to the search service. This content exists within your search service, apart from your primary data stores, which is necessary for the millisecond response times expected in modern applications. Except for specific indexing scenarios, the search service will never connect to or query your local data.\n",
    "\n",
    "The body of the request defines the schema of the search index. A fields collection requires one field to be designated as the key. For blob type, this field is often the \"metadata_storage_path\" that uniquely identifies each file in the container.\n",
    "\n",
    "Reference:\n",
    "\n",
    "https://learn.microsoft.com/en-us/azure/search/search-what-is-an-index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create an index\n",
    "# Queries operate over the searchable fields and filterable fields in the index\n",
    "index_payload = {\n",
    "    \"name\": index_name,\n",
    "    \"fields\": [\n",
    "        {\"name\": \"id\", \"type\": \"Edm.String\", \"key\": \"true\", \"searchable\": \"false\", \"retrievable\": \"true\", \"sortable\": \"false\", \"filterable\": \"false\",\"facetable\": \"false\"},\n",
    "        {\"name\": \"title\", \"type\": \"Edm.String\", \"searchable\": \"true\", \"retrievable\": \"true\", \"facetable\": \"false\", \"filterable\": \"true\", \"sortable\": \"false\"},\n",
    "        {\"name\": \"content\", \"type\": \"Edm.String\", \"searchable\": \"true\", \"retrievable\": \"true\", \"sortable\": \"false\", \"filterable\": \"false\",\"facetable\": \"false\"},\n",
    "        {\"name\": \"language\", \"type\": \"Edm.String\", \"searchable\": \"false\", \"retrievable\": \"true\", \"sortable\": \"true\", \"filterable\": \"true\", \"facetable\": \"true\"},\n",
    "        {\"name\": \"pages\",\"type\": \"Collection(Edm.String)\", \"searchable\": \"false\", \"retrievable\": \"true\", \"sortable\": \"false\", \"filterable\": \"false\", \"facetable\": \"false\"},\n",
    "        {\"name\": \"images_text\", \"type\": \"Collection(Edm.String)\", \"searchable\": \"true\", \"retrievable\": \"true\", \"sortable\": \"false\", \"filterable\": \"false\", \"facetable\": \"false\"},\n",
    "        {\"name\": \"keyPhrases\", \"type\": \"Collection(Edm.String)\", \"searchable\": \"true\", \"retrievable\": \"true\", \"sortable\": \"false\", \"filterable\": \"true\", \"facetable\": \"true\"},\n",
    "        {\"name\": \"persons\", \"type\": \"Collection(Edm.String)\", \"searchable\": \"true\", \"retrievable\": \"true\", \"sortable\": \"false\", \"filterable\": \"false\", \"facetable\": \"false\"},\n",
    "        {\"name\": \"locations\", \"type\": \"Collection(Edm.String)\", \"searchable\": \"true\", \"retrievable\": \"true\", \"sortable\": \"false\", \"filterable\": \"true\", \"facetable\": \"true\"},\n",
    "        {\"name\": \"organizations\", \"type\": \"Collection(Edm.String)\", \"searchable\": \"true\", \"retrievable\": \"true\", \"sortable\": \"false\", \"filterable\": \"true\", \"facetable\": \"true\"},\n",
    "        {\"name\": \"dateTimes\", \"type\": \"Collection(Edm.String)\", \"searchable\": \"true\", \"retrievable\": \"true\", \"sortable\": \"false\", \"filterable\": \"false\", \"facetable\": \"false\"},\n",
    "        {\"name\": \"urls\", \"type\": \"Collection(Edm.String)\", \"searchable\": \"false\", \"retrievable\": \"true\", \"sortable\": \"false\", \"filterable\": \"false\", \"facetable\": \"false\"},\n",
    "        {\"name\": \"emails\", \"type\": \"Collection(Edm.String)\", \"searchable\": \"true\", \"retrievable\": \"true\", \"sortable\": \"false\", \"filterable\": \"true\", \"facetable\": \"false\"},\n",
    "        {\"name\": \"metadata_storage_name\", \"type\": \"Edm.String\", \"searchable\": \"true\", \"retrievable\": \"true\", \"sortable\": \"false\", \"filterable\": \"false\", \"facetable\": \"false\"},\n",
    "        {\"name\": \"metadata_storage_path\", \"type\": \"Edm.String\", \"searchable\": \"true\", \"retrievable\": \"true\", \"sortable\": \"false\", \"filterable\": \"false\", \"facetable\": \"false\"}\n",
    "    ],\n",
    "    \"semantic\": {\n",
    "      \"configurations\": [\n",
    "        {\n",
    "          \"name\": \"my-semantic-config\",\n",
    "          \"prioritizedFields\": {\n",
    "            \"prioritizedContentFields\": [\n",
    "                {\n",
    "                    \"fieldName\": \"content\"\n",
    "                }\n",
    "                ]\n",
    "          }\n",
    "        }\n",
    "      ]\n",
    "    }\n",
    "}\n",
    "\n",
    "r = requests.put(AZURE_SEARCH_ENDPOINT + \"/indexes/\" + index_name,\n",
    "                 data=json.dumps(index_payload), headers=headers, params=params)\n",
    "print(r.status_code)\n",
    "print(r.ok)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create and Run the Indexer - (runs the pipeline)\n",
    "This process takes about 120 mins to load all the Arxiv CS pds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The three components you have created thus far (data source, skillset, index) are inputs to an indexer. Creating the indexer on Azure Cognitive Search is the event that puts the entire pipeline into motion."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create an indexer\n",
    "indexer_payload = {\n",
    "    \"name\": indexer_name,\n",
    "    \"dataSourceName\": datasource_name,\n",
    "    \"targetIndexName\": index_name,\n",
    "    \"skillsetName\": skillset_name,\n",
    "    \"schedule\" : { \"interval\" : \"PT2H\"}, # How often do you want to check for new content in the data source\n",
    "    \"fieldMappings\": [\n",
    "        {\n",
    "          \"sourceFieldName\" : \"metadata_storage_path\",\n",
    "          \"targetFieldName\" : \"id\",\n",
    "          \"mappingFunction\" : { \"name\" : \"base64Encode\" }\n",
    "        },\n",
    "        {\n",
    "          \"sourceFieldName\" : \"metadata_title\",\n",
    "          \"targetFieldName\" : \"title\"\n",
    "        }\n",
    "    ],\n",
    "    \"outputFieldMappings\":\n",
    "    [\n",
    "        {\n",
    "            \"sourceFieldName\": \"/document/content\",\n",
    "            \"targetFieldName\": \"content\"\n",
    "        },\n",
    "        {\n",
    "            \"sourceFieldName\": \"/document/pages/*\",\n",
    "            \"targetFieldName\": \"pages\"\n",
    "        },\n",
    "        {\n",
    "            \"sourceFieldName\" : \"/document/normalized_images/*/images_text\",\n",
    "            \"targetFieldName\" : \"images_text\"\n",
    "        },\n",
    "        {\n",
    "            \"sourceFieldName\": \"/document/language\",\n",
    "            \"targetFieldName\": \"language\"\n",
    "        },\n",
    "        {\n",
    "            \"sourceFieldName\": \"/document/pages/*/keyPhrases/*\",\n",
    "            \"targetFieldName\": \"keyPhrases\"\n",
    "        },\n",
    "        {\n",
    "          \"sourceFieldName\" : \"/document/pages/*/persons/*\", \n",
    "          \"targetFieldName\" : \"persons\"\n",
    "        },\n",
    "        {\n",
    "          \"sourceFieldName\" : \"/document/pages/*/locations/*\", \n",
    "          \"targetFieldName\" : \"locations\"\n",
    "        },\n",
    "        {\n",
    "            \"sourceFieldName\": \"/document/pages/*/organizations/*\",\n",
    "            \"targetFieldName\": \"organizations\"\n",
    "        },\n",
    "        {\n",
    "            \"sourceFieldName\": \"/document/pages/*/dateTimes/*\",\n",
    "            \"targetFieldName\": \"dateTimes\"\n",
    "        },\n",
    "        {\n",
    "            \"sourceFieldName\": \"/document/pages/*/urls/*\",\n",
    "            \"targetFieldName\": \"urls\"\n",
    "        },\n",
    "        {\n",
    "            \"sourceFieldName\": \"/document/pages/*/emails/*\",\n",
    "            \"targetFieldName\": \"emails\"\n",
    "        }\n",
    "    ],\n",
    "    \"parameters\":\n",
    "    {\n",
    "        \"maxFailedItems\": -1,\n",
    "        \"maxFailedItemsPerBatch\": -1,\n",
    "        \"configuration\":\n",
    "        {\n",
    "            \"dataToExtract\": \"contentAndMetadata\",\n",
    "            \"imageAction\": \"generateNormalizedImages\"\n",
    "        }\n",
    "    }\n",
    "}\n",
    "\n",
    "r = requests.put(AZURE_SEARCH_ENDPOINT + \"/indexers/\" + indexer_name,\n",
    "                 data=json.dumps(indexer_payload), headers=headers, params=params)\n",
    "print(r.status_code)\n",
    "print(r.ok)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Optionally, get indexer status to confirm that it's running\n",
    "r = requests.get(AZURE_SEARCH_ENDPOINT + \"/indexers/\" + indexer_name +\n",
    "                 \"/status\", headers=headers, params=params)\n",
    "# pprint(json.dumps(r.json(), indent=1))\n",
    "print(r.status_code)\n",
    "print(\"Status:\",r.json().get('lastResult').get('status'))\n",
    "print(\"Items Processed:\",r.json().get('lastResult').get('itemsProcessed'))\n",
    "print(r.ok)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**When the indexer finishes running we will have all 9.8k documents indexed in our Search Engine!.**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# References"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/search/azure-search-documents/samples\n",
    "\n",
    "https://learn.microsoft.com/en-us/azure/search/search-get-started-python\n",
    "\n",
    "https://github.com/Azure-Samples/azure-search-python-samples/blob/main/Tutorial-AI-Enrichment/PythonTutorial-AzureSearch-AIEnrichment.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8 - AzureML",
   "language": "python",
   "name": "python38-azureml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  },
  "vscode": {
   "interpreter": {
    "hash": "9ff083f0c83558f9261023d47a77b9b3eb892c62cdbe066d046abcad1a5edb5c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}