x09-featurestore.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a12e6d23-b0bc-463e-885c-ccfedccd9aac",
   "metadata": {},
   "source": [
    "# Feature Store\n",
    "\n",
    "The purpose of this notebook is to:\n",
    "\n",
    "* Create a Feature Store\n",
    "* Create an entity with some features\n",
    "* Batch ingest some feature data\n",
    "* Deploy a Cloud Function that can read this data and feed it to the model\n",
    "\n",
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6080f973-5159-4967-acff-8b3add013d57",
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.api_core import operations_v1\n",
    "from google.cloud.aiplatform_v1beta1 import FeaturestoreOnlineServingServiceClient, FeaturestoreServiceClient, FeatureSelector\n",
    "from google.cloud.aiplatform_v1beta1.types import featurestore_online_service as featurestore_online_service_pb2\n",
    "from google.cloud.aiplatform_v1beta1.types import entity_type as entity_type_pb2\n",
    "from google.cloud.aiplatform_v1beta1.types import feature as feature_pb2\n",
    "from google.cloud.aiplatform_v1beta1.types import featurestore as featurestore_pb2\n",
    "from google.cloud.aiplatform_v1beta1.types import featurestore_service as featurestore_service_pb2\n",
    "from google.cloud.aiplatform_v1beta1.types import io as io_pb2\n",
    "from google.cloud.aiplatform_v1beta1.types import ListFeaturestoresRequest, CreateFeaturestoreRequest, Featurestore\n",
    "\n",
    "from google.protobuf.timestamp_pb2 import Timestamp\n",
    "from google.cloud.aiplatform_v1beta1.types import featurestore_monitoring as featurestore_monitoring_pb2\n",
    "from google.protobuf.duration_pb2 import Duration\n",
    "\n",
    "import yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d299592a-4705-4537-8356-228b05dd65b3",
   "metadata": {},
   "source": [
    "## Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b1f0b2ac-442c-4caf-8778-cef2b543bd7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('mainconfig.yaml') as f:\n",
    "    main_config = yaml.safe_load(f)\n",
    "main_config = main_config['personal']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "4e037bf4-51f5-4da6-a314-fe9670bfe3a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Project ID: pbalm-cxb-aa\n",
      "Region: europe-west1\n"
     ]
    }
   ],
   "source": [
    "PROJECT = main_config['project'] \n",
    "REGION = main_config['region'] \n",
    "\n",
    "SERVICE_ACCOUNT = main_config['service_account']\n",
    "\n",
    "print(\"Project ID:\", PROJECT)\n",
    "print(\"Region:\", REGION)\n",
    "\n",
    "API_ENDPOINT = f\"{REGION}-aiplatform.googleapis.com\"  \n",
    "FEATURESTORE_ID = main_config['featurestore_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "744f7719-ecbd-47c0-8d66-2463f0ba6a8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "admin_client = FeaturestoreServiceClient(\n",
    "    client_options={\"api_endpoint\": API_ENDPOINT})\n",
    "data_client = FeaturestoreOnlineServingServiceClient(\n",
    "    client_options={\"api_endpoint\": API_ENDPOINT})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "872313b2-38dd-40d3-8c72-22bce8861672",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Existing feature stores in project pbalm-cxb-aa and region europe-west1:\n",
      "name: \"projects/188940921537/locations/europe-west1/featurestores/creditcards\"\n",
      "create_time {\n",
      "  seconds: 1657136254\n",
      "  nanos: 29531000\n",
      "}\n",
      "update_time {\n",
      "  seconds: 1657136254\n",
      "  nanos: 118341000\n",
      "}\n",
      "etag: \"AMEw9yN8kR8_IkgNwOP3UJEt3lhowvZv_EuG0MS0pCYZolTvWM4iwEF9MPof3Gpy30A=\"\n",
      "online_serving_config {\n",
      "  fixed_node_count: 3\n",
      "}\n",
      "state: STABLE\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(f'Existing feature stores in project {PROJECT} and region {REGION}:')\n",
    "for f in admin_client.list_featurestores(ListFeaturestoresRequest(parent=admin_client.common_location_path(PROJECT, REGION))):\n",
    "      print(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a81d9b3b-a752-4ddb-8872-ea630b814571",
   "metadata": {},
   "source": [
    "## Move this stuff to source code files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "a446f126-5d41-4089-856b-f6482db724de",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_fs(project, region, store_id, store_name=None):\n",
    "    base_path = admin_client.common_location_path(project, region)\n",
    "    \n",
    "    for f in admin_client.list_featurestores(ListFeaturestoresRequest(parent=admin_client.common_location_path(project, region))):\n",
    "        existing_id = f.name.split('/')[-1]\n",
    "        if store_id == existing_id:\n",
    "            print(f'Feature Store \"{store_id}\" already exists in {region}')\n",
    "            return\n",
    "    \n",
    "    if store_name is None:\n",
    "        store_name = f'{base_path}/{store_id}'\n",
    "    \n",
    "    req = CreateFeaturestoreRequest(\n",
    "        parent = base_path,\n",
    "        featurestore = Featurestore(\n",
    "            name=store_name,\n",
    "            online_serving_config=Featurestore.OnlineServingConfig(fixed_node_count=3)),\n",
    "        featurestore_id = store_id)\n",
    "    \n",
    "    lro = admin_client.create_featurestore(req)\n",
    "    name = lro.result()\n",
    "    print(f'Created Feature Store {name} in {region}')\n",
    "    return name\n",
    "\n",
    "\n",
    "def create_entity(project, region, store_id, entity, entity_descr, features, features_descr=None):\n",
    "    \n",
    "    if features_descr is None:\n",
    "        features_descr = features\n",
    "    \n",
    "    if len(features) != len(features_descr):\n",
    "        print(f'ERROR: Got {len(features)} features and {len(features_descr)} descriptions')\n",
    "        return\n",
    "    \n",
    "    print(f'Creating entity {entity} in Feature Store {store_id} ({region})')\n",
    "    \n",
    "    snapshot_analysis = featurestore_monitoring_pb2.FeaturestoreMonitoringConfig.SnapshotAnalysis(\n",
    "                    monitoring_interval=Duration(seconds=3600))  # 1 hour\n",
    "    \n",
    "    lro = admin_client.create_entity_type(\n",
    "        featurestore_service_pb2.CreateEntityTypeRequest(\n",
    "            parent=admin_client.featurestore_path(project, region, store_id),\n",
    "            entity_type_id=entity,\n",
    "            entity_type=entity_type_pb2.EntityType(\n",
    "             description=entity_descr,\n",
    "             monitoring_config=featurestore_monitoring_pb2.FeaturestoreMonitoringConfig(\n",
    "                snapshot_analysis=snapshot_analysis))\n",
    "        )\n",
    "    ).result()\n",
    "    \n",
    "    print(lro)\n",
    "    \n",
    "    def _create_f_request(name, descr):\n",
    "        return featurestore_service_pb2.CreateFeatureRequest(\n",
    "                feature=feature_pb2.Feature(\n",
    "                    value_type=feature_pb2.Feature.ValueType.DOUBLE,\n",
    "                    description=descr,\n",
    "                    monitoring_config=featurestore_monitoring_pb2.FeaturestoreMonitoringConfig(\n",
    "                        snapshot_analysis=snapshot_analysis)),\n",
    "                feature_id=name)\n",
    "    \n",
    "    requests = [_create_f_request(x[0], x[1]) for x in zip(features, features_descr)]\n",
    "    \n",
    "    print(f'\\nCreating features: {\",\".join(features)}')\n",
    "\n",
    "    lro = admin_client.batch_create_features(\n",
    "        parent=admin_client.entity_type_path(PROJECT, REGION, FEATURESTORE_ID, entity),\n",
    "        requests=requests).result()\n",
    "    \n",
    "    return lro\n",
    "\n",
    "\n",
    "def ingest_entities_csv(project, region, store_id, entity, features, gcs_uris):\n",
    "\n",
    "    timestamp = Timestamp()\n",
    "    timestamp.GetCurrentTime()\n",
    "    timestamp.nanos = 0\n",
    "    \n",
    "    specs = [featurestore_service_pb2.ImportFeatureValuesRequest.FeatureSpec(id=f) for f in features]\n",
    "    \n",
    "    import_request_transaction = featurestore_service_pb2.ImportFeatureValuesRequest(\n",
    "        entity_type=admin_client.entity_type_path(project, region, store_id, entity),\n",
    "        csv_source=io_pb2.CsvSource(gcs_source=io_pb2.GcsSource(uris=gcs_uris)),\n",
    "        feature_specs=specs,\n",
    "        entity_id_field=entity,\n",
    "        feature_time=timestamp, # unique timestamp for all\n",
    "        worker_count=5)\n",
    "    \n",
    "    print(f'Ingesting features for \"{entity}\" entity...')\n",
    "    ingestion_lro = admin_client.import_feature_values(import_request_transaction).result()\n",
    "    print('done')\n",
    "    \n",
    "    return ingestion_lro\n",
    "\n",
    "\n",
    "# entity is the name of the entity type you want to read, for example: user\n",
    "# entity_value is the specific instance of the entity that you want to have the feature of, for example a user ID\n",
    "def read_features(project, region, store_id, entity, features, entity_value):\n",
    "    feature_selector = FeatureSelector()\n",
    "    feature_selector.id_matcher.ids = features\n",
    "    \n",
    "    read_request = featurestore_online_service_pb2.ReadFeatureValuesRequest(\n",
    "        entity_type = admin_client.entity_type_path(project, region, store_id, entity),\n",
    "        entity_id = entity_value,\n",
    "        feature_selector=feature_selector)\n",
    "    \n",
    "    res = data_client.read_feature_values(read_request)\n",
    "    values = [d.value for d in res.entity_view.data]\n",
    "\n",
    "    # return a dict with { 'feature1': val1, 'feature2': val2, ... }\n",
    "    # exclude features that do not have a generate_time: these do not exist in the store\n",
    "    return {f:v.double_value for (f,v) in zip(features, values) if v.metadata.generate_time}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "070b4fb3-0186-4d1d-87d0-b7fb7dd02be8",
   "metadata": {},
   "source": [
    "## Create Feature Store and entity with features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fa460e5f-9f32-47dd-ac8c-418e19703c59",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created Feature Store name: \"projects/188940921537/locations/europe-west1/featurestores/creditcards\"\n",
      " in europe-west1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "name: \"projects/188940921537/locations/europe-west1/featurestores/creditcards\""
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "create_fs(PROJECT, REGION, FEATURESTORE_ID, \"Feature Store for credit card use case\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a1218e2c-af83-4b51-bd57-678ef6ef46dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name: \"projects/188940921537/locations/europe-west1/featurestores/creditcards\"\n",
       "create_time {\n",
       "  seconds: 1657136254\n",
       "  nanos: 29531000\n",
       "}\n",
       "update_time {\n",
       "  seconds: 1657136254\n",
       "  nanos: 118341000\n",
       "}\n",
       "etag: \"AMEw9yMQYcxSMo07Pxe1SoR-xw4qcMk92ByK6VI9LsploP5c_3tPvNcw0CO9rwiH9v0=\"\n",
       "online_serving_config {\n",
       "  fixed_node_count: 3\n",
       "}\n",
       "state: STABLE"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "admin_client.get_featurestore(name = admin_client.featurestore_path(PROJECT, REGION, FEATURESTORE_ID))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "44b4e1aa-c093-4a74-8477-8be70f99139b",
   "metadata": {},
   "outputs": [],
   "source": [
    "entity = 'user'\n",
    "entity_descr = 'User ID'\n",
    "features = ['v27', 'v28']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0d9185f4-c5bf-4f55-9405-5bc4f19b34fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating entity user in Feature Store creditcards (europe-west1)\n",
      "name: \"projects/188940921537/locations/europe-west1/featurestores/creditcards/entityTypes/user\"\n",
      "\n",
      "\n",
      "Creating features: v27,v28\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "features {\n",
       "  name: \"projects/188940921537/locations/europe-west1/featurestores/creditcards/entityTypes/user/features/v27\"\n",
       "}\n",
       "features {\n",
       "  name: \"projects/188940921537/locations/europe-west1/featurestores/creditcards/entityTypes/user/features/v28\"\n",
       "}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "create_entity(PROJECT, REGION, FEATURESTORE_ID, entity, entity_descr, features)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e961e97-3688-41b6-a7e8-bb80cd41141c",
   "metadata": {},
   "source": [
    "### Create the feature data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "95c67eb0-b478-4321-a588-be6809f67bd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "filename = f'features_{entity}.csv'\n",
    "\n",
    "with open(filename, 'w') as f:\n",
    "    line = f'{entity},{\",\".join(features)}\\n'\n",
    "    f.write(line)\n",
    "    for i in range(100):\n",
    "        f.write(f'user{i},{random.random()},{random.random()}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c93334b1-8735-4601-8acd-04e9f9615ccd",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "user,v27,v28\n",
      "user0,0.8752007622310332,0.5085713748296266\n",
      "user1,0.18172587234862592,0.331560453610137\n",
      "user2,0.7678604213202734,0.4796658653927143\n",
      "user3,0.03530153195587893,0.3925520421488291\n",
      "user4,0.714354519152235,0.7416135959249333\n",
      "user5,0.310460362693614,0.446304360638923\n",
      "user6,0.6262247551571973,0.6205434867159426\n",
      "user7,0.934891273361272,0.3159109422133256\n",
      "user8,0.005784604731528753,0.6083360616030765\n",
      "user9,0.4650906821533981,0.23484670927139173\n",
      "user10,0.38533364987649643,0.055908172244078735\n",
      "user11,0.19284495284841252,0.9662444569349677\n",
      "user12,0.9296274362017263,0.4560936299443399\n",
      "user13,0.1237296954914393,0.31201567278130604\n",
      "user14,0.4903181549866118,0.168048375398973\n",
      "user15,0.04618572281597866,0.7157594756749772\n",
      "user16,0.6975090670221781,0.9732154609744624\n",
      "user17,0.7749512534629103,0.11609543617175999\n",
      "user18,0.8966848026372027,0.4194811229151356\n",
      "user19,0.4974831622648439,0.8144141834082029\n",
      "user20,0.685089344258394,0.7309976629059198\n",
      "user21,0.2208765865449639,0.10142488435923813\n",
      "user22,0.6675515194463835,0.07415212109980818\n",
      "user23,0.39727150746375783,0.8383301500950694\n",
      "user24,0.47774870098626043,0.5296510277977463\n",
      "user25,0.5504820670058833,0.8223186662430814\n",
      "user26,0.5763362835701817,0.4867620411863589\n",
      "user27,0.1753288527369079,0.8028479118312256\n",
      "user28,0.2533469887420504,0.8126693424227323\n",
      "user29,0.5044280468782997,0.047272782324895846\n",
      "user30,0.5055643663018383,0.8299754872216456\n",
      "user31,0.772487314172012,0.6548359576571573\n",
      "user32,0.9690156302229948,0.46728012150849796\n",
      "user33,0.8595216519849646,0.10846151489699407\n",
      "user34,0.39212767561083106,0.9375114912192317\n",
      "user35,0.43524677619930896,0.8399408723328832\n",
      "user36,0.18964177172531227,0.6113697456905041\n",
      "user37,0.10347432317648686,0.7043942196909725\n",
      "user38,0.46912074005528304,0.22873981287322986\n",
      "user39,0.6803779091675075,0.14598766618874548\n",
      "user40,0.042278925440255355,0.2644590818516792\n",
      "user41,0.2767016178015572,0.9022310001640285\n",
      "user42,0.9524590635959072,0.48402415069365123\n",
      "user43,0.2700013663055131,0.8851410855316094\n",
      "user44,0.5762021031228354,0.6098948606458687\n",
      "user45,0.9254690096506768,0.6189600308924704\n",
      "user46,0.2842713598197727,0.5718896248860601\n",
      "user47,0.12598374613985153,0.34564352950085664\n",
      "user48,0.31933025597956555,0.7397820944469276\n",
      "user49,0.3430006078416473,0.7704222202900903\n",
      "user50,0.4160523061483171,0.004788568132110815\n",
      "user51,0.668399339907748,0.5753205345033995\n",
      "user52,0.23945420565582254,0.16180639362602578\n",
      "user53,0.5874251765048016,0.2782450685821123\n",
      "user54,0.12904654190211984,0.5586561030523157\n",
      "user55,0.3607785238202802,0.16770789333567315\n",
      "user56,0.41963913415585774,0.005607128011895002\n",
      "user57,0.06375235938847024,0.4231366146484993\n",
      "user58,0.3362461270665654,0.8530968478458815\n",
      "user59,0.6545694312379847,0.7348383424361273\n",
      "user60,0.5251304868853773,0.8746490240489575\n",
      "user61,0.9648807107799583,0.286297941036335\n",
      "user62,0.8332453927987582,0.21894724380988762\n",
      "user63,0.26291602539847114,0.5942065654985108\n",
      "user64,0.39368729349731435,0.224490650461302\n",
      "user65,0.8257122210620341,0.7004401376864008\n",
      "user66,0.4630816946490528,0.4666457510585724\n",
      "user67,0.404788421272006,0.006423985084326178\n",
      "user68,0.1359627510984206,0.346910157368695\n",
      "user69,0.5668327377716309,0.7243631820081757\n",
      "user70,0.3065485362416538,0.8590031608814841\n",
      "user71,0.5377107621373595,0.49387082758885137\n",
      "user72,0.9983116261478825,0.8284624765578574\n",
      "user73,0.7404785570724903,0.6997663090809809\n",
      "user74,0.7343747661353249,0.5648245655210457\n",
      "user75,0.551942771264598,0.44659194996363505\n",
      "user76,0.880714731127553,0.20550097013250213\n",
      "user77,0.083696447458799,0.3723744740459042\n",
      "user78,0.9045601842564021,0.693570131929841\n",
      "user79,0.38788216473282977,0.5207472514412907\n",
      "user80,0.7447271752496522,0.895063187716863\n",
      "user81,0.09078725936481802,0.5437453487538944\n",
      "user82,0.5652507931309744,0.918101810609941\n",
      "user83,0.5333563600995465,0.2174980830240829\n",
      "user84,0.4097438565575483,0.16998624985190103\n",
      "user85,0.21900059988735232,0.3019580774949657\n",
      "user86,0.4661716794443974,0.24257893035255385\n",
      "user87,0.08797278154154597,0.5928722069394852\n",
      "user88,0.3138474469118937,0.9144544365268454\n",
      "user89,0.8759096625138209,0.9418296910027474\n",
      "user90,0.43233606501215793,0.4782478397702635\n",
      "user91,0.9416494951196843,0.806256891634685\n",
      "user92,0.7779062300322946,0.7522797939488289\n",
      "user93,0.08492768270605855,5.3993977775501634e-05\n",
      "user94,0.5994417869948815,0.5387954257682637\n",
      "user95,0.8818240021759036,0.3420486529317205\n",
      "user96,0.33044465161460523,0.9422315030335132\n",
      "user97,0.9175349839104221,0.013577671867111896\n",
      "user98,0.1186484791889193,0.3108415357458678\n",
      "user99,0.8770988976812567,0.6458292822903422\n"
     ]
    }
   ],
   "source": [
    "!cat {filename}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "72bd5954-9c5d-46ce-ac3a-2c00d9843029",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'pbalm-cxb-aa-eu'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BUCKET = main_config['bucket']\n",
    "BUCKET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b1fa733e-a077-4054-84a5-8be3f36bdc3b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying file://features_user.csv [Content-Type=text/csv]...\n",
      "/ [1 files][  4.4 KiB/  4.4 KiB]                                                \n",
      "Operation completed over 1 objects/4.4 KiB.                                      \n"
     ]
    }
   ],
   "source": [
    "!gsutil cp {filename} gs://{BUCKET}/{filename} "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e228def0-6b85-4158-8549-abfe049120c4",
   "metadata": {},
   "source": [
    "## Ingest feature data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d6902667-c8ff-4475-aa65-f65d37c406c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ingesting features for \"user\" entity...\n",
      "done\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "imported_entity_count: 100\n",
       "imported_feature_value_count: 200"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gcs_uris = [f'gs://{BUCKET}/{filename}']\n",
    "\n",
    "ingest_entities_csv(PROJECT, REGION, FEATURESTORE_ID, entity, features, gcs_uris)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b4397d4-6586-4541-a85b-42b7c84ffd31",
   "metadata": {},
   "source": [
    "## Test Feature Store: Read values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "5edad53f-e3fb-4c0b-900d-e6a89458bd72",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'user90': {'v27': 0.43233606501215793, 'v28': 0.4782478397702635},\n",
       " 'user91': {'v27': 0.9416494951196843, 'v28': 0.806256891634685},\n",
       " 'user92': {'v27': 0.7779062300322946, 'v28': 0.7522797939488289},\n",
       " 'user93': {'v27': 0.08492768270605855, 'v28': 5.3993977775501634e-05},\n",
       " 'user94': {'v27': 0.5994417869948815, 'v28': 0.5387954257682637},\n",
       " 'user95': {'v27': 0.8818240021759036, 'v28': 0.3420486529317205},\n",
       " 'user96': {'v27': 0.33044465161460523, 'v28': 0.9422315030335132},\n",
       " 'user97': {'v27': 0.9175349839104221, 'v28': 0.013577671867111896},\n",
       " 'user98': {'v27': 0.1186484791889193, 'v28': 0.3108415357458678},\n",
       " 'user99': {'v27': 0.8770988976812567, 'v28': 0.6458292822903422},\n",
       " 'user100': {},\n",
       " 'user101': {}}"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features_data = {}\n",
    "for i in range(90,102):\n",
    "    entity_id = f'user{i}'\n",
    "    features_data[entity_id] = read_features(PROJECT, REGION, FEATURESTORE_ID, entity, features, entity_id)\n",
    "\n",
    "features_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea507bd6-a735-40a6-999d-7e85372d446c",
   "metadata": {},
   "outputs": [],
   "source": [
    "list(feature_pb2.Feature.ValueType)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53ed8e98-b1d6-4295-aed1-145a870f3791",
   "metadata": {},
   "outputs": [],
   "source": [
    "FEATURESTORE_ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0f79a59-3bab-462a-93e1-a55057e256c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "REGION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "f7a74edf-1f36-43cd-8869-797e50edd447",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "header {\n",
      "  entity_type: \"projects/188940921537/locations/europe-west1/featurestores/creditcards/entityTypes/user\"\n",
      "  feature_descriptors {\n",
      "    id: \"v27\"\n",
      "  }\n",
      "  feature_descriptors {\n",
      "    id: \"v28\"\n",
      "  }\n",
      "}\n",
      "entity_view {\n",
      "  entity_id: \"user100\"\n",
      "  data {\n",
      "  }\n",
      "  data {\n",
      "  }\n",
      "}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entity_value = 'user100'\n",
    "\n",
    "feature_selector = FeatureSelector()\n",
    "feature_selector.id_matcher.ids = features\n",
    "\n",
    "read_request = featurestore_online_service_pb2.ReadFeatureValuesRequest(\n",
    "    entity_type = admin_client.entity_type_path(PROJECT, REGION, FEATURESTORE_ID, entity),\n",
    "    entity_id = entity_value,\n",
    "    feature_selector=feature_selector)\n",
    "\n",
    "res = data_client.read_feature_values(read_request)\n",
    "print(res)\n",
    "values = [d.value for d in res.entity_view.data]\n",
    "\n",
    "# return a dict with { 'feature1': val1, 'feature2': val2, ... }\n",
    "{f:v.double_value for (f,v) in zip(features, values) if v.metadata.generate_time}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "f5c59063-6a85-4be0-b3e6-e27b515a730c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(res.entity_view.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "fd17eb1c-8c97-4701-aef3-f39d2974df9d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res.entity_view.data[0].value.metadata.generate_time is None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "5ab505c4-4314-4140-aef2-3e02307bb26a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "header {\n",
       "  entity_type: \"projects/188940921537/locations/europe-west1/featurestores/creditcards/entityTypes/user\"\n",
       "  feature_descriptors {\n",
       "    id: \"v27\"\n",
       "  }\n",
       "  feature_descriptors {\n",
       "    id: \"v28\"\n",
       "  }\n",
       "}\n",
       "entity_view {\n",
       "  entity_id: \"user100\"\n",
       "  data {\n",
       "  }\n",
       "  data {\n",
       "  }\n",
       "}"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f66615e-99dd-4161-abcb-87e3ac831569",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "python3",
   "name": "tf2-gpu.2-8.m93",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-8:m93"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}