08-model-monitoring.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "441c8b66",
   "metadata": {},
   "source": [
    "# 08 - Model Monitoring\n",
    "\n",
    "This notebook covers configuring model monitoring jobs for skew and drift detection:\n",
    "1. Set skew and drift threshold.\n",
    "2. Create a monitoring job for all the models under and endpoint.\n",
    "3. List the monitoring jobs.\n",
    "4. Simulate skewed prediction requests.\n",
    "5. Pause and delete the monitoring job."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2a0d93e",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c95c73cf",
   "metadata": {},
   "source": [
    "### Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aee62910",
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "from datetime import datetime\n",
    "import time\n",
    "import random\n",
    "\n",
    "from google.protobuf.duration_pb2 import Duration\n",
    "from google.cloud import aiplatform as vertex_ai\n",
    "from google.cloud import aiplatform_v1beta1 as vertex_ai_beta"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33eb5433",
   "metadata": {},
   "source": [
    "### Setup Google Cloud project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9e34ea5",
   "metadata": {},
   "outputs": [],
   "source": [
    "PROJECT = '[your-project-id]' # Change to your project id.\n",
    "REGION = 'europe-west1' # Change to your region.\n",
    "BUCKET = '[your-bucket-name]' # Change to your bucket name.\n",
    "\n",
    "if PROJECT == \"\" or PROJECT is None or PROJECT == \"[your-project-id]\":\n",
    "    # Get your GCP project id from gcloud\n",
    "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
    "    PROJECT = shell_output[0]\n",
    "    \n",
    "if BUCKET == \"\" or BUCKET is None or BUCKET == \"[your-bucket-name]\":\n",
    "    # Get your bucket name to GCP project id\n",
    "    BUCKET = PROJECT\n",
    "    # Try to create the bucket if it doesn't exists\n",
    "    ! gsutil mb -l $REGION gs://$BUCKET\n",
    "    print(\"\")\n",
    "\n",
    "PARENT = f\"projects/{PROJECT}/locations/{REGION}\"\n",
    "\n",
    "print(\"Project ID:\", PROJECT)\n",
    "print(\"Region:\", REGION)\n",
    "print(\"Bucket name:\", BUCKET)\n",
    "print(\"Vertex API Parent URI:\", PARENT)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0051e6da",
   "metadata": {},
   "source": [
    "### Set configurations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4ffa4e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET_DISPLAY_NAME = 'chicago-taxi-tips'\n",
    "ENDPOINT_DISPLAY_NAME = 'chicago-taxi-tips-classifier'\n",
    "MONITORING_JOB_NAME = f\"monitor-{ENDPOINT_DISPLAY_NAME}\"\n",
    "NOTIFY_EMAILS = [\"jgpuga@google.com\"] # Change to your email address.\n",
    "\n",
    "LOG_SAMPLE_RATE = 0.8\n",
    "MONITOR_INTERVAL = 3600\n",
    "TARGET_FEATURE_NAME = 'tip_bin'\n",
    "\n",
    "vertex_ai.init(project=PROJECT, location=REGION)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09153445",
   "metadata": {},
   "source": [
    "## Create Job Service Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c8c8872",
   "metadata": {},
   "outputs": [],
   "source": [
    "job_client_beta = vertex_ai_beta.JobServiceClient(\n",
    "    client_options={\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "250e521b",
   "metadata": {},
   "source": [
    "## 1. Set Skew and Drift Thresholds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10bd314f",
   "metadata": {},
   "outputs": [],
   "source": [
    "SKEW_THRESHOLDS = {\n",
    "    'trip_month': 0.3,\n",
    "    'trip_day': 0.3,\n",
    "    'trip_day_of_week': 0.3,\n",
    "    'trip_hour': 0.3,\n",
    "    'trip_seconds': 0.3,\n",
    "    'trip_miles': 0.3,\n",
    "    'payment_type': 0.3,\n",
    "    'pickup_grid': 0.3,\n",
    "    'dropoff_grid': 0.3,\n",
    "    'euclidean': 0.3,\n",
    "    'loc_cross': 0.3,  \n",
    "}\n",
    "\n",
    "DRIFT_THRESHOLDS = {\n",
    "    'trip_month': 0.3,\n",
    "    'trip_day': 0.3,\n",
    "    'trip_day_of_week': 0.3,\n",
    "    'trip_hour': 0.3,\n",
    "    'trip_seconds': 0.3,\n",
    "    'trip_miles': 0.3,\n",
    "    'payment_type': 0.3,\n",
    "    'pickup_grid': 0.3,\n",
    "    'dropoff_grid': 0.3,\n",
    "    'euclidean': 0.3,\n",
    "    'loc_cross': 0.3,  \n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1579ae9d",
   "metadata": {},
   "source": [
    "## 2. Create Monitoring Job"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32756e32",
   "metadata": {},
   "source": [
    "### Retrieve the Vertex dataset and endpoint models to monitor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f17f0d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = vertex_ai.TabularDataset.list(\n",
    "    filter=f\"display_name={DATASET_DISPLAY_NAME}\", \n",
    "    order_by=\"update_time\")[-1]\n",
    "\n",
    "bq_source_uri = dataset.gca_resource.metadata[\"inputConfig\"][\"bigquerySource\"][\"uri\"]\n",
    "    \n",
    "endpoint = vertex_ai.Endpoint.list(\n",
    "    filter=f'display_name={ENDPOINT_DISPLAY_NAME}', \n",
    "    order_by=\"update_time\")[-1]\n",
    "\n",
    "endpoint_uri = endpoint.gca_resource.name\n",
    "\n",
    "model_ids = [model.id for model in endpoint.list_models()]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8f3315d",
   "metadata": {},
   "source": [
    "### Configure the monitoring job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2998243",
   "metadata": {},
   "outputs": [],
   "source": [
    "skew_thresholds = {\n",
    "    feature: vertex_ai_beta.ThresholdConfig(value=float(value))\n",
    "    for feature, value in SKEW_THRESHOLDS.items()\n",
    "}\n",
    "\n",
    "drift_thresholds = {\n",
    "    feature: vertex_ai_beta.ThresholdConfig(value=float(value))\n",
    "    for feature, value in DRIFT_THRESHOLDS.items()\n",
    "}\n",
    "\n",
    "skew_config = vertex_ai_beta.ModelMonitoringObjectiveConfig.TrainingPredictionSkewDetectionConfig(\n",
    "    skew_thresholds=skew_thresholds\n",
    ")\n",
    "\n",
    "drift_config = vertex_ai_beta.ModelMonitoringObjectiveConfig.PredictionDriftDetectionConfig(\n",
    "    drift_thresholds=drift_thresholds\n",
    ")\n",
    "\n",
    "sampling_config = vertex_ai_beta.SamplingStrategy(\n",
    "    random_sample_config=vertex_ai_beta.SamplingStrategy.RandomSampleConfig(\n",
    "        sample_rate=LOG_SAMPLE_RATE\n",
    "    )\n",
    ")\n",
    "\n",
    "schedule_config = vertex_ai_beta.ModelDeploymentMonitoringScheduleConfig(\n",
    "    monitor_interval=Duration(seconds=MONITOR_INTERVAL)\n",
    ")\n",
    "\n",
    "training_dataset = vertex_ai_beta.ModelMonitoringObjectiveConfig.TrainingDataset(\n",
    "    target_field=TARGET_FEATURE_NAME,\n",
    "    bigquery_source = vertex_ai_beta.types.io.BigQuerySource(\n",
    "        input_uri=bq_source_uri\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "objective_template = vertex_ai_beta.ModelDeploymentMonitoringObjectiveConfig(\n",
    "    objective_config=vertex_ai_beta.ModelMonitoringObjectiveConfig(\n",
    "        training_dataset=training_dataset,\n",
    "        training_prediction_skew_detection_config=skew_config,\n",
    "        prediction_drift_detection_config=drift_config,\n",
    "    )\n",
    ")\n",
    "\n",
    "deployment_objective_configs = []\n",
    "for model_id in model_ids:\n",
    "    objective_config = copy.deepcopy(objective_template)\n",
    "    objective_config.deployed_model_id = model_id\n",
    "    deployment_objective_configs.append(objective_config)\n",
    "\n",
    "alerting_config = vertex_ai_beta.ModelMonitoringAlertConfig(\n",
    "    email_alert_config=vertex_ai_beta.ModelMonitoringAlertConfig.EmailAlertConfig(\n",
    "        user_emails=NOTIFY_EMAILS\n",
    "    )\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7311422c",
   "metadata": {},
   "source": [
    "### Instantiate a monitoring job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b414c32",
   "metadata": {},
   "outputs": [],
   "source": [
    "job = vertex_ai_beta.ModelDeploymentMonitoringJob(\n",
    "    display_name=MONITORING_JOB_NAME,\n",
    "    endpoint=endpoint_uri,\n",
    "    model_deployment_monitoring_objective_configs=deployment_objective_configs,\n",
    "    logging_sampling_strategy=sampling_config,\n",
    "    model_deployment_monitoring_schedule_config=schedule_config,\n",
    "    model_monitoring_alert_config=alerting_config,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e87fd5c",
   "metadata": {},
   "source": [
    "### Submit the job for creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7d54b6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "response = job_client_beta.create_model_deployment_monitoring_job(\n",
    "    parent=PARENT, model_deployment_monitoring_job=job\n",
    ")\n",
    "response"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c8d2120",
   "metadata": {},
   "source": [
    "## 3. List Monitoring Jobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef38d00d",
   "metadata": {},
   "outputs": [],
   "source": [
    "monitoring_jobs = job_client_beta.list_model_deployment_monitoring_jobs(parent=PARENT)\n",
    "monitoring_job = [entry for entry in monitoring_jobs if entry.display_name == MONITORING_JOB_NAME][0]\n",
    "monitoring_job"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb136f64",
   "metadata": {},
   "source": [
    "## 4. Simulate skewed prediction requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07ff9ab8",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_requests = 100\n",
    "\n",
    "print(\"Simulation started...\")\n",
    "for idx in range(num_requests):\n",
    "    request = [{\n",
    "        \"dropoff_grid\": [\"POINT(-87.6 41.9)\"],\n",
    "        \"euclidean\": [2064.2696],\n",
    "        \"loc_cross\": [\"\"],\n",
    "        \"payment_type\": [\"Credit Card\"],\n",
    "        \"pickup_grid\": [\"POINT(-87.6 41.9)\"],\n",
    "        \"trip_miles\": [1.37],\n",
    "        \"trip_day\": [int(random.uniform(10, 50))],\n",
    "        \"trip_hour\": [int(random.uniform(10, 50))],\n",
    "        \"trip_month\": [int(random.uniform(1, 10))],\n",
    "        \"trip_day_of_week\": [int(random.uniform(1, 7))],\n",
    "        \"trip_seconds\": [int(random.uniform(60, 600))]\n",
    "    }]\n",
    "    \n",
    "    endpoint.predict(request)\n",
    "    time.sleep(0.5)\n",
    "    \n",
    "    if idx % 10 == 0:\n",
    "        print(f'{idx + 1} of {num_requests} prediction requests were invoked.')\n",
    "print(\"Simulation finished.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06a03835",
   "metadata": {},
   "source": [
    "## 5. Pause Monitoring Job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e4ba104",
   "metadata": {},
   "outputs": [],
   "source": [
    "job_client_beta.pause_model_deployment_monitoring_job(name=monitoring_job.name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8fb6f259",
   "metadata": {},
   "source": [
    "## Delete Monitoring Job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4668f9dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "job_client_beta.delete_model_deployment_monitoring_job(name=monitoring_job.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac101746",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "python3",
   "name": "common-cpu.m87",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/base-cpu:m87"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}