Spark DataFrameReader API.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "f14f0bba-e192-4d59-bcad-e6070d71964c",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "\n",
    "def load_survey_df(spark, data_file):\n",
    "        # header uses first line as name of columns \n",
    "        # infers input schema automatically from data. reqiores one extra pass over data\n",
    "        return spark.read\\\n",
    "                .option(\"header\", \"true\")\\ \n",
    "                .option(\"inferSchema\", \"true\")\\\n",
    "                .csv(data_file)\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "        spark = SparkSession.builder\\\n",
    "                .appName(\"Hello Spark\")\\\n",
    "                .master(\"local[3]\")\\\n",
    "                .getOrCreate()\n",
    "\n",
    "        # spark.stop()\n",
    "\n",
    "        survey_df = load_survey_df(spark, \"/databricks-datasets/Rdatasets/data-001/csv/ggplot2/diamonds.csv\")\n",
    "\n",
    "        survey_df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "8b68e52f-c200-48a8-96fd-d27a1b19d912",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "\n",
    "spark = SparkSession\\\n",
    "            .builder\\\n",
    "            .appName(\"SparkDemo\")\\\n",
    "            .getOrCreate()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "3fca8cbd-e4bf-4d00-909b-084556a2b72a",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "airbnb_df = spark.read\\\n",
    "    .format(\"csv\")\\\n",
    "        .option(\"header\",\"true\")\\\n",
    "            .option(\"inferSchema\", \"true\")\\\n",
    "                .load(\"/databricks-datasets/learning-spark-v2/sf-airbnb/sf-airbnb.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "18c60abf-80e4-476b-9817-dd6dafb0e887",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "display(airbnb_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "1920718a-743f-49ac-a5e7-7d66bc0e440d",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "airbnb_df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "65aa4b53-3c5b-4e1c-bcdf-3085cb0dac87",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# json file format doen't comes with header and infers schema by default\n",
    "summary_df = spark.read\\\n",
    "    .format(\"json\")\\\n",
    "        .load(\"/databricks-datasets/learning-spark-v2/flights/summary-data/json/2015-summary.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "b860d27a-2bb8-4166-a82c-d492225f7a81",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "display(uspop_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "062c9901-44f1-46b7-b448-2457a3371e02",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "uspop_df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "61b110a9-03d2-447a-9ff1-36d613146245",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "loan_df = spark.read\\\n",
    "    .format(\"parquet\")\\\n",
    "        .load(\"/databricks-datasets/learning-spark-v2/loans/loan-risks.snappy.parquet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "0520e6e3-f434-4efd-aea3-c274e2f05106",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "display(loan_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "8aa65bff-77bf-43f9-b04d-ac74665a64fc",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "loan_df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "fca8c6aa-aab0-4c3d-8e0e-42882cd41008",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "from pyspark.sql.types import *\n",
    "\n",
    "# programmatic method to define schemas explicitly\n",
    "summarySchemaStruct = StructType([\n",
    "    StructField(\"DEST_COUNTRY_NAME\", StringType()),\n",
    "    StructField(\"ORIGIN_COUNTRY_NAME\", StringType()),\n",
    "    StructField(\"count\", IntegerType())\n",
    "])\n",
    "\n",
    "summ_df = spark.read\\\n",
    "    .format(\"csv\")\\\n",
    "        .option(\"header\",\"true\")\\\n",
    "            .schema(summarySchemaStruct)\\\n",
    "                .option(\"mode\",\"FAILFAST\")\\\n",
    "                    .load(\"/databricks-datasets/learning-spark-v2/flights/summary-data/csv/2015-summary.csv\")\n",
    "\n",
    "summ_df.show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "3c7a1681-9da4-4d30-bc2d-98cc46bd81af",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# DDL method to define schemas explicitly\n",
    "\n",
    "summarySchemaDDL = \"\"\" DEST_COUNTRY_NAME STRING, ORIGIN_COUNTRY_NAME STRING, count INT\"\"\"\n",
    "\n",
    "summ_df = spark.read\\\n",
    "    .format(\"csv\")\\\n",
    "        .option(\"header\",\"true\")\\\n",
    "            .schema(summarySchemaDDL)\\\n",
    "                .option(\"mode\",\"FAILFAST\")\\\n",
    "                    .load(\"/databricks-datasets/learning-spark-v2/flights/summary-data/csv/2015-summary.csv\")\n",
    "\n",
    "summ_df.show(10)"
   ]
  }
 ],
 "metadata": {
  "application/vnd.databricks.v1+notebook": {
   "dashboards": [],
   "environmentMetadata": null,
   "language": "python",
   "notebookMetadata": {
    "pythonIndentUnit": 4
   },
   "notebookName": "Spark DataFrameReader API",
   "widgets": {}
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}