Generating Example Data - Air Quality.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fb7f6cb3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "OpenBLAS WARNING - could not determine the L2 cache size on this system, assuming 256k\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'Faker' object has no attribute 'user_name'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_1336/3407003262.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     17\u001b[0m })\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m \u001b[0mgenerator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgenerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0morder_dataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgenerator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.7/site-packages/datafuzz/generators/core.py\u001b[0m in \u001b[0;36mgenerate\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     86\u001b[0m         \"\"\"\n\u001b[1;32m     87\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimeseries\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgenerate_timeseries\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.7/site-packages/datafuzz/generators/core.py\u001b[0m in \u001b[0;36mgenerate_timeseries\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    140\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    141\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart_time\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparser\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 142\u001b[0;31m             \u001b[0mrow\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgenerate_row\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    143\u001b[0m             \u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'timestamp'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstart_time\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misoformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecords\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.7/site-packages/datafuzz/generators/core.py\u001b[0m in \u001b[0;36mgenerate_row\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    111\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfield_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m'faker.'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfield_val\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    112\u001b[0m                 \u001b[0mfield_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfield_val\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'faker.'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 113\u001b[0;31m                 \u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfield_name\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfake\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfield_val\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    114\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfield_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    115\u001b[0m                 \u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfield_name\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchoice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfield_val\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.7/site-packages/faker/proxy.py\u001b[0m in \u001b[0;36m__getattribute__\u001b[0;34m(self, attr)\u001b[0m\n\u001b[1;32m     99\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'Faker' object has no attribute 'user_name'"
     ]
    }
   ],
   "source": [
    "from datetime import datetime, timedelta\n",
    "from datafuzz.generators import DatasetGenerator\n",
    "\n",
    "generator = DatasetGenerator({\n",
    "    'output': 'pandas',\n",
    "    'schema': {\n",
    "        'air_quality_index': range(0, 400, 20),\n",
    "        'user_id': 'faker.user_name',\n",
    "        'location': 'faker.location_on_land',\n",
    "        'top_pollutant': ['PM2.5', 'PM10', 'O3', 'NO2', 'SO2', 'CO'],\n",
    "        'top_pollutant_concentration': range(1, 100)\n",
    "    },\n",
    "    'num_rows': 100000,\n",
    "    'start_time': datetime.now() - timedelta(days=20),\n",
    "    'output': 'dataset',\n",
    "    'increments': 'seconds',\n",
    "})\n",
    "\n",
    "generator.generate()\n",
    "\n",
    "order_dataset = generator.to_output()\n",
    "order_dataset.records.to_csv('data/air_quality.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e05ca00e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>air_quality_index</th>\n",
       "      <th>user_id</th>\n",
       "      <th>location</th>\n",
       "      <th>top_pollutant</th>\n",
       "      <th>top_pollutant_concentration</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>200</td>\n",
       "      <td>maynardmorgan</td>\n",
       "      <td>('10.66668', '-61.51889', 'Port of Spain', 'TT...</td>\n",
       "      <td>PM10</td>\n",
       "      <td>88</td>\n",
       "      <td>2022-04-07T14:38:34.869188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>200</td>\n",
       "      <td>millerjacqueline</td>\n",
       "      <td>('34.25628', '-78.04471', 'Leland', 'US', 'Ame...</td>\n",
       "      <td>PM10</td>\n",
       "      <td>45</td>\n",
       "      <td>2022-04-07T14:39:16.869188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>ikirk</td>\n",
       "      <td>('43.48961', '-8.2194', 'Ferrol', 'ES', 'Europ...</td>\n",
       "      <td>NO2</td>\n",
       "      <td>31</td>\n",
       "      <td>2022-04-07T14:39:51.869188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>300</td>\n",
       "      <td>williamszachary</td>\n",
       "      <td>('42.06166', '-1.60452', 'Tudela', 'ES', 'Euro...</td>\n",
       "      <td>CO</td>\n",
       "      <td>15</td>\n",
       "      <td>2022-04-07T14:40:34.869188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>280</td>\n",
       "      <td>sfarrell</td>\n",
       "      <td>('-28.24', '-48.67028', 'Imbituba', 'BR', 'Ame...</td>\n",
       "      <td>PM2.5</td>\n",
       "      <td>62</td>\n",
       "      <td>2022-04-07T14:41:12.869188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>220</td>\n",
       "      <td>phenson</td>\n",
       "      <td>('46.1351', '-60.1831', 'Sydney', 'CA', 'Ameri...</td>\n",
       "      <td>CO</td>\n",
       "      <td>99</td>\n",
       "      <td>2022-05-18T04:48:08.869188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>0</td>\n",
       "      <td>richardwatson</td>\n",
       "      <td>('45.43341', '-73.86586', 'Beaconsfield', 'CA'...</td>\n",
       "      <td>NO2</td>\n",
       "      <td>64</td>\n",
       "      <td>2022-05-18T04:48:39.869188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>80</td>\n",
       "      <td>joseph60</td>\n",
       "      <td>('-8.66667', '-35.71667', 'Catende', 'BR', 'Am...</td>\n",
       "      <td>NO2</td>\n",
       "      <td>35</td>\n",
       "      <td>2022-05-18T04:49:07.869188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>40</td>\n",
       "      <td>craigelijah</td>\n",
       "      <td>('12.37169', '123.62494', 'Masbate', 'PH', 'As...</td>\n",
       "      <td>NO2</td>\n",
       "      <td>34</td>\n",
       "      <td>2022-05-18T04:49:32.869188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>340</td>\n",
       "      <td>monicatrujillo</td>\n",
       "      <td>('23.73333', '114.68333', 'Heyuan', 'CN', 'Asi...</td>\n",
       "      <td>PM2.5</td>\n",
       "      <td>52</td>\n",
       "      <td>2022-05-18T04:49:54.869188</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100000 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       air_quality_index           user_id  \\\n",
       "0                    200     maynardmorgan   \n",
       "1                    200  millerjacqueline   \n",
       "2                      0             ikirk   \n",
       "3                    300   williamszachary   \n",
       "4                    280          sfarrell   \n",
       "...                  ...               ...   \n",
       "99995                220           phenson   \n",
       "99996                  0     richardwatson   \n",
       "99997                 80          joseph60   \n",
       "99998                 40       craigelijah   \n",
       "99999                340    monicatrujillo   \n",
       "\n",
       "                                                location top_pollutant  \\\n",
       "0      ('10.66668', '-61.51889', 'Port of Spain', 'TT...          PM10   \n",
       "1      ('34.25628', '-78.04471', 'Leland', 'US', 'Ame...          PM10   \n",
       "2      ('43.48961', '-8.2194', 'Ferrol', 'ES', 'Europ...           NO2   \n",
       "3      ('42.06166', '-1.60452', 'Tudela', 'ES', 'Euro...            CO   \n",
       "4      ('-28.24', '-48.67028', 'Imbituba', 'BR', 'Ame...         PM2.5   \n",
       "...                                                  ...           ...   \n",
       "99995  ('46.1351', '-60.1831', 'Sydney', 'CA', 'Ameri...            CO   \n",
       "99996  ('45.43341', '-73.86586', 'Beaconsfield', 'CA'...           NO2   \n",
       "99997  ('-8.66667', '-35.71667', 'Catende', 'BR', 'Am...           NO2   \n",
       "99998  ('12.37169', '123.62494', 'Masbate', 'PH', 'As...           NO2   \n",
       "99999  ('23.73333', '114.68333', 'Heyuan', 'CN', 'Asi...         PM2.5   \n",
       "\n",
       "       top_pollutant_concentration                   timestamp  \n",
       "0                               88  2022-04-07T14:38:34.869188  \n",
       "1                               45  2022-04-07T14:39:16.869188  \n",
       "2                               31  2022-04-07T14:39:51.869188  \n",
       "3                               15  2022-04-07T14:40:34.869188  \n",
       "4                               62  2022-04-07T14:41:12.869188  \n",
       "...                            ...                         ...  \n",
       "99995                           99  2022-05-18T04:48:08.869188  \n",
       "99996                           64  2022-05-18T04:48:39.869188  \n",
       "99997                           35  2022-05-18T04:49:07.869188  \n",
       "99998                           34  2022-05-18T04:49:32.869188  \n",
       "99999                           52  2022-05-18T04:49:54.869188  \n",
       "\n",
       "[100000 rows x 6 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "pd.read_csv('data/air_quality.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b91098e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}