SDV.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "74223d8a",
   "metadata": {},
   "source": [
    "# Synthetic Data Vault (SDV)\n",
    "In this notebook, we will use Synthetic Data Vault (SDV) to create sequential synthetic data. The foundation of this notebook is the [tutorial from SDV](https://colab.research.google.com/drive/1cT4-jFK2Bxe93QudC_CwHq_yVCcNcxal?usp=sharing) and the notebook on DataSynthesizer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "091b3869",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import relevant modules\n",
    "import sdv\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76460230",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "156f93cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['unit_nr', 'timecycle', 'ops_set1', 'ops_set2', 'ops_set3', 'sens_1',\n",
       "       'sens_2', 'sens_3', 'sens_4', 'sens_5', 'sens_6', 'sens_7', 'sens_8',\n",
       "       'sens_9', 'sens_10', 'sens_11', 'sens_12', 'sens_13', 'sens_14',\n",
       "       'sens_15', 'sens_16', 'sens_17', 'sens_18', 'sens_19', 'sens_20',\n",
       "       'sens_21', 'RUL'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('CMAPSS/train_FD001.txt', sep=\" \", header=None)\n",
    "\n",
    "# drop last two columns with N/A values\n",
    "data = data.iloc[:, :-2]\n",
    "\n",
    "# rename columns according to readme.txt\n",
    "col_names = [\"unit_nr\", \"timecycle\", \"ops_set1\", \"ops_set2\", \"ops_set3\"]\n",
    "for i in range(1,22):\n",
    "    col_names.append(f\"sens_{i}\")\n",
    "data.columns = col_names\n",
    "\n",
    "# Compute Remaining Useful Life (RUL) for each index (engine)\n",
    "def add_remaining_useful_life(df):\n",
    "    # Get the total number of cycles for each unit\n",
    "    grouped_by_unit = df.groupby(by=\"unit_nr\")\n",
    "    max_cycle = grouped_by_unit[\"timecycle\"].max()\n",
    "    \n",
    "    # Merge the max cycle back into the original frame\n",
    "    result_frame = df.merge(max_cycle.to_frame(name='max_cycle'), left_on='unit_nr', right_index=True)\n",
    "    \n",
    "    # Calculate remaining useful life for each row\n",
    "    remaining_useful_life = result_frame[\"max_cycle\"] - result_frame[\"timecycle\"]\n",
    "    result_frame[\"RUL\"] = remaining_useful_life\n",
    "    \n",
    "    # drop max_cycle as it's no longer needed\n",
    "    result_frame = result_frame.drop(\"max_cycle\", axis=1)\n",
    "    return result_frame\n",
    "\n",
    "data = add_remaining_useful_life(data)\n",
    "\n",
    "data.to_csv('CMAPSS/train_FD001_pre.csv', index=False)\n",
    "\n",
    "data_length = len(data)\n",
    "\n",
    "# display data\n",
    "data\n",
    "data.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "096ea575",
   "metadata": {},
   "source": [
    "# Create synthetic data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c6e40bde",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create metadata\n",
    "from sdv.metadata import SingleTableMetadata\n",
    "\n",
    "metadata = SingleTableMetadata()\n",
    "\n",
    "# load data\n",
    "metadata.detect_from_dataframe(data=data)\n",
    "\n",
    "# set unit-nr as sequence key, timecycle as sequence index\n",
    "metadata.update_column(\n",
    "    column_name='unit_nr',\n",
    "    sdtype='id')\n",
    "metadata.set_sequence_key('unit_nr')\n",
    "metadata.set_sequence_index('timecycle')\n",
    "\n",
    "# show metadata\n",
    "# metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "67671337",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 100 | Loss -0.4249756932258606: 100%|███| 100/100 [02:08<00:00,  1.29s/it]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "In this example, we do not need context_columns.\n",
    "As all columns have an alternating value, none\n",
    "remains constant or context-dependent.\n",
    "\n",
    "\"\"\"\n",
    "from sdv.sequential import PARSynthesizer\n",
    "\n",
    "# create synthesizer\n",
    "synthesizer = PARSynthesizer(\n",
    "    metadata,\n",
    "    epochs=100,\n",
    "    verbose=True\n",
    "    )\n",
    "\n",
    "synthesizer.fit(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c17d5e97",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 100/100 [01:11<00:00,  1.41it/s]\n"
     ]
    }
   ],
   "source": [
    "# find amount of unique unit-nrs\n",
    "n_unitnrs = len(data['unit_nr'].unique())\n",
    "\n",
    "# create synthetic data with that same amount of unit-nrs\n",
    "synthetic_data = synthesizer.sample(num_sequences=n_unitnrs)\n",
    "synthetic_data\n",
    "\n",
    "# export SyntData to folder\n",
    "synthetic_data.to_csv('./CMAPSS/Synthetic/SDV_FD001_RUL.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dfc8bc8e",
   "metadata": {},
   "source": [
    "# Analysis\n",
    "Cool! We've created our first synthetic data, now we wish to analyse this synthetic data. More specifically, we would like to compare the output and distributions of our synthetic data compared to the original data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0dc6db1b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>unit_nr</th>\n",
       "      <th>timecycle</th>\n",
       "      <th>ops_set1</th>\n",
       "      <th>ops_set2</th>\n",
       "      <th>ops_set3</th>\n",
       "      <th>sens_1</th>\n",
       "      <th>sens_2</th>\n",
       "      <th>sens_3</th>\n",
       "      <th>sens_4</th>\n",
       "      <th>sens_5</th>\n",
       "      <th>...</th>\n",
       "      <th>sens_13</th>\n",
       "      <th>sens_14</th>\n",
       "      <th>sens_15</th>\n",
       "      <th>sens_16</th>\n",
       "      <th>sens_17</th>\n",
       "      <th>sens_18</th>\n",
       "      <th>sens_19</th>\n",
       "      <th>sens_20</th>\n",
       "      <th>sens_21</th>\n",
       "      <th>RUL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.0</td>\n",
       "      <td>20631.00</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>2.063100e+04</td>\n",
       "      <td>...</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>2.063100e+04</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.0</td>\n",
       "      <td>20631.0</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>51.506568</td>\n",
       "      <td>108.807862</td>\n",
       "      <td>-0.000009</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.680934</td>\n",
       "      <td>1590.523119</td>\n",
       "      <td>1408.933782</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>2388.096152</td>\n",
       "      <td>8143.752722</td>\n",
       "      <td>8.442146</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>393.210654</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.816271</td>\n",
       "      <td>23.289705</td>\n",
       "      <td>107.807862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>29.227633</td>\n",
       "      <td>68.880990</td>\n",
       "      <td>0.002187</td>\n",
       "      <td>0.000293</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.500053</td>\n",
       "      <td>6.131150</td>\n",
       "      <td>9.000605</td>\n",
       "      <td>1.776400e-15</td>\n",
       "      <td>...</td>\n",
       "      <td>0.071919</td>\n",
       "      <td>19.076176</td>\n",
       "      <td>0.037505</td>\n",
       "      <td>1.387812e-17</td>\n",
       "      <td>1.548763</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.180746</td>\n",
       "      <td>0.108251</td>\n",
       "      <td>68.880990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.008700</td>\n",
       "      <td>-0.000600</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>641.210000</td>\n",
       "      <td>1571.040000</td>\n",
       "      <td>1382.250000</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>2387.880000</td>\n",
       "      <td>8099.940000</td>\n",
       "      <td>8.324900</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>388.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.140000</td>\n",
       "      <td>22.894200</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>26.000000</td>\n",
       "      <td>52.000000</td>\n",
       "      <td>-0.001500</td>\n",
       "      <td>-0.000200</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.325000</td>\n",
       "      <td>1586.260000</td>\n",
       "      <td>1402.360000</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>2388.040000</td>\n",
       "      <td>8133.245000</td>\n",
       "      <td>8.414900</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>392.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.700000</td>\n",
       "      <td>23.221800</td>\n",
       "      <td>51.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>52.000000</td>\n",
       "      <td>104.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.640000</td>\n",
       "      <td>1590.100000</td>\n",
       "      <td>1408.040000</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>2388.090000</td>\n",
       "      <td>8140.540000</td>\n",
       "      <td>8.438900</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>393.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.830000</td>\n",
       "      <td>23.297900</td>\n",
       "      <td>103.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>77.000000</td>\n",
       "      <td>156.000000</td>\n",
       "      <td>0.001500</td>\n",
       "      <td>0.000300</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>643.000000</td>\n",
       "      <td>1594.380000</td>\n",
       "      <td>1414.555000</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>2388.140000</td>\n",
       "      <td>8148.310000</td>\n",
       "      <td>8.465600</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>394.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.950000</td>\n",
       "      <td>23.366800</td>\n",
       "      <td>155.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>362.000000</td>\n",
       "      <td>0.008700</td>\n",
       "      <td>0.000600</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>644.530000</td>\n",
       "      <td>1616.910000</td>\n",
       "      <td>1441.490000</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>2388.560000</td>\n",
       "      <td>8293.720000</td>\n",
       "      <td>8.584800</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>400.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>39.430000</td>\n",
       "      <td>23.618400</td>\n",
       "      <td>361.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            unit_nr     timecycle      ops_set1      ops_set2  ops_set3  \\\n",
       "count  20631.000000  20631.000000  20631.000000  20631.000000   20631.0   \n",
       "mean      51.506568    108.807862     -0.000009      0.000002     100.0   \n",
       "std       29.227633     68.880990      0.002187      0.000293       0.0   \n",
       "min        1.000000      1.000000     -0.008700     -0.000600     100.0   \n",
       "25%       26.000000     52.000000     -0.001500     -0.000200     100.0   \n",
       "50%       52.000000    104.000000      0.000000      0.000000     100.0   \n",
       "75%       77.000000    156.000000      0.001500      0.000300     100.0   \n",
       "max      100.000000    362.000000      0.008700      0.000600     100.0   \n",
       "\n",
       "         sens_1        sens_2        sens_3        sens_4        sens_5  ...  \\\n",
       "count  20631.00  20631.000000  20631.000000  20631.000000  2.063100e+04  ...   \n",
       "mean     518.67    642.680934   1590.523119   1408.933782  1.462000e+01  ...   \n",
       "std        0.00      0.500053      6.131150      9.000605  1.776400e-15  ...   \n",
       "min      518.67    641.210000   1571.040000   1382.250000  1.462000e+01  ...   \n",
       "25%      518.67    642.325000   1586.260000   1402.360000  1.462000e+01  ...   \n",
       "50%      518.67    642.640000   1590.100000   1408.040000  1.462000e+01  ...   \n",
       "75%      518.67    643.000000   1594.380000   1414.555000  1.462000e+01  ...   \n",
       "max      518.67    644.530000   1616.910000   1441.490000  1.462000e+01  ...   \n",
       "\n",
       "            sens_13       sens_14       sens_15       sens_16       sens_17  \\\n",
       "count  20631.000000  20631.000000  20631.000000  2.063100e+04  20631.000000   \n",
       "mean    2388.096152   8143.752722      8.442146  3.000000e-02    393.210654   \n",
       "std        0.071919     19.076176      0.037505  1.387812e-17      1.548763   \n",
       "min     2387.880000   8099.940000      8.324900  3.000000e-02    388.000000   \n",
       "25%     2388.040000   8133.245000      8.414900  3.000000e-02    392.000000   \n",
       "50%     2388.090000   8140.540000      8.438900  3.000000e-02    393.000000   \n",
       "75%     2388.140000   8148.310000      8.465600  3.000000e-02    394.000000   \n",
       "max     2388.560000   8293.720000      8.584800  3.000000e-02    400.000000   \n",
       "\n",
       "       sens_18  sens_19       sens_20       sens_21           RUL  \n",
       "count  20631.0  20631.0  20631.000000  20631.000000  20631.000000  \n",
       "mean    2388.0    100.0     38.816271     23.289705    107.807862  \n",
       "std        0.0      0.0      0.180746      0.108251     68.880990  \n",
       "min     2388.0    100.0     38.140000     22.894200      0.000000  \n",
       "25%     2388.0    100.0     38.700000     23.221800     51.000000  \n",
       "50%     2388.0    100.0     38.830000     23.297900    103.000000  \n",
       "75%     2388.0    100.0     38.950000     23.366800    155.000000  \n",
       "max     2388.0    100.0     39.430000     23.618400    361.000000  \n",
       "\n",
       "[8 rows x 27 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_description = data.describe()\n",
    "data_description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5180fc20",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>unit_nr</th>\n",
       "      <th>timecycle</th>\n",
       "      <th>ops_set1</th>\n",
       "      <th>ops_set2</th>\n",
       "      <th>ops_set3</th>\n",
       "      <th>sens_1</th>\n",
       "      <th>sens_2</th>\n",
       "      <th>sens_3</th>\n",
       "      <th>sens_4</th>\n",
       "      <th>sens_5</th>\n",
       "      <th>...</th>\n",
       "      <th>sens_13</th>\n",
       "      <th>sens_14</th>\n",
       "      <th>sens_15</th>\n",
       "      <th>sens_16</th>\n",
       "      <th>sens_17</th>\n",
       "      <th>sens_18</th>\n",
       "      <th>sens_19</th>\n",
       "      <th>sens_20</th>\n",
       "      <th>sens_21</th>\n",
       "      <th>RUL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>18564.000000</td>\n",
       "      <td>18564.000000</td>\n",
       "      <td>18564.000000</td>\n",
       "      <td>18564.000000</td>\n",
       "      <td>18564.0</td>\n",
       "      <td>18564.00</td>\n",
       "      <td>18564.000000</td>\n",
       "      <td>18564.000000</td>\n",
       "      <td>18564.000000</td>\n",
       "      <td>1.856400e+04</td>\n",
       "      <td>...</td>\n",
       "      <td>18564.000000</td>\n",
       "      <td>18564.000000</td>\n",
       "      <td>18564.000000</td>\n",
       "      <td>1.856400e+04</td>\n",
       "      <td>18564.000000</td>\n",
       "      <td>18564.0</td>\n",
       "      <td>18564.0</td>\n",
       "      <td>18564.000000</td>\n",
       "      <td>18564.000000</td>\n",
       "      <td>18564.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>52300.959491</td>\n",
       "      <td>101.884292</td>\n",
       "      <td>-0.000005</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.752021</td>\n",
       "      <td>1591.245154</td>\n",
       "      <td>1410.282246</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>2388.101472</td>\n",
       "      <td>8146.460691</td>\n",
       "      <td>8.446927</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>393.414727</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.796656</td>\n",
       "      <td>23.273122</td>\n",
       "      <td>98.710946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>30224.243654</td>\n",
       "      <td>69.723950</td>\n",
       "      <td>0.001011</td>\n",
       "      <td>0.000149</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.285639</td>\n",
       "      <td>3.321873</td>\n",
       "      <td>5.296097</td>\n",
       "      <td>3.552809e-15</td>\n",
       "      <td>...</td>\n",
       "      <td>0.035969</td>\n",
       "      <td>11.956419</td>\n",
       "      <td>0.020248</td>\n",
       "      <td>1.040862e-17</td>\n",
       "      <td>0.917487</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.101342</td>\n",
       "      <td>0.059943</td>\n",
       "      <td>33.933783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>297.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.003940</td>\n",
       "      <td>-0.000590</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>641.606238</td>\n",
       "      <td>1577.999235</td>\n",
       "      <td>1390.250104</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>2387.948887</td>\n",
       "      <td>8099.940000</td>\n",
       "      <td>8.363616</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>390.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.366522</td>\n",
       "      <td>23.052963</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>20604.000000</td>\n",
       "      <td>47.000000</td>\n",
       "      <td>-0.000668</td>\n",
       "      <td>-0.000093</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.563708</td>\n",
       "      <td>1589.167107</td>\n",
       "      <td>1406.853930</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>2388.080701</td>\n",
       "      <td>8138.860253</td>\n",
       "      <td>8.435490</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>393.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.730902</td>\n",
       "      <td>23.233326</td>\n",
       "      <td>76.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>57382.000000</td>\n",
       "      <td>93.000000</td>\n",
       "      <td>-0.000009</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.741310</td>\n",
       "      <td>1590.986394</td>\n",
       "      <td>1410.020844</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>2388.096152</td>\n",
       "      <td>8145.760799</td>\n",
       "      <td>8.443472</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>393.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.804221</td>\n",
       "      <td>23.276960</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>75683.000000</td>\n",
       "      <td>141.000000</td>\n",
       "      <td>0.000660</td>\n",
       "      <td>0.000100</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.940499</td>\n",
       "      <td>1593.400952</td>\n",
       "      <td>1413.857345</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>2388.123623</td>\n",
       "      <td>8154.348230</td>\n",
       "      <td>8.459784</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>394.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.859594</td>\n",
       "      <td>23.311051</td>\n",
       "      <td>120.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>99455.000000</td>\n",
       "      <td>362.000000</td>\n",
       "      <td>0.004212</td>\n",
       "      <td>0.000584</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>643.919006</td>\n",
       "      <td>1606.104940</td>\n",
       "      <td>1431.071379</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>2388.285619</td>\n",
       "      <td>8209.460559</td>\n",
       "      <td>8.541943</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>397.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>39.212197</td>\n",
       "      <td>23.503433</td>\n",
       "      <td>234.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            unit_nr     timecycle      ops_set1      ops_set2  ops_set3  \\\n",
       "count  18564.000000  18564.000000  18564.000000  18564.000000   18564.0   \n",
       "mean   52300.959491    101.884292     -0.000005      0.000004     100.0   \n",
       "std    30224.243654     69.723950      0.001011      0.000149       0.0   \n",
       "min      297.000000      1.000000     -0.003940     -0.000590     100.0   \n",
       "25%    20604.000000     47.000000     -0.000668     -0.000093     100.0   \n",
       "50%    57382.000000     93.000000     -0.000009      0.000002     100.0   \n",
       "75%    75683.000000    141.000000      0.000660      0.000100     100.0   \n",
       "max    99455.000000    362.000000      0.004212      0.000584     100.0   \n",
       "\n",
       "         sens_1        sens_2        sens_3        sens_4        sens_5  ...  \\\n",
       "count  18564.00  18564.000000  18564.000000  18564.000000  1.856400e+04  ...   \n",
       "mean     518.67    642.752021   1591.245154   1410.282246  1.462000e+01  ...   \n",
       "std        0.00      0.285639      3.321873      5.296097  3.552809e-15  ...   \n",
       "min      518.67    641.606238   1577.999235   1390.250104  1.462000e+01  ...   \n",
       "25%      518.67    642.563708   1589.167107   1406.853930  1.462000e+01  ...   \n",
       "50%      518.67    642.741310   1590.986394   1410.020844  1.462000e+01  ...   \n",
       "75%      518.67    642.940499   1593.400952   1413.857345  1.462000e+01  ...   \n",
       "max      518.67    643.919006   1606.104940   1431.071379  1.462000e+01  ...   \n",
       "\n",
       "            sens_13       sens_14       sens_15       sens_16       sens_17  \\\n",
       "count  18564.000000  18564.000000  18564.000000  1.856400e+04  18564.000000   \n",
       "mean    2388.101472   8146.460691      8.446927  3.000000e-02    393.414727   \n",
       "std        0.035969     11.956419      0.020248  1.040862e-17      0.917487   \n",
       "min     2387.948887   8099.940000      8.363616  3.000000e-02    390.000000   \n",
       "25%     2388.080701   8138.860253      8.435490  3.000000e-02    393.000000   \n",
       "50%     2388.096152   8145.760799      8.443472  3.000000e-02    393.000000   \n",
       "75%     2388.123623   8154.348230      8.459784  3.000000e-02    394.000000   \n",
       "max     2388.285619   8209.460559      8.541943  3.000000e-02    397.000000   \n",
       "\n",
       "       sens_18  sens_19       sens_20       sens_21           RUL  \n",
       "count  18564.0  18564.0  18564.000000  18564.000000  18564.000000  \n",
       "mean    2388.0    100.0     38.796656     23.273122     98.710946  \n",
       "std        0.0      0.0      0.101342      0.059943     33.933783  \n",
       "min     2388.0    100.0     38.366522     23.052963      0.000000  \n",
       "25%     2388.0    100.0     38.730902     23.233326     76.000000  \n",
       "50%     2388.0    100.0     38.804221     23.276960    100.000000  \n",
       "75%     2388.0    100.0     38.859594     23.311051    120.000000  \n",
       "max     2388.0    100.0     39.212197     23.503433    234.000000  \n",
       "\n",
       "[8 rows x 27 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "syn_description = synthetic_data.describe()\n",
    "syn_description"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6572767f",
   "metadata": {},
   "source": [
    "What directly stands out is that the unit-nrs are not ordered from 1-100. However, this is due to the very nature of SDV: it will never create instances that overlap with the original data [source](https://colab.research.google.com/drive/1YLk2uwn8yrSRPy0soEeJwu8Hdk_tGTlE#scrollTo=IjvfTOMVKQot). Therefore, this behaviour is to be expected.\n",
    "\n",
    "The descriptions look rather similar, most means and standarddeviations are close to the description of the original data. Therefore it would be interesting to test the SDV data on a ML task."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63e40533",
   "metadata": {},
   "source": [
    "## SDV evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "69aaf808",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating report ...\n",
      "(1/2) Evaluating Column Shapes: : 100%|████████| 27/27 [00:00<00:00, 122.04it/s]\n",
      "(2/2) Evaluating Column Pair Trends: : 100%|█| 351/351 [00:00<00:00, 510.20it/s]\n",
      "\n",
      "Overall Quality Score: 80.91%\n",
      "\n",
      "Properties:\n",
      "- Column Shapes: 82.84%\n",
      "- Column Pair Trends: 78.98%\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Overall Quality Score: 81.07%\\n\\nProperties:\\n- Column Shapes: 82.68%\\n- Column Pair Trends: 79.45%\\n'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sdv.evaluation.single_table import evaluate_quality\n",
    "\n",
    "quality_report = evaluate_quality(\n",
    "    data,\n",
    "    synthetic_data,\n",
    "    metadata\n",
    ")\n",
    "\n",
    "# with 128 epochs:\n",
    "\"\"\"Overall Quality Score: 81.07%\n",
    "\n",
    "Properties:\n",
    "- Column Shapes: 82.68%\n",
    "- Column Pair Trends: 79.45%\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d56c9f37",
   "metadata": {},
   "source": [
    "# Testing properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e33bef6c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}