analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "import polars as pl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = './volumes/output.csv'\n",
    "columns=['timestamp', 'job_id', 'position', 'location', 'job', 'company', 'attempted', 'result']\n",
    "\n",
    "df = pl.read_csv(path,new_columns=columns, has_header=False,try_parse_dates=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.filter(pl.col(\"result\") ==True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = './volumes/output2.csv'\n",
    "df.write_csv(file=path,include_header=False,datetime_format='%Y-%m-%d %H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method DataFrame.describe of shape: (2_414, 8)\n",
       "┌────────────┬────────────┬────────────┬────────────┬────────────┬────────────┬───────────┬────────┐\n",
       "│ timestamp  ┆ job_id     ┆ position   ┆ location   ┆ job        ┆ company    ┆ attempted ┆ result │\n",
       "│ ---        ┆ ---        ┆ ---        ┆ ---        ┆ ---        ┆ ---        ┆ ---       ┆ ---    │\n",
       "│ datetime[μ ┆ i64        ┆ str        ┆ str        ┆ str        ┆ str        ┆ bool      ┆ bool   │\n",
       "│ s]         ┆            ┆            ┆            ┆            ┆            ┆           ┆        │\n",
       "╞════════════╪════════════╪════════════╪════════════╪════════════╪════════════╪═══════════╪════════╡\n",
       "│ 2023-10-24 ┆ 3740116807 ┆ null       ┆ &location= ┆ Data       ┆ Encoretech ┆ true      ┆ true   │\n",
       "│ 04:40:48   ┆            ┆            ┆ European   ┆ Analyst    ┆            ┆           ┆        │\n",
       "│            ┆            ┆            ┆ Economic   ┆            ┆            ┆           ┆        │\n",
       "│            ┆            ┆            ┆ Area       ┆            ┆            ┆           ┆        │\n",
       "│ 2023-10-24 ┆ 3281711737 ┆ null       ┆ &location= ┆ Data       ┆ Kabel      ┆ true      ┆ true   │\n",
       "│ 04:44:40   ┆            ┆            ┆ European   ┆ Consultant ┆            ┆           ┆        │\n",
       "│            ┆            ┆            ┆ Economic   ┆ (Azure,    ┆            ┆           ┆        │\n",
       "│            ┆            ┆            ┆ Area       ┆ Power Bi…  ┆            ┆           ┆        │\n",
       "│ 2023-10-24 ┆ 3705985001 ┆ null       ┆ &location= ┆ Data       ┆ Universal  ┆ true      ┆ true   │\n",
       "│ 04:45:16   ┆            ┆            ┆ European   ┆ Analyst    ┆ Diagnostic ┆           ┆        │\n",
       "│            ┆            ┆            ┆ Economic   ┆            ┆ s S.A.     ┆           ┆        │\n",
       "│            ┆            ┆            ┆ Area       ┆            ┆            ┆           ┆        │\n",
       "│ 2023-10-24 ┆ 3744782669 ┆ null       ┆ &location= ┆ Senior     ┆ Luxoft     ┆ true      ┆ true   │\n",
       "│ 04:47:27   ┆            ┆            ┆ European   ┆ Data       ┆            ┆           ┆        │\n",
       "│            ┆            ┆            ┆ Economic   ┆ Analyst    ┆            ┆           ┆        │\n",
       "│            ┆            ┆            ┆ Area       ┆            ┆            ┆           ┆        │\n",
       "│ …          ┆ …          ┆ …          ┆ …          ┆ …          ┆ …          ┆ …         ┆ …      │\n",
       "│ 2023-11-12 ┆ 3762074670 ┆ Data       ┆ &location= ┆ SATIŞ      ┆ MACRO MH   ┆ true      ┆ true   │\n",
       "│ 14:49:35   ┆            ┆ Scientist  ┆ Turkey     ┆ SONRASI    ┆ MATERIAL   ┆           ┆        │\n",
       "│            ┆            ┆            ┆            ┆ HİZMETLER  ┆ HANDLING   ┆           ┆        │\n",
       "│            ┆            ┆            ┆            ┆ ASİSTANI   ┆            ┆           ┆        │\n",
       "│ 2023-11-12 ┆ 3763664773 ┆ Data       ┆ &location= ┆ Accountant ┆ Datall Co. ┆ true      ┆ true   │\n",
       "│ 14:50:01   ┆            ┆ Scientist  ┆ Turkey     ┆            ┆            ┆           ┆        │\n",
       "│ 2023-11-12 ┆ 3762123810 ┆ Data       ┆ &location= ┆ C sınıfı   ┆ GMK ENDÜST ┆ true      ┆ true   │\n",
       "│ 14:51:47   ┆            ┆ Scientist  ┆ Turkey     ┆ isg veya   ┆ RİYEL      ┆           ┆        │\n",
       "│            ┆            ┆            ┆            ┆ tekniker   ┆ ÜRÜNLER    ┆           ┆        │\n",
       "│            ┆            ┆            ┆            ┆            ┆ TİCARET …  ┆           ┆        │\n",
       "│ 2023-11-12 ┆ 3762105263 ┆ Data       ┆ &location= ┆ Muhasebe   ┆ Serbest    ┆ true      ┆ true   │\n",
       "│ 14:53:35   ┆            ┆ Scientist  ┆ Turkey     ┆ elemanı    ┆ Muhasebeci ┆           ┆        │\n",
       "│            ┆            ┆            ┆            ┆            ┆ Mali       ┆           ┆        │\n",
       "│            ┆            ┆            ┆            ┆            ┆ Müşavir …  ┆           ┆        │\n",
       "└────────────┴────────────┴────────────┴────────────┴────────────┴────────────┴───────────┴────────┘>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr > th,\n",
       ".dataframe > tbody > tr > td {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (0, 8)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>timestamp</th><th>job_id</th><th>position</th><th>location</th><th>job</th><th>company</th><th>attempted</th><th>result</th></tr><tr><td>datetime[μs]</td><td>i64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>bool</td><td>bool</td></tr></thead><tbody></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (0, 8)\n",
       "┌──────────────┬────────┬──────────┬──────────┬─────┬─────────┬───────────┬────────┐\n",
       "│ timestamp    ┆ job_id ┆ position ┆ location ┆ job ┆ company ┆ attempted ┆ result │\n",
       "│ ---          ┆ ---    ┆ ---      ┆ ---      ┆ --- ┆ ---     ┆ ---       ┆ ---    │\n",
       "│ datetime[μs] ┆ i64    ┆ str      ┆ str      ┆ str ┆ str     ┆ bool      ┆ bool   │\n",
       "╞══════════════╪════════╪══════════╪══════════╪═════╪═════════╪═══════════╪════════╡\n",
       "└──────────────┴────────┴──────────┴──────────┴─────┴─────────┴───────────┴────────┘"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df.filter(pl.col('company').str.to_lowercase().str.strip_chars().str.contains('gmk'))\n",
    "df.filter(pl.col('company').str.to_lowercase().str.contains('clarus'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rows: 2414\n",
      "Columns: 8\n",
      "$ timestamp <datetime[μs]> 2023-10-24 04:40:48, 2023-10-24 04:44:40, 2023-10-24 04:45:16, 2023-10-24 04:47:27, 2023-10-24 04:50:09, 2023-10-24 04:50:55, 2023-10-24 04:52:50, 2023-10-24 04:55:15, 2023-10-24 04:57:05, 2023-10-24 05:12:44\n",
      "$ job_id             <i64> 3740116807, 3281711737, 3705985001, 3744782669, 3705877501, 3737987670, 3728215830, 3730247083, 3735059598, 3741786261\n",
      "$ position           <str> None, None, None, None, None, None, None, None, None, None\n",
      "$ location           <str> '&location=European Economic Area', '&location=European Economic Area', '&location=European Economic Area', '&location=European Economic Area', '&location=European Economic Area', '&location=European Economic Area', '&location=European Economic Area', '&location=European Economic Area', '&location=European Economic Area', '&location=European Economic Area'\n",
      "$ job                <str> 'Data Analyst', 'Data Consultant (Azure, Power Bi)', 'Data Analyst', 'Senior Data Analyst', 'Data Analyst', 'Data Analyst', 'Consultor/a BI', 'Data Expert Analyst', 'Data Analyst (Power BI) & AWS Services (S3/Redshift)', 'Analytics Consultant'\n",
      "$ company            <str> 'Encoretech', 'Kabel', 'Universal Diagnostics S.A.', 'Luxoft', 'RealNaut', 'Remote', 'NTT DATA', 'Keepler Data Tech', 'Bluetab, an IBM Company', 'P3'\n",
      "$ attempted         <bool> True, True, True, True, True, True, True, True, True, True\n",
      "$ result            <bool> True, True, True, True, True, True, True, True, True, True\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.glimpse()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr > th,\n",
       ".dataframe > tbody > tr > td {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>location</th><th>result</th><th>count</th></tr><tr><td>str</td><td>bool</td><td>u32</td></tr></thead><tbody><tr><td>&quot;&amp;location=Turk…</td><td>true</td><td>541</td></tr><tr><td>&quot;&amp;location=Denm…</td><td>true</td><td>76</td></tr><tr><td>&quot;&amp;location=Germ…</td><td>true</td><td>300</td></tr><tr><td>&quot;&amp;location=Neth…</td><td>true</td><td>765</td></tr><tr><td>&quot;&amp;location=Euro…</td><td>true</td><td>732</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 3)\n",
       "┌──────────────────────────────────┬────────┬───────┐\n",
       "│ location                         ┆ result ┆ count │\n",
       "│ ---                              ┆ ---    ┆ ---   │\n",
       "│ str                              ┆ bool   ┆ u32   │\n",
       "╞══════════════════════════════════╪════════╪═══════╡\n",
       "│ &location=Turkey                 ┆ true   ┆ 541   │\n",
       "│ &location=Denmark                ┆ true   ┆ 76    │\n",
       "│ &location=Germany                ┆ true   ┆ 300   │\n",
       "│ &location=Netherlands            ┆ true   ┆ 765   │\n",
       "│ &location=European Economic Area ┆ true   ┆ 732   │\n",
       "└──────────────────────────────────┴────────┴───────┘"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.group_by([\"location\",\"result\"]).agg([pl.count()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_successfull.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path = './volumes/output2.csv'\n",
    "df.to_csv(file_path,index=False,header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby(['location','result']).size().reset_index(name='counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby(['position','result']).size().reset_index(name='counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dev",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}