pushing notebooks for plotting

plots/for_paper/for_paper.ipynb

@@ -0,0 +1,203 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import wandb\n",
+    "\n",
+    "import seaborn as sns\n",
+    "import numpy as np\n",
+    "\n",
+    "import matplotlib.pyplot as plt\n",
+    "sns.set_style(\"white\", {\"grid.color\": \".6\", \"grid.linestyle\": \":\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_runs_df_stable_sig(project, entity=\"jurujin\", runtime_limit=6*3600):\n",
+    "    '''\n",
+    "    Returns df with data from wandb project for stable-sig\n",
+    "    '''\n",
+    "    api = wandb.Api()\n",
+    "    runs = api.runs(entity + \"/\" + project)\n",
+    "\n",
+    "    summary_list, config_list, name_list = [], [], []\n",
+    "    for run in runs:\n",
+    "        summary_list.append(run.summary._json_dict)\n",
+    "        config_list.append({k: v for k, v in run.config.items() if not k.startswith(\"_\")})\n",
+    "        name_list.append(run.name)\n",
+    "\n",
+    "\n",
+    "    summary_df = pd.DataFrame(summary_list)\n",
+    "    config_df = pd.DataFrame(config_list)\n",
+    "\n",
+    "    df = pd.concat([summary_df.reset_index(drop=True), config_df.reset_index(drop=True)], axis=1)\n",
+    "    df[\"name\"] = name_list\n",
+    "    df = df[df[\"_runtime\"] > runtime_limit]\n",
+    "\n",
+    "    return df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Чтобы выбрать долгие FID запуски надо: df[\"_runtime\"] > 7200\n",
+    "\n",
+    "def get_runs_df(project, entity=\"jurujin\", runtime_limit=6*3600, resolution=False):\n",
+    "    '''\n",
+    "    Returns df with data from wandb project\n",
+    "    '''\n",
+    "    df = get_runs_df_stable_sig(project, entity, runtime_limit)\n",
+    "\n",
+    "    if resolution:\n",
+    "        df[\"det_resol\"] = df[\"no_w_det_dist_mean\"] - df[\"w_det_dist_mean\"]\n",
+    "\n",
+    "    return df.sort_values(by=[\"w_radius\", \"msg_scaler\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "detection_projects = [\n",
+    "    \"detect_msg_all_att_vae\",\n",
+    "    \"detect_msg_all_att_no_vae\", \n",
+    "\n",
+    "    # \"clip_different_msg\"    # Testing CLIP quality for different message\n",
+    "]\n",
+    "\n",
+    "stable_signature_detection_projects = [\n",
+    "    \"eval_stable_tree_all_attacks\"\n",
+    "]\n",
+    "\n",
+    "fid_projects = [\n",
+    "    # \"fid_gt_msg_all_att_vae\",\n",
+    "    # \"fid_gt_msg_all_att_no_vae\",\n",
+    "\n",
+    "    # \"fid_gen_msg_all_att_vae\",\n",
+    "    # \"fid_gen_msg_all_att_no_vae\",\n",
+    "\n",
+    "    \"fid_gen_message_dependency\",\n",
+    "    \"fid_gt_message_dependency\",\n",
+    "    \n",
+    "]\n",
+    "\n",
+    "detection_cols = [\n",
+    "    \"name\",\n",
+    "    \"TPR@1%FPR\", \"auc\", \"acc\",\n",
+    "    \"Bit_acc\", \"Word_acc\",\n",
+    "    \"det_resol\",\n",
+    "\n",
+    "    \"w_clip_score_mean\",\n",
+    "\n",
+    "    \"w_det_dist_mean\",\n",
+    "    \"no_w_det_dist_mean\",\n",
+    "\n",
+    "    \"w_det_dist_std\",\n",
+    "    \"no_w_det_dist_std\",\n",
+    "    \n",
+    "    \"msg\", \"w_radius\", \"msg_scaler\",\n",
+    "\n",
+    "    \"jpeg_ratio\", \"crop_scale\", \"crop_ratio\", \"gaussian_blur_r\", \"gaussian_std\", \"brightness_factor\", \"r_degree\"\n",
+    "]\n",
+    "\n",
+    "stable_signature_detection_cols = [\n",
+    "    \"name\", \"Bit_acc\", \"Word_acc\"\n",
+    "]\n",
+    "\n",
+    "fid_cols = [\n",
+    "    \"name\",\n",
+    "    \"psnr_w\", \"ssim_w\",\n",
+    "    \"psnr_no_w\", \"ssim_no_w\",\n",
+    "    \"fid_w\", \"fid_no_w\",\n",
+    "    \"msg\", \"w_radius\", \"msg_scaler\",\n",
+    "\n",
+    "]\n",
+    "\n",
+    "fid_att_cols = [\n",
+    "    \"name\",\n",
+    "    \"psnr_w\", \"ssim_w\",\n",
+    "    \"psnr_no_w\", \"ssim_no_w\",\n",
+    "    \"fid_w\", \"fid_no_w\",\n",
+    "    \"msg\", \"w_radius\", \"msg_scaler\",\n",
+    "\n",
+    "    \"jpeg_ratio\", \"crop_scale\", \"crop_ratio\", \"gaussian_blur_r\", \"gaussian_std\", \"brightness_factor\", \"r_degree\"\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for project in detection_projects:\n",
+    "    get_runs_df(project, resolution=True, runtime_limit=4 * 3600).to_csv(f\"./detection/{project}.csv\", index=False, columns=detection_cols)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for project in fid_projects:\n",
+    "    get_runs_df(project).sort_values(by=\"name\", ascending=False).to_csv(f\"./fid/{project}.csv\", index=False, columns=fid_cols)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for project in stable_signature_detection_projects:\n",
+    "    get_runs_df_stable_sig(project, runtime_limit=0).to_csv(f\"./detection/{project}.csv\", index=False, columns=stable_signature_detection_cols)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "clip_different_message = \"clip_different_msg\"\n",
+    "get_runs_df(clip_different_message, runtime_limit=0, resolution=True).sort_values(by=\"name\", ascending=False).to_csv(f\"./detection/{clip_different_message}.csv\", index=False, columns=detection_cols)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "plot",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}