Jali_4p0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4599d9e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from util.SongDataStructure import *\n",
    "from util.pitch_interval_estimation import *\n",
    "import numpy as np\n",
    "import json\n",
    "from matplotlib import pyplot as plt\n",
    "import os\n",
    "from util.ioUtil import get_audio_from_video\n",
    "from scipy.interpolate import interp1d, splrep, splev\n",
    "from models.vowel_modification_detector import vowel_mod_detector\n",
    "vowel_mod = vowel_mod_detector()\n",
    "import copy\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94e562b8",
   "metadata": {},
   "source": [
    "# Pre-process data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5fb9da5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# dir = \"E:/MASC/Sig_videos/song2face4/\"\n",
    "# file_name_template = \"song2face_4\"\n",
    "# get_audio_from_video(file_name_template+\".mp4\", dir)\n",
    "# os.system('ffmpeg -i ' + os.path.join(dir, file_name_template+\"/audio.mp3 \") + os.path.join(dir, file_name_template+\"/audio.wav\"))\n",
    "# lyric = Minimal_song_data_structure(os.path.join(dir, file_name_template+\"/audio.wav\"), os.path.join(dir, file_name_template+\".txt\"))\n",
    "# lyric.compute_self_phoneme_alignment()\n",
    "# lyric.write_textgrid(dir, \"MVP\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "111bb266",
   "metadata": {},
   "source": [
    "## Input Block"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75609c1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "dir = \"E:/MASC/Jali_sing/Sig_videos/cry_me_a_river_ella_fitzgerald\"\n",
    "file_name_template = \"audio_1\"\n",
    "script_path = os.path.join(dir, \"audio_1_full.TextGrid\")\n",
    "output_template = \"jali_MVP\"\n",
    "spike_width = 0.5\n",
    "speech_vowel_threshold = 0.3\n",
    "vowel_blend_distance = 0.3\n",
    "vowel_blend_magnitude = 0.95"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5204667a",
   "metadata": {},
   "outputs": [],
   "source": [
    "dir = \"E:/MASC/lmbmm_vocal_sep_data/NUS/test/\"\n",
    "file_name_template = \"0\"\n",
    "script_path = os.path.join(dir, \"0.TextGrid\")\n",
    "output_template = \"\"\n",
    "spike_width = 0.5\n",
    "speech_vowel_threshold = 0.3\n",
    "vowel_blend_distance = 0.3\n",
    "vowel_blend_magnitude = 0.95"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1106bf10",
   "metadata": {},
   "outputs": [],
   "source": [
    "VOICED = set(['AA', 'AE', 'AH', 'AO', 'AW', 'AY', 'EH', 'EY', 'IH', 'IY', 'OW', 'OY', 'UH', 'UW', \"ER\"])\n",
    "CMU2VISEME = {\"AA\":\"Ah\", \"AO\":\"Ah\", \"AY\":\"Ah\", \"AW\":\"Ah\",\"AE\":\"Aa\",\n",
    "              \"EY\":\"Aa\",\"UH\":\"Uh\", \"UW\":\"U\",\"IH\": \"Ih\",\"IY\": \"Ih\",\"EH\": \"Eh\",\"HH\": \"Eh\",\"UH\": \"Eh\",\"AH\": \"Eh\",\n",
    "              \"ER\": \"Eh\",\"OW\":\"Oo\",\"OY\":\"Oh\",\"R\":\"R\",\"D\":\"LNTD\",\"T\": \"LNTD\",\"L\":\"LNTD\",\"N\":\"LNTD\",\"NG\":\"LNTD\",\n",
    "              \"F\":\"FV\",\"V\":\"FV\",\"B\":\"BP\",\"M\":\"M\",\"P\":\"BP\",\"CH\":\"ShChZh\",\"SH\":\"ShChZh\",\"ZH\":\"ShChZh\",\n",
    "              \"S\": \"SZ\", \"Z\": \"SZ\",\"DH\":\"Th\", \"TH\":\"Th\",\"G\":\"GK\", \"K\":\"GK\",\"Y\":\"Y\",\"JH\":\"J\",\"W\":\"W\",}\n",
    "VOWELS_SLIDERS_JALI = set(['Ih_pointer', 'Ee_pointer', 'Eh_pointer', 'Aa_pointer', 'U_pointer', 'Uh_pointer'\n",
    "                           , 'Oo_pointer', 'Oh_pointer', 'Schwa_pointer', 'Eu_pointer', \"Ah_pointer\"])\n",
    "CONSONANTS_SLIDERS_JALI = set([\"M_pointer\", \"BP_pointer\", \"JY_pointer\", \"Th_pointer\", \"ShChZh_pointer\", \"SZ_pointer\", \"GK_pointer\", \"LNTD_pointer\", \"R_pointer\", \"W_pointer\", \"FV_pointer\"])\n",
    "CONSONANTS_SLIDERS_NOJAW_JALI = set([\"Ya_pointer\", \"Ja_pointer\", \"Ra_pointer\", \"FVa_pointer\", \"LNTDa_pointer\", \"Ma_pointer\", \"BPa_pointer\", \"Wa_pointer\", \"Tha_pointer\", \"GKa_pointer\"])\n",
    "JALI_SLIDERS_SET = set.union(VOWELS_SLIDERS_JALI, CONSONANTS_SLIDERS_JALI, CONSONANTS_SLIDERS_NOJAW_JALI)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c42ceb41",
   "metadata": {},
   "outputs": [],
   "source": [
    "CMU2VISEME = {\"AA\":\"Ah\", \"AO\":\"Ah\", \"AY\":\"Ah\", \"AW\":\"Ah\",\"AE\":\"Eh\",\n",
    "              \"EY\":\"Eh\",\"UH\":\"Ah\", \"UW\":\"U\",\"IH\": \"Ee\",\"IY\": \"Ee\",\"EH\": \"Eh\",\"HH\": \"Eh\",\"UH\": \"U\",\"AH\": \"Eh\",\n",
    "              \"ER\": \"Eh\",\"OW\":\"Oo\",\"OY\":\"Oo\",\"R\":\"R\",\"D\":\"LNTD\",\"T\": \"LNTD\",\"L\":\"LNTD\",\"N\":\"LNTD\",\"NG\":\"LNTD\",\n",
    "              \"F\":\"FV\",\"V\":\"FV\",\"B\":\"BP\",\"M\":\"M\",\"P\":\"BP\",\"CH\":\"ShChZh\",\"SH\":\"ShChZh\",\"ZH\":\"ShChZh\",\n",
    "              \"S\": \"SZ\", \"Z\": \"SZ\",\"DH\":\"Th\", \"TH\":\"Th\",\"G\":\"GK\", \"K\":\"GK\",\"Y\":\"Y\",\"JH\":\"J\",\"W\":\"W\",}\n",
    "\n",
    "dip_percentage = {\"U_pointer\":1.05, \"Oo_pointer\":0.95, \"Ah_pointer\":0.9, \"Eh_pointer\":0.9, \n",
    "                  \"Ee_pointer\":1.1, \"LNTD_pointer\":0.9, \"M_pointer\":0.9}\n",
    "\n",
    "\n",
    "cmu_sets = CMU_phonemes_dicts()\n",
    "jali_sets = JALI_visemes_dicts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6be63245",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_basic_viseme_curve(start, end, value, sustain=1, decay = 0.75, onset=0.1, offset=0):\n",
    "    if end - start < 0.1:\n",
    "        end = start + 0.1\n",
    "    interval = []\n",
    "    interval.append([start-onset, 0, \"bound\"])\n",
    "    # second point is when the belting starts \n",
    "    interval.append([start, 1 * value, \"internal\"])\n",
    "    # third point emphasizes decay, it happens 75% down the interval\n",
    "    if sustain < 1:\n",
    "        interval.append([start + sustain * (end - start), decay * value, \"internal\"])\n",
    "        # last point is where the furrowing ends\n",
    "        interval.append([end+offset, 0, \"bound\"])\n",
    "    elif sustain == 1:\n",
    "        interval.append([end, value, \"internal\"])\n",
    "        # last point is where the furrowing ends\n",
    "        interval.append([end+offset, 0, \"bound\"])\n",
    "    return interval\n",
    "def get_kth_neighbour(input_list, i, k):\n",
    "    if i+k < 0 or i+k >= len(input_list):\n",
    "        return None\n",
    "    return input_list[i+k]\n",
    "def difference_of_ctrl_pts(ctrl_pts1, ctrl_pts2):\n",
    "    if len(ctrl_pts1) != len(ctrl_pts2):\n",
    "        print(False)\n",
    "        return False\n",
    "    for i in range(0, len(ctrl_pts1)):\n",
    "        if len(ctrl_pts1[i]) != len(ctrl_pts2[i]):\n",
    "            print(False)\n",
    "            return False\n",
    "    for i in range(0, len(ctrl_pts1)):\n",
    "        for j in range(0, len(ctrl_pts1[i])):\n",
    "            if ctrl_pts1[i][j][0] != ctrl_pts2[i][j][0] or ctrl_pts1[i][j][1] != ctrl_pts2[i][j][1]:\n",
    "                print(False)\n",
    "                return False\n",
    "    return True\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ff00098b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_viseme_curve(viseme_curve:Viseme_curve, viseme_slider_name:str):\n",
    "    v_list = viseme_curve.viseme_list[-1]\n",
    "    ctrl_pts = viseme_curve.viseme_ctrl_pts[-1]\n",
    "    \n",
    "    x = []\n",
    "    y = []\n",
    "    for i in range(0, len(v_list)):\n",
    "        if v_list[i] == viseme_slider_name:\n",
    "            for k in range(0, len(ctrl_pts[i])):\n",
    "                x.append(ctrl_pts[i][k][0])\n",
    "                y.append(ctrl_pts[i][k][1])\n",
    "#     tck = splrep(x, y, s=0)\n",
    "    A = [x, y]\n",
    "    A = np.array(A)\n",
    "    A = A[::, A[0,].argsort()[::-1]]\n",
    "    x = A[0]\n",
    "    y = A[1]\n",
    "    \n",
    "#     x_range = np.arange(x.min(), x.max(), 0.01)\n",
    "#     y_interp = splev(x_range, tck, der=0)\n",
    "#     plt.plot(x_range, y_interp)\n",
    "    plt.plot(x, y)\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2238832f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1eccd2c7250>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAjVklEQVR4nO3deXyV5Z3+8c+XPQkhgSRsYQk7KMpi2GrVKmLdim3HtmjViru1drEzo05b22k7/bXVmWrtQmmtaAX3jdpWrStqZd93QiAkLFnIAmRfvr8/kunEGM1JOMlzcnK9Xy9e5Dzn5pzrJnDxcJ9nMXdHREQ6v25BBxARkfBQoYuIRAkVuohIlFChi4hECRW6iEiU6BHUGycnJ3taWlpQby8i0imtW7euwN1TmnsusEJPS0tj7dq1Qb29iEinZGZZH/WcllxERKKECl1EJEqo0EVEooQKXUQkSqjQRUSihApdRCRKqNBFRKJEYMehi4h0BWVVNRwuqeBISUXDz+VMGZ7IWeOaPTfopKjQRUTaqLbOyTtewcGicg4Wl3OouIJDxeUcKq5/fLikgpLy6g/9uls/NUaFLiLSkdyd/BOVZBeWkV1YTnZhGTlF5eQU1/98qLic6toP3iQoMbYnQxJiSE2MYUbaAIYk9mFIQh+GJMQwJKEPg/r1oU/P7u2SV4UuIl1adW0dB4vK2X+0lKyjZWQdLeNAYRkHCkvJLiynvLr2A+NT4nuTmhjD6cMSuWjyEFL7xzCsfwzDEmMYmhhDXO/galWFLiJRr67OOXysgn35pewrOEFmQSn7CkrZX1BKdlE5tXX/t5cd07M7IwbEMjIpjrPGpTBiQCwjBsQyfEAMw/rHttvedTio0EUkalRU15KZX8re/BNk5J0gI/8EmQ0lXlFd989xsb26k5YUx6lDE7jk9CGMTIojLSmOtKRYUuJ7Y2YBzqLtVOgi0ulUVNeSkXeCPXnH2Z17gj25x8nIO8GBwjL+d2fbDIb3j2V0ShyfGJPEmJS+jEqOY3RKHAM7cWl/HBW6iESsujonu6iMHYePs/PIMXYdOc6uI8fZf7T0n8Xdo5sxKrl+b/uyqamMHdiXsQPryzuSl0fagwpdRCJCRXUtO48cZ9uhErYfOsaOw/UFXlpV/6GkGYwcEMvEwf24dMpQxg/qy/hB8aQlxdGrh86RBBW6iASgtLKGbYeOsfVgCVsPlbDt4DEy8k/888PJ+N49mDSkH5efMYyJQ/oxaUg/xg/qS2wvVdbH0e+OiLSryppath86xuacEjblFLM5p4S9+SfwhiWTgfG9mZyawAWnDuLUof04dWgCw/rHROUad3tToYtI2Lg7WUfLWH+giI3ZxWzKLmb74WP/PPkmuW9vTh+WwKWnD+H0YQlMHprAwH59Ak4dPVToItJmFdW1bMouZt2BItZnFbH+QDGFpVVA/aGBpw9L4LpPjmLqsESmDE9kSEIf7Xm3oxYL3cwmAE822jQauMfd7280JgF4DBjR8Jr3ufvD4Y0qIkErKq1izf5C1mYVsWZ/IVsPlvxz73t0ShznTRzI9BH9mTYikfGD4uneTeXdkVosdHffBUwFMLPuwEHg+SbDbgO2u/tnzCwF2GVmS929Ksx5RaQD5R+vZNW+o6zMPMrqfYXszj0BQK/u3ZgyPIEbzhrNGSP6M31kfwbE9Qo4rbR2yWUusNfds5psdyDe6v8v1RcoBGrCkE9EOlBRaRUrM4/yj71HeT/zKBl59QUe16s7Z6QN4LKpqcwcNYDTUhO63DHenUFrC30B8Hgz238FLAcOAfHAl9y9rplxIhJByqtqWbO/kPcyCng3o4Dth4/hXr/+PSNtAJefMYzZo5OYPLQfPbrrWO9IF3Khm1kvYD5wdzNPfxrYCJwHjAH+bmbvuPuxJq9xE3ATwIgRI9oYWUTayt3ZeeQ4K3bn886eAlbvL6Sqpo6e3Y3pI/rzrfPHc+bYJE4flkhPFXin05o99IuA9e6e28xzC4GfursDGWa2D5gIrG48yN0XA4sB0tPT/UOvIiJhV1Jezbt7CnhrVx5v784n73glABMGxXPN7JF8clwyM0cN0Ek7UaA138EraH65BeAA9evr75jZIGACkHmS2USkDdydvfkneG1HHm/szGNdVhG1dU5CTE8+OS6Zc8ancPa4FAYn6PjvaBNSoZtZHDAPuLnRtlsA3H0R8CNgiZltAQy4090Lwh9XRJpTU1vH6v2F/H17Lq/vyONAYRkAk4b045ZzRnPuhIFMHZ6odfAoF1Khu3spkNRk26JGXx8CLghvNBH5OGVVNazYXcCr247w+s48Ssqr6dWjG2eOSeKms0dz3sSBDE2MCTqmdCAtmol0IscqqnljRx5/23qYt3fnU1FdR2JsT+ZOGsgFpwzirHEpgd4CTYKl77xIhDteUc1rO3L5y+bDrNhdQFVtHYP69eaL6cO58NTBzBw1QEspAqjQRSJSWVUNr+/IY/mmQ7y9O5+qmjqGJvTh6jkjufi0wUwb3p9uOq1emlChi0SI6to63tmTzwsbDvHajlzKqmoZ1K83V80ayaVThjB1WKJKXD6WCl0kQO7OppwSnlufw0ubD1NYWkVibE8+Oy2V+VOGMiNtgC5wJSFToYsE4EhJBc9tyOHZdTnszS+lV49uzJs0iM9OS+Wc8Sm6pZq0iQpdpINU1tTy6rZcnl6Xw7t78qlzmJk2gBvPGs1Fpw0hIaZn0BGlk1Ohi7Sz3bnHeWJ1Ns9vyKGorJqhCX247dyxXH7GMEYmxQUdT6KICl2kHVRU1/KXzYdZtvoA67KK6NnduOCUwXxxxnA+OTZZ6+LSLlToImG0r6CUpSuzeHpdDiXl1YxOjuM7F0/i89NTSerbO+h4EuVU6CInqbbOeWNnHo++v5939hTQo5vx6cmD+fKsEcwZnaR7aEqHUaGLtFFJeTVPrcnm0ZX7yS4sZ3C/Pnx73ni+NHM4A+N1JUPpeCp0kVbaV1DKkvf28fS6HMqqapmZNoC7LpzEBacO0k0hJFAqdJEQuDtrs4pYvCKT13bk0qObMX9KKgvPTGNyakLQ8UQAFbrIx6qtc17ddoTfrchkY3YxibE9+dq5Y7l6zkgtq0jEUaGLNKOyppbn1x/kdysy2VdQysikWH702clcPn0YMb10t3uJTCp0kUbKqmpYtuoAv38nk9xjlZyWmsCvr5zOhZMH69hxiXgtFrqZTQCebLRpNHCPu9/fZNyngPuBnkCBu58TrpAi7e14RTWPvp/FQ+/uo7C0ijmjk/jvL0zlzLE67FA6jxYL3d13AVMBzKw7cBB4vvEYM0sEfgNc6O4HzGxg2JOKtINjFdUseW8/D727j5Lyas6dkMLXzhvHGSP7Bx1NpNVau+QyF9jr7llNtl8JPOfuBwDcPS8c4UTay4nKGpa8t4/FKzI5VlHD+ZMG8vW54zh9WGLQ0UTarLWFvgB4vJnt44GeZvYWEA884O6PnmQ2kbArr6rl0ff3s+jtvRSVVXP+pIF88/zxOvRQokLIhW5mvYD5wN0f8TpnUL8HHwO8b2Yr3X13k9e4CbgJYMSIEW3NLNJqVTV1PLk2mwdf30Pe8UrOHp/Ct+eNZ8rwxKCjiYRNa/bQLwLWu3tuM8/lAEfdvRQoNbMVwBTgA4Xu7ouBxQDp6enetsgioaurc17acpj7XtnFgcIyZqT151dXTmfmqAFBRxMJu9YU+hU0v9wC8CLwKzPrAfQCZgG/OMlsIifl3T0F/PTlHWw9eIyJg+N5+NoZfGpCio5akagVUqGbWRwwD7i50bZbANx9kbvvMLOXgc1AHfAHd9/aDnlFWrQ79zg/+esO3tqVT2piDL/40hQum5KqGyxL1Aup0BuWUpKabFvU5PG9wL3hiybSOvnHK/mfv+/myTUHiOvdg/+4eCLXzEmjT0+d2Sldg84UlU6vsqaWJe/t58E3MqioruWaOWl8Y+44+sf1CjqaSIdSoUun9vqOXH740nayjpZx3sSBfOeSSYxJ6Rt0LJFAqNClU9pXUMoP/7yNN3flMyYljkeum8k541OCjiUSKBW6dCrlVbX8+s0MFq/IpFePbnz3kkl85RNpurGECCp06URe257LD/68jZyicj4/LZW7Lp6oa5KLNKJCl4h3uKSc77+4jVe35zJuYF+euGk2s0cntfwLRboYFbpErNo659H393PfK7uodefOCydyw1mjtLwi8hFU6BKRdh45xp3PbmFTdjHnjE/hx5+dzPABsUHHEoloKnSJKJU1tfz6jQx+89Ze+sX05IEFU5k/ZahO1xcJgQpdIsbG7GL+7elN7Mk7weenpfLdS09hgE4OEgmZCl0CV1Fdy/2v7WHxir0M6teHhxfO4NwJuumVSGup0CVQm3OKueOpTWTknWDBjOH8xyWT6NenZ9CxRDolFboEorq2jgffyODXb2aQ0re3zvQUCQMVunS4vfkn+NaTG9mcU8LnpqXyg8+cSkKs9spFTpYKXTqMu/PYqgP811+206dnd3775elcdNqQoGOJRA0VunSIoycq+fdnNvP6zjzOGpfMfV+YwqB+Om1fJJxU6NLu3tmTzx1PbaKkvJrvf+YUvjInTXcPEmkHLZ5DbWYTzGxjox/HzOybHzF2hpnVmNnlYU8qnU51bR0//dtOrn5oNQkxPXnxtjNZeOYolblIO2lxD93ddwFTAcysO3AQeL7puIbnfga8Gt6I0hnlFJXx9cc3sP5AMVfMHME9l55CTC/dCk6kPbV2yWUusNfds5p57nbgWWDGSaeSTu3v23P516c3UVvnPHjFND4zZWjQkUS6hNYW+gLg8aYbzSwV+BxwLir0Lqumto57X9nF71ZkMjm1H7++cjojk+KCjiXSZYRc6GbWC5gP3N3M0/cDd7p73cddRMnMbgJuAhgxYkSrgkpkyz1Wwe3LNrB6fyFXzx7Jdy+dRO8eWmIR6Uit2UO/CFjv7rnNPJcOPNFQ5snAxWZW4+4vNB7k7ouBxQDp6enepsQScVZlHuW2ZRsorazhgQVTuWxqatCRRLqk1hT6FTSz3ALg7qP+92szWwK81LTMJfq4O398bz8/+esORg6I5fEbZzFuUHzQsUS6rJAK3czigHnAzY223QLg7ovaJ5pEsvKqWu56bjMvbjzEBacM4r+/OIV4XVRLJFAhFbq7lwJJTbY1W+Tufu3Jx5JIllNUxs1/Wsf2w8f4t09P4NZzxujYcpEIoDNFpVVWZh7lq0vXU11bxx+/MoNzJ+q65SKRQoUuIVu26gD3vLiVEUmx/P6adMak9A06kog0okKXFtXU1vFff93Bw+/t55zxKTx45TTdhEIkAqnQ5WMdr6jma8s28PbufK47cxT/cfFEenRv8RJAIhIAFbp8pJyiMq5fspa9+Sf4f58/jStm6mQwkUimQpdmbcou5vpH1lJZU8sj183kzLHJQUcSkRao0OVD/r49l9sfX09y3946WUikE1Ghywf8aWUW339xK6elJvDQtTNI7ts76EgiEiIVugD1p/H//JVd/PatvcydOJAHr5xGbC/98RDpTPQ3VqiurePOZzfz3PqDXDlrBD+cf6qOZBHphFToXVxZVQ23LV3Pm7vyuWPeeG4/bywfdwlkEYlcKvQurKi0iuseWcOm7GIdligSBVToXVTusQqufmgV+4+W8Zsvn8GFkwcHHUlETpIKvQvKOlrKVQ+tovBEFUsWzuATY3SMuUg0UKF3MbuOHOeqh1ZRXVvHshtnM2V4YtCRRCRMVOhdyNaDJVz90Cp6du/GUzfPYbxOGBKJKir0LmJdVhHXPryafn16suzGWYxMigs6koiEmQq9C1iZeZTrlqxhYHxvlt44m9TEmKAjiUg7aPHsETObYGYbG/04ZmbfbDLmy2a22cy2mNk/zGxKuyWWVvlHRgHXPrya1MQYnrp5jspcJIq1uIfu7ruAqQBm1h04CDzfZNg+4Bx3LzKzi4DFwKzwRpXWemdPPjc8spa0pDiW3jhL12URiXKtXXKZC+x196zGG939H40ergSGnWwwOTkrdudzw6NrGZ0cx9IbZpGkMheJeq29YMcC4PEWxlwP/K25J8zsJjNba2Zr8/PzW/nWEqp39xRw46NrGZPSl2U3zlaZi3QRIRe6mfUC5gNPf8yYc6kv9Dube97dF7t7urunp6SktDarhOAfGQVc/8gaRjXsmQ+I6xV0JBHpIK1ZcrkIWO/uuc09aWanA38ALnL3o+EIJ62zKvMo1z+ylpFJsSpzkS6oNUsuV/ARyy1mNgJ4Drja3XeHI5i0zvoDRVy3ZA1DE/uw9AYts4h0RSHtoZtZHDAPuLnRtlsA3H0RcA+QBPym4dKrNe6eHva00qytB0v4yh9Xkxzfm2U3ziYlXmUu0hWFVOjuXkp9YTfetqjR1zcAN4Q3moRiT+5xrvnjauJ792DpDbMY1K9P0JFEJCC6LU0nduBoGV/+wyq6dzOW3TibYf1jg44kIgFSoXdSeccquOqhVVTV1rH0hlmkJevaLCJdnQq9Eyouq+Lqh1ZTcKKSh6+doasmigigQu90yqtquW7JGvYVlPL7a9KZNqJ/0JFEJEKo0DuR6to6blu2no3ZxfzyimmcOVZ3GhKR/6PL53YS7s5dz27hjZ15/ORzp+keoCLyIdpD7yR++vJOnl2fwx3zxnPlrBFBxxGRCKRC7wQe+cd+fvd2JlfNHsHt540NOo6IRCgVeoR7eesRfvDnbcw7ZRD/OX8yDWfiioh8iAo9gq3LKuIbT2xgyrBEfrlgGt27qcxF5KOp0CPU/oJSbnhkDUMS+vDQV9KJ6dU96EgiEuFU6BGoqLSKhUvWALBk4UxdOVFEQqLDFiNMZU0tNz+2joNF5Sy9Uaf0i0joVOgRxN25+9ktrN5XyAMLpjIjbUDQkUSkE9GSSwT5zVt7eW7DQe6YN57LpqYGHUdEOhkVeoR4eeth7n1lF5dNHapjzUWkTVToEWDrwRK+9eQmpg5P5Gf/crqONReRNmmx0M1sgpltbPTjmJl9s8kYM7NfmlmGmW02s+ntljjK5B+v5MZH19I/tieLrzmDPj11eKKItE2LH4q6+y5gKoCZdQcOAs83GXYRMK7hxyzgtw0/y8eorKnllsfWUVxWzTO3zmFgvG4fJyJt19oll7nAXnfParL9MuBRr7cSSDSzIWFJGKXcne+/uI11WUXc94UpnDo0IehIItLJtbbQFwCPN7M9Fchu9DinYdsHmNlNZrbWzNbm5+e38q2jy59WZvHEmmy+du5YLjld//aJyMkLudDNrBcwH3i6rW/m7ovdPd3d01NSUtr6Mp3eqsyj/Oeft3P+pIHcMW980HFEJEq0Zg/9ImC9u+c289xBYHijx8MatkkTh0vKuW3ZekYmxfKLL02lmy64JSJh0ppCv4Lml1sAlgPXNBztMhsocffDJ50uylTW1HLrY+spr6pl8dVnEN+nZ9CRRCSKhHTqv5nFAfOAmxttuwXA3RcBfwUuBjKAMmBh2JNGgR8s38bG7GIWXXUGYwfGBx1HRKJMSIXu7qVAUpNtixp97cBt4Y0WXZ5ak83jq7P56qfG6H6gItIudKZoB9h2qITvvbiVM8cm8e0LJgQdR0SilAq9nZWUV3PrY+vpH9tLdx0SkXaly+e2I3fnX5/exKHicp68eY5uVCEi7Up76O3o9+9k8vftuXznkkmcMbJ/0HFEJMqp0NvJuqxCfvbyLi4+bTDXfiIt6Dgi0gWo0NtBUWkVty/bQGpiDD/V5XBFpINoDT3M6uqcbz+9iYITVTx76yfop5OHRKSDaA89zB56dx9v7Mzju5dO4rRhuoKiiHQcFXoYbc4p5uev7OTTpw7i6tkjg44jIl2MCj1MjldUc/vjG0jp21u3kRORQGgNPQzcne+9sJXswjKevHkOibG9go4kIl2Q9tDD4Ln1B3lh4yG+ef54ZqQNCDqOiHRRKvSTlHW0lHte3MrMUQO47dyxQccRkS5MhX4Sqmvr+OaTG+nWzfjFl6bqOi0iEiitoZ+EB9/IYMOBYh68YhqpiTFBxxGRLk576G20dn8hv3pjD5+fnspnpgwNOo6IiAq9LU5U1vCtpzYyNDGG/5x/atBxRESAEAvdzBLN7Bkz22lmO8xsTpPnE8zsz2a2ycy2mVlU34Luxy9tJ6eonP/54lTdF1REIkaoa+gPAC+7++Vm1guIbfL8bcB2d/+MmaUAu8xsqbtXhTNsJHhtey5PrMnmlnPGMHOUDlEUkcjRYqGbWQJwNnAtQENJNy1qB+Kt/vTIvkAhUBPWpBHg6IlK7npuMxMHx/OteeOCjiMi8gGhLLmMAvKBh81sg5n9wczimoz5FTAJOARsAb7h7nVNX8jMbjKztWa2Nj8//2Szdyh35zvPb+VYeQ33L5hK7x7dg44kIvIBoRR6D2A68Ft3nwaUAnc1GfNpYCMwFJgK/MrM+jV9IXdf7O7p7p6ekpJyMrk73PJNh3h52xG+NW88Ewd/aGoiIoELpdBzgBx3X9Xw+BnqC76xhcBzXi8D2AdMDF/MYOUdq+CeF7cxbUQiN509Oug4IiLNarHQ3f0IkG1mExo2zQW2Nxl2oGE7ZjYImABkhjFnYNydu5/bQkV1Lfd9YYrOBhWRiBXqUS63A0sbjnDJBBaa2S0A7r4I+BGwxMy2AAbc6e4F7RG4oz27/iCv78zje5eewpiUvkHHERH5SCEVurtvBNKbbF7U6PlDwAXhixUZco9V8MM/b2Nm2gAW6kbPIhLhdKboR6g/qmULVbV1/Pzy0+mmpRYRiXAq9I+wfNMhXtuRx79eMIG05KZHaYqIRB4VejPyj1fy/eX1R7UsPHNU0HFEREKiQm/GD5Zvo6yylnsvP11HtYhIp6FCb+LVbUf4y5bDfH3uWMYOjA86johIyFTojRyrqOZ7L25l4uB4bj5nTNBxRERaRXcsauTnL+8k73glv7s6nZ7d9W+diHQuaq0Ga/YX8tjKAyz8xCimDk8MOo6ISKup0IHKmlrufm4LqYkxfPuC8UHHERFpEy25AIvfziQj7wQPXzuDuN76LRGRzqnL76HvKyjlwTczuOS0IZw7cWDQcURE2qxLF7q7870XttK7ezfu+cwpQccRETkpXbrQX9x4iHczCvj3CycwqF+foOOIiJyULlvoJWXV/Pgv25kyPJErZ40MOo6IyEnrsoV+36u7KCyt4iefm6zT+0UkKnTJQt+SU8Jjq7K4Zk4apw5NCDqOiEhYdLlCr61zvvvCFpL79uYOHXMuIlEkpEI3s0Qze8bMdprZDjOb08yYT5nZRjPbZmZvhz9qeDy++gCbckr47iWT6NenZ9BxRETCJtSzaB4AXnb3yxvuKxrb+EkzSwR+A1zo7gfMLCIP6D56opJ7X9nFnNFJzJ8yNOg4IiJh1WKhm1kCcDZwLYC7VwFVTYZdCTzn7gcaxuSFN2Z43PfqLkora/jhZadipg9CRSS6hLLkMgrIBx42sw1m9gcza3pPtvFAfzN7y8zWmdk1zb2Qmd1kZmvNbG1+fv5JRm+dTdnFPLEmm4VnpjFukK5zLiLRJ5RC7wFMB37r7tOAUuCuZsacAVwCfBr4npl96BNHd1/s7ununp6SknJyyVuhrs65Z/k2kvv25utzx3XY+4qIdKRQCj0HyHH3VQ2Pn6G+4JuOecXdS929AFgBTAlfzJPzzPocNmUXc/dFE4nXB6EiEqVaLHR3PwJkm9mEhk1zge1Nhr0IfNLMephZLDAL2BHWpG1UUl7Nz/62k/SR/fnctNSg44iItJtQj3K5HVjacIRLJrDQzG4BcPdF7r7DzF4GNgN1wB/cfWu7JG6lB1/fQ2FZFY/Mn6kPQkUkqoVU6O6+EUhvsnlRkzH3AveGJ1Z47M0/wZJ/7GfBjOFMTtUZoSIS3aL6TNEfv7SdmJ7d+fYFE1oeLCLSyUVtob+5M483d+XzjfPHkdy3d9BxRETaXVQWenVtHT/6y3ZGJ8dxzZy0oOOIiHSIqCz0P72fRWZ+Kd+9dBK9ekTlFEVEPiTq2q64rIoHXt/DWeOSOXdCRF5SRkSkXURdof/y9QyOV1TznUsm6TBFEelSoqrQM/NP8Oj7+/nSjOFMHNwv6DgiIh0qqgr9p3/bSe8e3fjWPN24QkS6nqgp9JWZR3l1ey5fPXcsA+P7BB1HRKTDRUWh19U5P/nrDoYk9OH6T44KOo6ISCCiotBf2nKYzTklfPuCCfTp2T3oOCIigej0hV5ZU8vPX97JpCH9dDVFEenSOn2h/+n9LHKKyrn7ool076bDFEWk6+rUhV5SVs2Db2Rw1rhkzh7fcXdAEhGJRJ260H/zdgbHKqq5+6JJQUcREQlcpy30wyXlLHlvP5+dmsopQ3USkYhIpy30B17bQ507d+gkIhERIMRCN7NEM3vGzHaa2Q4zm/MR42aYWY2ZXR7emB+UkXeCp9Zm8+VZIxk+ILY930pEpNMI9Z6iDwAvu/vlDfcV/VCLmll34GfAq2HM16x7X9lJTM/ufO28se39ViIinUaLe+hmlgCcDTwE4O5V7l7czNDbgWeBvHAGbGr9gSJe2ZbLjWeP1p2IREQaCWXJZRSQDzxsZhvM7A9mFtd4gJmlAp8DfvtxL2RmN5nZWjNbm5+f3+bQZ41L5oazRrf514uIRKNQCr0HMB34rbtPA0qBu5qMuR+4093rPu6F3H2xu6e7e3pKStuOG58+oj9/un4WfXuHulokItI1hNKKOUCOu69qePwMHy70dOCJhhtKJAMXm1mNu78QrqAiIvLxWix0dz9iZtlmNsHddwFzge1NxvzzEodmtgR4SWUuItKxQl23uB1Y2nCESyaw0MxuAXD3Re0VTkREQhdSobv7RuqXVRprtsjd/dqTiyQiIm3Rac8UFRGRD1Khi4hECRW6iEiUUKGLiEQJc/dg3tgsH8hq4y9PBgrCGKez6Irz7opzhq457644Z2j9vEe6e7NnZgZW6CfDzNa6e9OjbqJeV5x3V5wzdM15d8U5Q3jnrSUXEZEooUIXEYkSnbXQFwcdICBdcd5dcc7QNefdFecMYZx3p1xDFxGRD+use+giItKECl1EJEpEdKGb2YVmtsvMMsys6TXYMbPeZvZkw/OrzCwtgJhhF8K87zCz7Wa22cxeN7ORQeQMp5bm3Gjcv5iZm1lUHN4WyrzN7IsN3+9tZrasozOGWwh/vkeY2ZsNd0jbbGYXB5EznMzsj2aWZ2ZbP+J5M7NfNvyebDaz6W16I3ePyB9Ad2AvMBroBWwCTmky5qvAooavFwBPBp27g+Z9LhDb8PWtnX3eocy5YVw8sAJYCaQHnbuDvtfjgA1A/4bHA4PO3QFzXgzc2vD1KcD+oHOHYd5nU3/nt60f8fzFwN8AA2YDq9ryPpG8hz4TyHD3THevAp4ALmsy5jLgkYavnwHmWsNtkzqxFuft7m+6e1nDw5XAsA7OGG6hfK8BfgT8DKjoyHDtKJR53wj82t2LANy9XW/C3gFCmbMD/Rq+TgAOdWC+duHuK4DCjxlyGfCo11sJJJrZkNa+TyQXeiqQ3ehxTsO2Zse4ew1QAiR1SLr2E8q8G7ue+n/ZO7MW59zwX9Dh7v6XjgzWzkL5Xo8HxpvZe2a20swu7LB07SOUOf8AuMrMcoC/Un+DnWjX2r/3zdKdljsxM7uK+huPnBN0lvZkZt2A/wGuDThKEHpQv+zyKer/J7bCzE5z9+IgQ7WzK4Al7v7fZjYH+JOZTfYWbkIvkb2HfhAY3ujxsIZtzY4xsx7U//fsaIekaz+hzBszOx/4DjDf3Ss7KFt7aWnO8cBk4C0z20/9GuPyKPhgNJTvdQ6w3N2r3X0fsJv6gu+sQpnz9cBTAO7+PtCH+gtYRbOQ/t63JJILfQ0wzsxGNdzLdAGwvMmY5cBXGr6+HHjDGz5h6MRanLeZTQN+R32Zd/Y1VWhhzu5e4u7J7p7m7mnUf24w393XBhM3bEL5M/4C9XvnmFky9UswmR2YMdxCmfMB6m9Gj5lNor7Q8zs0ZcdbDlzTcLTLbKDE3Q+3+lWC/vS3hU+GL6Z+j2Qv8J2GbT+k/i8z1H+jnwYygNXA6KAzd9C8XwNygY0NP5YHnbm959xk7FtEwVEuIX6vjfrlpu3AFmBB0Jk7YM6nAO9RfwTMRuCCoDOHYc6PA4eBaur/13U9cAtwS6Pv868bfk+2tPXPt079FxGJEpG85CIiIq2gQhcRiRIqdBGRKKFCFxGJEip0EZEooUIXEYkSKnQRkSjx/wFB8NQWg/AorwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# using tanh function to get a smooth curve\n",
    "def Viseme_A(peak=None, lowest=None):\n",
    "    if not peak is None:\n",
    "        total = np.log((1+peak)/(1-peak))/2\n",
    "        b = np.log((1+lowest)/(1-lowest))/2\n",
    "        a = total-b\n",
    "    else:\n",
    "        peak = 0.99\n",
    "        lowest = 0.8\n",
    "        total = np.log((1+peak)/(1-peak))/2\n",
    "        b = np.log((1+lowest)/(1-lowest))/2\n",
    "        a = total-b\n",
    "    def fn(val, val_max, val_min, max_val = 8):\n",
    "        val = (val - val_min) / (val_max - val_min)\n",
    "        if val_min == val_max:\n",
    "            val = 1\n",
    "#         print(val)\n",
    "#         return (lowest + val * ((peak - lowest)))*max_val\n",
    "        return np.tanh((val)*a+b) * max_val\n",
    "#         return (np.exp(val * 8)/np.exp(8) * (peak-lowest) + lowest) * max_val\n",
    "    return fn\n",
    "viseme_A = Viseme_A()\n",
    "plt.plot(np.arange(0, 1, 0.01), viseme_A(np.arange(0, 1, 0.01), 1, 0, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93116468",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "82ac693d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load file for Jali\n",
    "lyric = Minimal_song_data_structure(os.path.join(dir, file_name_template+\".wav\"), os.path.join(dir, file_name_template+\".txt\"),\n",
    "                                                                                             script_path)\n",
    "# lyric = Minimal_song_data_structure(os.path.join(dir, file_name_template + \".pt\"),\n",
    "#                                     None,\n",
    "#                                     os.path.join(dir, file_name_template + \".TextGrid\"),\n",
    "#                                     use_torch=True)\n",
    "\n",
    "# break the thing into sentence structures (if possible)\n",
    "sentences = []\n",
    "current_sentence = []\n",
    "for i in range(0, len(lyric.phoneme_list)):\n",
    "    if lyric.phoneme_list[i] == \"EOS_tag\":\n",
    "        sentences.append(current_sentence)\n",
    "        current_sentence = []\n",
    "    else:\n",
    "        current_sentence.append(i)\n",
    "        if i == len(lyric.phoneme_list) - 1:\n",
    "            sentences.append(current_sentence)\n",
    "sentences = sentences[1:] \n",
    "# sentence stores the indexes\n",
    "if len(sentences) == 0:\n",
    "    sentences = [list(range(0, len(lyric.phoneme_list)))]\n",
    "phoneme_list = lyric.phoneme_list\n",
    "phoneme_interval = lyric.phoneme_intervals"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19ad5989",
   "metadata": {},
   "source": [
    "### Animating Consonant and Vowel visemes with Musical Articulation and relative Pitch Change (Ver original)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43fef378",
   "metadata": {},
   "source": [
    "### Ver 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1523ced7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_intervals(slopes, intervals, freq_interp, xs, min_length=0.3, flatness_threshold=60):\n",
    "    flat_intervals = []\n",
    "    flat_slopes = []\n",
    "    pass_next = False\n",
    "    for i in range(0, len(slopes)):\n",
    "        if pass_next:\n",
    "            pass_next = False\n",
    "            continue\n",
    "        if abs(slopes[i]) <= flatness_threshold:\n",
    "            if xs[intervals[i][1]] - xs[intervals[i][0]] >= min_length:\n",
    "                flat_intervals.append([xs[intervals[i][0]], xs[intervals[i][1]]])\n",
    "                flat_slopes.append(slopes[i])\n",
    "            else:\n",
    "                next_index = min(i+1, len(slopes)-1) \n",
    "                if next_index == i:\n",
    "                    continue\n",
    "                next_slope = slopes[next_index]\n",
    "                next_interval = intervals[next_index]\n",
    "                if xs[next_interval[0]] - xs[intervals[i][1]] <= 0.07 and abs(next_slope) <= flatness_threshold and (\n",
    "                freq_interp((xs[next_interval[0]] + xs[next_interval[1]])/2.0) - \n",
    "                    freq_interp((xs[intervals[i][0]] + xs[intervals[i][1]])/2.0)) <= 50:\n",
    "                    if xs[next_interval[1]] - xs[intervals[i][0]] >= min_length:\n",
    "                        flat_intervals.append([xs[intervals[i][0]], xs[next_interval[1]]])\n",
    "                        pass_next = True\n",
    "                        flat_slopes.append(next_slope)\n",
    "    return flat_intervals, flat_slopes        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "eebf0c96",
   "metadata": {},
   "outputs": [],
   "source": [
    "## phone_id = sentence[i]\n",
    "## return a pair of lists, [[singing_viseme, singing_curve, singing_phoneme], [speech_viseme, speech_curve, speech_phoneme]]\n",
    "def generate_pitch_sensitive_viseme_curve(lyric, phoneme_list, phoneme_interval, phone_id):\n",
    "    xI, yI = lyric.get_I_interval(phoneme_interval[phone_id])\n",
    "    xf, yf = lyric.get_f_interval(phoneme_interval[phone_id])\n",
    "    xF, yF = lyric.get_F1_interval(phoneme_interval[phone_id])\n",
    "    length_of_interval = xI[-1] - xI[0] \n",
    "    if length_of_interval <= speech_vowel_threshold: # if the vowel is approximately a speech vowel, then it is handled like speech\n",
    "        onset = 0.12\n",
    "        offset = 0.12\n",
    "        if phoneme_list[phone_id] in cmu_sets.lip_heavy:\n",
    "            onset = 0.16\n",
    "            offset = 0.16\n",
    "        value = 7\n",
    "        sustain = 0.75\n",
    "        decay = 0.75\n",
    "        curve = generate_basic_viseme_curve(phoneme_interval[phone_id][0], phoneme_interval[phone_id][1], value, sustain=sustain, \n",
    "                                    decay = decay, onset=onset, offset=offset)\n",
    "        singing_stuff = [CMU2VISEME[phoneme_list[phone_id]] + \"_pointer\", curve, phoneme_list[phone_id]]\n",
    "        speaking_stuff = [CMU2VISEME[phoneme_list[phone_id]] + \"_pointer\", curve, phoneme_list[phone_id]]\n",
    "        return [singing_stuff, speaking_stuff]\n",
    "    else:\n",
    "        onset = 0.12\n",
    "        offset = 0.12\n",
    "        if phoneme_list[phone_id] in cmu_sets.lip_heavy:\n",
    "            onset = 0.16\n",
    "            offset = 0.16\n",
    "        value = 7\n",
    "        sustain = 0.75\n",
    "        decay = 0.75\n",
    "        control_pts = generate_basic_viseme_curve(phoneme_interval[phone_id][0], phoneme_interval[phone_id][1], value, sustain=sustain, \n",
    "                                    decay = decay, onset=onset, offset=offset)\n",
    "        control_pts_with_pitch_change = []\n",
    "#                 vib = get_vib(yf-savgol_filter(yf, 29, 1), xf, lyric.dt)\n",
    "        vib = lyric.compute_vibrato_intervals(yf-savgol_filter(yf, 29, 1), xf, lyric.dt)\n",
    "        vib_interval_indexs = lyric.get_subarrays_indexes_from_time_interval(vib, xf)\n",
    "        for vib_int in vib_interval_indexs:\n",
    "            yf[vib_int[0]:vib_int[1]] = yf[vib_int[0]:vib_int[1]].mean()\n",
    "\n",
    "        slopes_f, intervals_f = efficient_piece_wise_linear_intervals(xf, yf)\n",
    "        kpx_f, kpy_f = get_key_points(xf, yf, intervals_f, slopes_f)\n",
    "        flat_intervals, flat_slopes = merge_intervals(slopes_f, intervals_f, interp1d(xf, yf), xf)\n",
    "        # onset and offset of these would be the same as regular vowels \n",
    "        control_pts_with_pitch_change.append([phoneme_interval[phone_id][0] - onset, 0, \"bound\"])\n",
    "        # here I will determine the average pitch when singing this vowel. Which is used to \n",
    "        avg_pitch = 0\n",
    "        total_weight = 0\n",
    "        pitch_values = []\n",
    "        for si in range(0, len(slopes_f)):\n",
    "            if abs(slopes_f[si]) <= threshold_slope:\n",
    "                avg_pitch = avg_pitch + (yf[intervals_f[si][0]] * (intervals_f[si][1] - intervals_f[si][0]))\n",
    "                pitch_values.append(yf[intervals_f[si][0]])\n",
    "                total_weight = total_weight + (intervals_f[si][1] - intervals_f[si][0])\n",
    "        if avg_pitch == 0:\n",
    "                avg_pitch = yf.mean()\n",
    "                total_weight = 1\n",
    "        pitch_values = np.array(pitch_values)\n",
    "        avg_pitch = avg_pitch / total_weight\n",
    "        start = 0\n",
    "        first_points_of_intervals = []\n",
    "        interp_pitch = interp1d(kpx_f, kpy_f)\n",
    "        # now find the first key point - i.e. the beginning of the first plateau\n",
    "        # the first key point is defined as the point where a plateau first appears \n",
    "        # it can also be the first point that reaches the same pitch as the first plateau\n",
    "        # this ensures that undetected vibrato will not mess up with the timing\n",
    "        # and cause the mouth to open too slowly \n",
    "\n",
    "        for si in range(0, len(flat_intervals)):\n",
    "            current_interval_x_range = np.arange(flat_intervals[si][0], flat_intervals[si][1], 0.01)\n",
    "            current_interval_f_range = interp_pitch(current_interval_x_range)\n",
    "            start = si\n",
    "            # make sure the search do not exceed the interpolation range\n",
    "            end_x_search = min(flat_intervals[si][0], kpx_f[-1])\n",
    "            begin_x_search = max(intervals_f[0][0], kpx_f[0])\n",
    "            x_range = np.arange(begin_x_search, end_x_search, 0.01)\n",
    "            f_range = interp_pitch(x_range)\n",
    "            first_val = interp_pitch(flat_intervals[si][0])\n",
    "            for ssi in range(0, x_range.shape[0]-1):\n",
    "                if ((f_range[ssi] < first_val) and  (f_range[ssi+1] >= first_val) \n",
    "                    or (f_range[ssi] >= first_val) and  (f_range[ssi+1] < first_val)):\n",
    "                    end_x_search = x_range[ssi]\n",
    "                    break\n",
    "            val0 = viseme_A(current_interval_f_range.max(), yf.max(), yf.min())\n",
    "            height_ratio = dip_percentage[CMU2VISEME[phoneme_list[phone_id]] + \"_pointer\"]\n",
    "            val1 = viseme_A(current_interval_f_range.max(), yf.max(), yf.min()) * height_ratio\n",
    "            control_pts_with_pitch_change.append([flat_intervals[si][0], val0, \"internal\"])\n",
    "            if len(flat_intervals) > 1:\n",
    "#                         dif = max((flat_intervals[si][1] - flat_intervals[si][0]) * 0.9, flat_intervals[si+1][0] - flat_intervals[si][1]- 0.025)\n",
    "                dif = flat_intervals[si+1][0] - flat_intervals[si][0]- 0.1\n",
    "            else:\n",
    "#                         dif = max((flat_intervals[si][1] - flat_intervals[si][0]) * 0.9, flat_intervals[si][1] - flat_intervals[si][0]- 0.025)\n",
    "                dif = flat_intervals[si][1] - flat_intervals[si][0]- 0.1\n",
    "            control_pts_with_pitch_change.append([flat_intervals[si][0] + dif, val1, \"internal\"])                 \n",
    "            break\n",
    "        # now determine intermediate key-points that correlates with lipshape\n",
    "        for si in range(start+1, len(flat_intervals)):\n",
    "            current_interval_x_range = np.arange(flat_intervals[si][0], flat_intervals[si][1], 0.01)\n",
    "            current_interval_f_range = interp_pitch(current_interval_x_range)\n",
    "\n",
    "            val0 = viseme_A(current_interval_f_range.max(), yf.max(), yf.min())\n",
    "            height_ratio = dip_percentage[CMU2VISEME[phoneme_list[phone_id]] + \"_pointer\"]\n",
    "#                     height_ratio = 0.9\n",
    "            val1 = viseme_A(current_interval_f_range.max(), yf.max(), yf.min()) * height_ratio\n",
    "            if len(flat_intervals) - 1> si:\n",
    "                # if it's not the last interval\n",
    "                # dif = max((flat_intervals[si][1] - flat_intervals[si][0]) * 0.9, flat_intervals[si+1][0] - flat_intervals[si][1]- 0.025)\n",
    "                dif = flat_intervals[si+1][0] - flat_intervals[si][0]- 0.1\n",
    "            else:\n",
    "                # if it is the last interval\n",
    "#                         dif = max((flat_intervals[si][1] - flat_intervals[si][0]) * 0.9, flat_intervals[si][1] - flat_intervals[si][0]- 0.025)\n",
    "                dif = flat_intervals[si][1] - flat_intervals[si][0]- 0.1\n",
    "            prev_interval_end_pt_x = control_pts_with_pitch_change[-1][0]\n",
    "            prev_interval_end_pt_y = min(control_pts_with_pitch_change[-1][1], val0)\n",
    "#                     prev_interval_end_pt_y = min(control_pts_with_pitch_change[-1][1], val0) * dip_percentage[CMU2VISEME[phoneme_list[sentence[i]]] + \"_pointer\"]\n",
    "\n",
    "            control_pts_with_pitch_change.append([(flat_intervals[si][0] + prev_interval_end_pt_x)/2.0, prev_interval_end_pt_y, \"internal_ph\"])    \n",
    "            control_pts_with_pitch_change.append([flat_intervals[si][0], val0, \"internal\"])\n",
    "#                     if len(flat_intervals) - 1 == si:\n",
    "#                         control_pts_with_pitch_change.append([flat_intervals[si][0] + dif, val0 * 0.8, \"internal\"])\n",
    "#                     else:\n",
    "#                         control_pts_with_pitch_change.append([flat_intervals[si][0] + dif, val1, \"internal_ph\"])\n",
    "            control_pts_with_pitch_change.append([flat_intervals[si][0] + dif, val1, \"internal\"])\n",
    "        control_pts_with_pitch_change.append([phoneme_interval[phone_id][1] + offset, 0, \"bound\"])\n",
    "        # if I cannot find any flat intervals in the vowel, then this step is spared\n",
    "        if len(control_pts_with_pitch_change) < 4:\n",
    "            onset = 0.12\n",
    "            offset = 0.12\n",
    "            if phoneme_list[phone_id] in cmu_sets.lip_heavy:\n",
    "                onset = 0.16\n",
    "                offset = 0.16\n",
    "            value = 7\n",
    "            sustain = 0.75\n",
    "            decay = 0.75\n",
    "            control_pts_with_pitch_change = generate_basic_viseme_curve(phoneme_interval[phone_id][0], phoneme_interval[phone_id][1], value, sustain=sustain, \n",
    "                                        decay = decay, onset=onset, offset=offset)\n",
    "            extended_vowel_list.append(CMU2VISEME[phoneme_list[phone_id]] + \"_pointer\")\n",
    "            extended_vowel_intervals.append(phoneme_interval[phone_id])\n",
    "        else:\n",
    "            for item in flat_intervals:\n",
    "                extended_vowel_list.append(CMU2VISEME[phoneme_list[phone_id]] + \"_pointer_pitch_change\")\n",
    "                extended_vowel_intervals.append(item)\n",
    "        control_pts_with_pitch_change_final = []\n",
    "        skip = False\n",
    "        control_pts_with_pitch_change.sort(key=lambda x:x[0])\n",
    "        for mi in range(0, len(control_pts_with_pitch_change)):\n",
    "            if skip:\n",
    "                skip = False\n",
    "                continue\n",
    "            if control_pts_with_pitch_change[mi][2] == \"internal_ph\":\n",
    "                if mi != len(control_pts_with_pitch_change)-2:\n",
    "                    new_pt = [control_pts_with_pitch_change[mi+1][0], \n",
    "                              control_pts_with_pitch_change[mi+1][1],\n",
    "                              \"internal\"]\n",
    "                    control_pts_with_pitch_change_final.append(new_pt)\n",
    "                    skip = True\n",
    "                else:\n",
    "                    new_pt = [control_pts_with_pitch_change[mi][0], \n",
    "                              control_pts_with_pitch_change[mi][1],\n",
    "                               \"internal\"]\n",
    "                    control_pts_with_pitch_change_final.append(new_pt)  \n",
    "            else:\n",
    "                control_pts_with_pitch_change_final.append(control_pts_with_pitch_change[mi])\n",
    "                \n",
    "        singing_stuff = [CMU2VISEME[phoneme_list[phone_id]] + \"_pointer\", control_pts_with_pitch_change_final, phoneme_list[phone_id]]\n",
    "        speaking_stuff = [CMU2VISEME[phoneme_list[phone_id]] + \"_pointer\", control_pts, phoneme_list[phone_id]]\n",
    "        return [singing_stuff, speaking_stuff]\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a4190739",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "visemes_sing = Viseme_curve()\n",
    "visemes_sing.new_pass()\n",
    "visemes_speak = Viseme_curve()\n",
    "visemes_speak.new_pass()\n",
    "\n",
    "extended_vowel_intervals = []\n",
    "extended_vowel_list = []\n",
    "\n",
    "# animate only vowels and see how it goes\n",
    "max_activation = 8\n",
    "# threshold_slope = 200 # for formants\n",
    "threshold_slope = 60 # for pitch\n",
    "\n",
    "for sentence in sentences:\n",
    "    for i in range(0, len(sentence)):\n",
    "        # skip the misc symbols in the sentence\n",
    "        if phoneme_list[sentence[i]] in VOICED: # basically vowels\n",
    "            singing_stuff, speaking_stuff = generate_pitch_sensitive_viseme_curve(lyric, phoneme_list, phoneme_interval, sentence[i])\n",
    "            visemes_sing.add(singing_stuff[0], singing_stuff[1], singing_stuff[2])\n",
    "            visemes_speak.add(speaking_stuff[0], speaking_stuff[1], speaking_stuff[2])\n",
    "                \n",
    "        ### Dealing with consonants here. I'm just going to insert them between without too much modification\n",
    "        ### it will about the same as pure Jali\n",
    "        elif phoneme_list[sentence[i]] in cmu_sets.consonants:\n",
    "#             if phoneme_list[sentence[i]] in cmu_sets.voiced:\n",
    "            onset = 0.12\n",
    "            offset = 0.12\n",
    "            if phoneme_list[sentence[i]] == \"HH\" or phoneme_list[sentence[i]] in cmu_sets.sibilant:\n",
    "                viseme_jali = CMU2VISEME[phoneme_list[sentence[i]]] + \"_pointer\"\n",
    "            else:\n",
    "                viseme_jali = CMU2VISEME[phoneme_list[sentence[i]]] + \"a_pointer\"\n",
    "            if viseme_jali in jali_sets.lip_heavy:\n",
    "                onset = 0.16\n",
    "                offset = 0.16\n",
    "            start = phoneme_interval[sentence[i]][0]\n",
    "            end = phoneme_interval[sentence[i]][1]\n",
    "            if (end - start) <= 0.1:\n",
    "                value = 6\n",
    "                sustain = 0.75\n",
    "                decay = 0.75\n",
    "            elif (end - start) <= 0.3:\n",
    "                value = 6\n",
    "                sustain = 0.75\n",
    "                decay = 0.75\n",
    "            else:\n",
    "                value = 8\n",
    "                sustain = 0.75\n",
    "                decay = 0.75\n",
    "            if phoneme_list[sentence[i]] in cmu_sets.lip_closer:\n",
    "                value = 10\n",
    "            viseme_curve = generate_basic_viseme_curve(start, end, value, sustain=sustain, decay=decay, onset=onset, offset=offset)\n",
    "            visemes_sing.add(viseme_jali, viseme_curve, phoneme_list[sentence[i]])\n",
    "            visemes_speak.add(viseme_jali, viseme_curve, phoneme_list[sentence[i]])\n",
    "            # add jaw component to voiced consonants if it's sufficiently voiced\n",
    "            prev_vowel = -1\n",
    "            for si in range(i-1, -1, -1):\n",
    "                if phoneme_list[si] in cmu_sets.vowels:\n",
    "                    prev_vowel = si\n",
    "                    break\n",
    "            if (phoneme_list[sentence[i]] in set([\"N\", \"NG\"]) \n",
    "                and phoneme_interval[sentence[i]][1] - phoneme_interval[sentence[i]][0] > 1/20\n",
    "                and (phoneme_interval[sentence[i]][0] - phoneme_interval[sentence[prev_vowel]][1] > 0.1 \n",
    "                    or prev_vowel == -1\n",
    "                    )):\n",
    "                singing_stuff, speaking_stuff = generate_pitch_sensitive_viseme_curve(lyric, phoneme_list, phoneme_interval, sentence[i])\n",
    "                visemes_sing.add(singing_stuff[0], singing_stuff[1], singing_stuff[2])\n",
    "                visemes_speak.add(speaking_stuff[0], speaking_stuff[1], speaking_stuff[2])\n",
    "            elif (phoneme_list[sentence[i]] == \"L\"\n",
    "                and phoneme_interval[sentence[i]][1] - phoneme_interval[sentence[i]][0] > 1/20):\n",
    "                singing_stuff, speaking_stuff = generate_pitch_sensitive_viseme_curve(lyric, phoneme_list, phoneme_interval, sentence[i])\n",
    "                visemes_sing.add(singing_stuff[0], singing_stuff[1], singing_stuff[2])\n",
    "                visemes_speak.add(speaking_stuff[0], speaking_stuff[1], speaking_stuff[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5bb84b23",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wa_pointer [[1.3643125, 0, 'bound'], [1.5243125, 6, 'internal'], [1.5993125, 4.5, 'internal'], [1.7843125, 0, 'bound']]\n",
      "U_pointer [[1.4123125, 0, 'bound'], [1.7723125000000002, 7.915407692444188, 'internal'], [3.0023125000000013, 8.311178077066398, 'internal'], [3.1023125000000014, 7.8966231091209, 'internal'], [4.432312500000002, 8.291454264576945, 'internal'], [4.532312500000002, 7.832702485400464, 'internal'], [5.072312500000003, 8.224337609670487, 'internal'], [8.485849943073065, 0, 'bound']]\n",
      "Wa_pointer [[9.710996919510263, 0, 'bound'], [9.870996919510263, 6, 'internal'], [9.982120503432117, 4.5, 'internal'], [10.179161698072734, 0, 'bound']]\n",
      "LNTDa_pointer [[17.0795, 0, 'bound'], [17.1995, 6, 'internal'], [17.307499999999997, 4.5, 'internal'], [17.4635, 0, 'bound']]\n",
      "LNTD_pointer [[17.0795, 0, 'bound'], [17.1995, 7, 'internal'], [17.307499999999997, 5.25, 'internal'], [17.4635, 0, 'bound']]\n",
      "Ah_pointer [[17.223499999999998, 0, 'bound'], [17.703500000000055, 7.865992949115078, 'internal'], [19.68350000000038, 7.079393654203571, 'internal'], [19.9435, 0, 'bound']]\n",
      "Ya_pointer [[19.686230325391065, 0, 'bound'], [19.846230325391065, 6, 'internal'], [19.921230325391065, 4.5, 'internal'], [20.106230325391067, 0, 'bound']]\n",
      "U_pointer [[19.753976206191012, 0, 'bound'], [19.913976206191013, 7, 'internal'], [20.254119051547754, 5.25, 'internal'], [20.5275, 0, 'bound']]\n",
      "SZ_pointer [[20.2235, 0, 'bound'], [20.3835, 10, 'internal'], [20.6355, 7.5, 'internal'], [20.8795, 0, 'bound']]\n",
      "Eh_pointer [[20.5995, 0, 'bound'], [21.059500000000053, 7.8263273509398275, 'internal'], [21.489500000000135, 7.043694615845845, 'internal'], [21.7835, 0, 'bound']]\n",
      "Ya_pointer [[21.5195, 0, 'bound'], [21.6795, 6, 'internal'], [21.792537625899286, 4.5, 'internal'], [21.99021683453238, 0, 'bound']]\n",
      "U_pointer [[21.71021683453238, 0, 'bound'], [21.83021683453238, 7, 'internal'], [22.005179208633095, 5.25, 'internal'], [22.183500000000002, 0, 'bound']]\n",
      "Ra_pointer [[21.9435, 0, 'bound'], [22.0635, 6, 'internal'], [22.1385, 4.5, 'internal'], [22.283500000000004, 0, 'bound']]\n",
      "LNTDa_pointer [[21.9755, 0, 'bound'], [22.0955, 6, 'internal'], [22.179618627813348, 4.5, 'internal'], [22.3276581704178, 0, 'bound']]\n",
      "LNTD_pointer [[21.9755, 0, 'bound'], [22.0955, 7, 'internal'], [22.179618627813348, 5.25, 'internal'], [22.3276581704178, 0, 'bound']]\n",
      "Oo_pointer [[22.047658170417797, 0, 'bound'], [22.207658170417798, 7, 'internal'], [22.495539542604448, 5.25, 'internal'], [22.7515, 0, 'bound']]\n",
      "LNTDa_pointer [[22.4715, 0, 'bound'], [22.5915, 6, 'internal'], [22.735500000000002, 4.5, 'internal'], [22.9035, 0, 'bound']]\n",
      "LNTDa_pointer [[22.6635, 0, 'bound'], [22.7835, 6, 'internal'], [22.8585, 4.5, 'internal'], [23.003500000000003, 0, 'bound']]\n",
      "Ee_pointer [[22.6955, 0, 'bound'], [22.965500000000024, 7.33644364054075, 'internal'], [23.895500000000183, 8.070088004594826, 'internal'], [24.183500000000002, 0, 'bound']]\n",
      "Ya_pointer [[25.702125, 0, 'bound'], [25.862125, 6, 'internal'], [25.937125, 4.5, 'internal'], [26.122125, 0, 'bound']]\n",
      "U_pointer [[25.782125, 0, 'bound'], [25.942125, 7, 'internal'], [26.170125, 5.25, 'internal'], [26.406125, 0, 'bound']]\n",
      "GKa_pointer [[26.158125, 0, 'bound'], [26.278125, 6, 'internal'], [26.398125, 4.5, 'internal'], [26.558125, 0, 'bound']]\n",
      "Ra_pointer [[26.318125, 0, 'bound'], [26.438125, 6, 'internal'], [26.546125, 4.5, 'internal'], [26.702125000000002, 0, 'bound']]\n",
      "Ah_pointer [[26.462125, 0, 'bound'], [26.582125, 7, 'internal'], [26.978125, 5.25, 'internal'], [27.230125, 0, 'bound']]\n",
      "LNTDa_pointer [[26.990125, 0, 'bound'], [27.110125, 6, 'internal'], [27.185125, 4.5, 'internal'], [27.330125000000002, 0, 'bound']]\n",
      "Tha_pointer [[27.022125, 0, 'bound'], [27.142125, 6, 'internal'], [27.226125000000003, 4.5, 'internal'], [27.374125000000003, 0, 'bound']]\n",
      "Eh_pointer [[27.134125, 0, 'bound'], [27.254125000000002, 7, 'internal'], [27.410125, 5.25, 'internal'], [27.582125, 0, 'bound']]\n",
      "LNTDa_pointer [[27.358125, 0, 'bound'], [27.478125000000002, 6, 'internal'], [27.682125, 4.5, 'internal'], [27.870125, 0, 'bound']]\n",
      "LNTD_pointer [[27.358125, 0, 'bound'], [27.478125000000002, 7, 'internal'], [27.682125, 5.25, 'internal'], [27.870125, 0, 'bound']]\n",
      "Ah_pointer [[27.630125, 0, 'bound'], [27.94012500000003, 7.081725633995797, 'internal'], [28.320125000000104, 6.373553070596217, 'internal'], [28.420125000000105, 7.081725633995797, 'internal'], [28.75012500000017, 6.373553070596217, 'internal'], [28.974125, 0, 'bound']]\n",
      "LNTDa_pointer [[28.734125, 0, 'bound'], [28.854125, 6, 'internal'], [28.929125, 4.5, 'internal'], [29.074125000000002, 0, 'bound']]\n",
      "LNTDa_pointer [[28.766125, 0, 'bound'], [28.886125, 6, 'internal'], [28.994125, 4.5, 'internal'], [29.150125000000003, 0, 'bound']]\n",
      "Ah_pointer [[28.910125, 0, 'bound'], [29.030125, 7, 'internal'], [29.354125, 5.25, 'internal'], [29.582125, 0, 'bound']]\n",
      "LNTDa_pointer [[29.342125, 0, 'bound'], [29.462125, 6, 'internal'], [29.537125000000003, 4.5, 'internal'], [29.682125000000003, 0, 'bound']]\n",
      "Tha_pointer [[29.374125, 0, 'bound'], [29.494125, 6, 'internal'], [29.569125, 4.5, 'internal'], [29.714125000000003, 0, 'bound']]\n",
      "Ra_pointer [[29.470125, 0, 'bound'], [29.590125, 6, 'internal'], [29.665125000000003, 4.5, 'internal'], [29.810125000000003, 0, 'bound']]\n",
      "U_pointer [[29.462125, 0, 'bound'], [29.642125000000004, 7.559478477682853, 'internal'], [31.212125000000263, 7.937452401566996, 'internal'], [31.606125000000002, 0, 'bound']]\n",
      "Wa_pointer [[31.8358125, 0, 'bound'], [31.9958125, 6, 'internal'], [32.0798125, 4.5, 'internal'], [32.2678125, 0, 'bound']]\n",
      "Eh_pointer [[31.9878125, 0, 'bound'], [32.1078125, 7, 'internal'], [32.3238125, 5.25, 'internal'], [32.515812499999996, 0, 'bound']]\n",
      "LNTDa_pointer [[32.2758125, 0, 'bound'], [32.3958125, 6, 'internal'], [32.4708125, 4.5, 'internal'], [32.6158125, 0, 'bound']]\n",
      "Ya_pointer [[32.267812500000005, 0, 'bound'], [32.4278125, 6, 'internal'], [32.502812500000005, 4.5, 'internal'], [32.6878125, 0, 'bound']]\n",
      "U_pointer [[32.2998125, 0, 'bound'], [32.4598125, 7, 'internal'], [32.5918125, 5.25, 'internal'], [32.7958125, 0, 'bound']]\n",
      "GKa_pointer [[32.547812500000006, 0, 'bound'], [32.667812500000004, 6, 'internal'], [32.74281250000001, 4.5, 'internal'], [32.8878125, 0, 'bound']]\n",
      "Eh_pointer [[32.6438125, 0, 'bound'], [32.7638125, 7, 'internal'], [32.9678125, 5.25, 'internal'], [33.155812499999996, 0, 'bound']]\n",
      "LNTDa_pointer [[32.9158125, 0, 'bound'], [33.0358125, 6, 'internal'], [33.1198125, 4.5, 'internal'], [33.2678125, 0, 'bound']]\n",
      "GKa_pointer [[33.0438125, 0, 'bound'], [33.1638125, 6, 'internal'], [33.2718125, 4.5, 'internal'], [33.4278125, 0, 'bound']]\n",
      "Ra_pointer [[33.18781250000001, 0, 'bound'], [33.307812500000004, 6, 'internal'], [33.38281250000001, 4.5, 'internal'], [33.5278125, 0, 'bound']]\n",
      "Ah_pointer [[33.2838125, 0, 'bound'], [33.60381249999996, 7.815304997299942, 'internal'], [34.073812499999846, 7.033774497569948, 'internal'], [34.5958125, 0, 'bound']]\n",
      "Ma_pointer [[34.371812500000004, 0, 'bound'], [34.4918125, 10, 'internal'], [34.5878125, 7.5, 'internal'], [34.7398125, 0, 'bound']]\n",
      "Ee_pointer [[34.499812500000004, 0, 'bound'], [34.6198125, 7, 'internal'], [34.8358125, 5.25, 'internal'], [35.027812499999996, 0, 'bound']]\n",
      "Eh_pointer [[34.80381250000001, 0, 'bound'], [34.923812500000004, 7, 'internal'], [34.99881250000001, 5.25, 'internal'], [35.1438125, 0, 'bound']]\n",
      "Ra_pointer [[34.8998125, 0, 'bound'], [35.0198125, 6, 'internal'], [35.139812500000005, 4.5, 'internal'], [35.2998125, 0, 'bound']]\n",
      "Ee_pointer [[35.05981250000001, 0, 'bound'], [35.179812500000004, 7, 'internal'], [35.340036161804505, 5.25, 'internal'], [35.51344404907267, 0, 'bound']]\n",
      "FVa_pointer [[35.27344404907267, 0, 'bound'], [35.39344404907267, 10, 'internal'], [35.46844404907267, 7.5, 'internal'], [35.61344404907267, 0, 'bound']]\n",
      "Eh_pointer [[35.34697883885553, 0, 'bound'], [35.77697883885546, 7.577938615119296, 'internal'], [36.516978838855294, 6.820144753607366, 'internal'], [36.80555535525708, 0, 'bound']]\n",
      "GKa_pointer [[37.16433864348888, 0, 'bound'], [37.28433864348888, 6, 'internal'], [37.35933864348888, 4.5, 'internal'], [37.50433864348888, 0, 'bound']]\n",
      "Ra_pointer [[37.2588833732097, 0, 'bound'], [37.378883373209696, 6, 'internal'], [37.4538833732097, 4.5, 'internal'], [37.598883373209695, 0, 'bound']]\n",
      "Ah_pointer [[37.332418162992546, 0, 'bound'], [37.452418162992544, 7, 'internal'], [38.07596391574814, 5.25, 'internal'], [38.4038125, 0, 'bound']]\n",
      "Ma_pointer [[38.179812500000004, 0, 'bound'], [38.2998125, 10, 'internal'], [38.374812500000004, 7.5, 'internal'], [38.5198125, 0, 'bound']]\n",
      "Ee_pointer [[38.243812500000004, 0, 'bound'], [38.3638125, 7, 'internal'], [38.4958125, 5.25, 'internal'], [38.6598125, 0, 'bound']]\n",
      "Eh_pointer [[38.435812500000004, 0, 'bound'], [38.5558125, 7, 'internal'], [38.639812500000005, 5.25, 'internal'], [38.7878125, 0, 'bound']]\n",
      "Ra_pointer [[38.563812500000004, 0, 'bound'], [38.6838125, 6, 'internal'], [38.760865136789654, 4.5, 'internal'], [38.90654934905287, 0, 'bound']]\n",
      "Ee_pointer [[38.66654934905287, 0, 'bound'], [38.78654934905287, 7, 'internal'], [38.91349671226322, 5.25, 'internal'], [39.0758125, 0, 'bound']]\n",
      "FVa_pointer [[38.8358125, 0, 'bound'], [38.9558125, 10, 'internal'], [39.0308125, 7.5, 'internal'], [39.1758125, 0, 'bound']]\n",
      "Eh_pointer [[38.9158125, 0, 'bound'], [39.0358125, 7, 'internal'], [39.3118125, 5.25, 'internal'], [39.5238125, 0, 'bound']]\n",
      "Ah_pointer [[40.80771077037209, 0, 'bound'], [40.92771077037209, 7, 'internal'], [41.19546191346823, 5.25, 'internal'], [41.40471229450027, 0, 'bound']]\n",
      "GKa_pointer [[41.181088511203406, 0, 'bound'], [41.3010885112034, 6, 'internal'], [41.376088511203406, 4.5, 'internal'], [41.5210885112034, 0, 'bound']]\n",
      "Ra_pointer [[41.23349240465342, 0, 'bound'], [41.353492404653416, 6, 'internal'], [41.42849240465342, 4.5, 'internal'], [41.573492404653415, 0, 'bound']]\n",
      "Ah_pointer [[41.29244678478468, 0, 'bound'], [41.41244678478468, 7, 'internal'], [41.66075109558514, 5.25, 'internal'], [41.8635191991853, 0, 'bound']]\n",
      "LNTDa_pointer [[41.6235191991853, 0, 'bound'], [41.7435191991853, 6, 'internal'], [41.8185191991853, 4.5, 'internal'], [41.9635191991853, 0, 'bound']]\n",
      "Eh_pointer [[41.68125077696235, 0, 'bound'], [41.80125077696235, 7, 'internal'], [41.96548716201775, 5.25, 'internal'], [42.140232623702886, 0, 'bound']]\n",
      "Ra_pointer [[41.901250000000005, 0, 'bound'], [42.02125, 6, 'internal'], [42.096250000000005, 4.5, 'internal'], [42.24125, 0, 'bound']]\n",
      "Ee_pointer [[41.93325, 0, 'bound'], [42.05325, 7, 'internal'], [42.185249999999996, 5.25, 'internal'], [42.34925, 0, 'bound']]\n",
      "FVa_pointer [[42.10925, 0, 'bound'], [42.22925, 10, 'internal'], [42.30425, 7.5, 'internal'], [42.44925, 0, 'bound']]\n",
      "Eh_pointer [[42.17325, 0, 'bound'], [42.33324999999999, 7.810970982958964, 'internal'], [42.80324999999988, 7.0298738846630675, 'internal'], [43.25009048049634, 0, 'bound']]\n",
      "Oo_pointer [[43.032829737311076, 0, 'bound'], [43.50282973731101, 7.8933689168606485, 'internal'], [43.80282973731093, 7.4987004710176155, 'internal'], [44.149249999999995, 0, 'bound']]\n",
      "FVa_pointer [[43.86925, 0, 'bound'], [43.98925, 10, 'internal'], [44.06425, 7.5, 'internal'], [44.20925, 0, 'bound']]\n",
      "Eh_pointer [[43.91725, 0, 'bound'], [44.03725, 7, 'internal'], [44.157250000000005, 5.25, 'internal'], [44.31725, 0, 'bound']]\n",
      "Ya_pointer [[44.053250000000006, 0, 'bound'], [44.21325, 6, 'internal'], [44.335247149363084, 4.5, 'internal'], [44.535912865817444, 0, 'bound']]\n",
      "U_pointer [[44.21591286581745, 0, 'bound'], [44.37591286581745, 7.627195685065001, 'internal'], [46.68591286581697, 8.008555469318251, 'internal'], [47.06125, 0, 'bound']]\n"
     ]
    }
   ],
   "source": [
    "for i in range(0, len(visemes_sing.viseme_ctrl_pts[-1])):\n",
    "    print(visemes_sing.viseme_list[-1][i], visemes_sing.viseme_ctrl_pts[-1][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8aa1252b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# enforce co-articulation rules of consonants for speech visemes\n",
    "i = 0;\n",
    "viseme_list = copy.deepcopy(visemes_speak.viseme_list[-1])\n",
    "pure_phoneme_list = copy.deepcopy(visemes_speak.pure_phoneme[-1])\n",
    "viseme_intervals = copy.deepcopy(visemes_speak.viseme_ctrl_pts[-1])\n",
    "visemes_speak.new_pass()\n",
    "while i < len(viseme_list):\n",
    "    increment = 1\n",
    "    i_next = min(i + 1, len(viseme_list)-1)\n",
    "    if ((viseme_list[i_next] == viseme_list[i] and not viseme_list[i] in jali_sets.vowels) or (pure_phoneme_list[i] == \"HH\" and pure_phoneme_list[i_next] in cmu_sets.vowels)\n",
    "        and viseme_intervals[i][-1][0] >= viseme_intervals[i_next][0][0]):\n",
    "        # remove repeated vowels or consonants\n",
    "        int_curr = viseme_intervals[i]\n",
    "        int_next = viseme_intervals[i_next]\n",
    "        viseme_interval = [int_curr[0], [int_curr[1][0], max(int_curr[1][1], int_next[1][1]), int_curr[1][2]]]\n",
    "        for interv in int_next[2:]:\n",
    "            viseme_interval.append(interv)                \n",
    "        visemes_speak.add(viseme_list[i_next], copy.deepcopy(viseme_interval), pure_phoneme_list[i_next])\n",
    "#         if viseme_list[i_next] in jali_sets.lip_rounder:\n",
    "#             visemes_sing(jali_sets.lip_rounder_no_jaw_dict[viseme_list[i_next]], viseme_interval, pure_phoneme_list[i_next])\n",
    "        increment = 2\n",
    "    elif viseme_list[i] in jali_sets.lip_heavy:\n",
    "        # if the viseme is a lip-heavy viseme, the it is voice simutaneously as nearby labial dental and bilabials \n",
    "        current_interval = copy.deepcopy(viseme_intervals[i])\n",
    "        if not get_kth_neighbour(viseme_list, i, -1) is None:\n",
    "            if current_interval[0][0] <= viseme_intervals[i-1][-1][0] - lyric.dt and viseme_intervals[i-1][-1][0] in jali_sets.lip_rounder:\n",
    "                current_interval[0][0] = viseme_intervals[i-1][0][0]\n",
    "                current_interval[1][0] = viseme_intervals[i-1][1][0]\n",
    "        if not get_kth_neighbour(viseme_list, i, +1) is None:\n",
    "            if current_interval[-1][0] <= viseme_intervals[i+1][0][0] - lyric.dt and viseme_intervals[i+1][-1][0] in jali_sets.lip_rounder:\n",
    "                current_interval[2][0] = viseme_intervals[i+1][0][0]\n",
    "                current_interval[3][0] = viseme_intervals[i+1][1][0]\n",
    "#         if viseme_list[i] in jali_sets.lip_rounder:\n",
    "#             visemes_sing(jali_sets.lip_rounder_no_jaw_dict[viseme_list[i]], current_interval, pure_phoneme_list[i])\n",
    "        visemes_speak.add(viseme_list[i], copy.deepcopy(current_interval), pure_phoneme_list[i])\n",
    "    else:\n",
    "        visemes_speak.add(viseme_list[i], copy.deepcopy(viseme_intervals[i]), pure_phoneme_list[i])\n",
    "    i = i + increment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "cdccded5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# enforce co-articulation rules of consonants for singing visemes\n",
    "i = 0;\n",
    "viseme_list = copy.deepcopy(visemes_sing.viseme_list[-1])\n",
    "pure_phoneme_list = copy.deepcopy(visemes_sing.pure_phoneme[-1])\n",
    "viseme_intervals = copy.deepcopy(visemes_sing.viseme_ctrl_pts[-1])\n",
    "visemes_sing.new_pass()\n",
    "while i < len(viseme_list):\n",
    "    increment = 1\n",
    "    i_next = min(i + 1, len(viseme_list)-1)\n",
    "    if ((viseme_list[i_next] == viseme_list[i] and not viseme_list[i] in jali_sets.vowels) or (pure_phoneme_list[i] == \"HH\" and pure_phoneme_list[i_next] in cmu_sets.vowels)\n",
    "        and viseme_intervals[i][-1][0] >= viseme_intervals[i_next][0][0]):\n",
    "        # remove repeated vowels or consonants\n",
    "        int_curr = viseme_intervals[i]\n",
    "        int_next = viseme_intervals[i_next]\n",
    "        viseme_interval = [int_curr[0], [int_curr[1][0], max(int_curr[1][1], int_next[1][1]), int_curr[1][2]]]\n",
    "        for interv in int_next[2:]:\n",
    "            viseme_interval.append(interv)                \n",
    "        visemes_sing.add(viseme_list[i_next], copy.deepcopy(viseme_interval), pure_phoneme_list[i_next])\n",
    "#         if viseme_list[i_next] in jali_sets.lip_rounder:\n",
    "#             visemes_sing(jali_sets.lip_rounder_no_jaw_dict[viseme_list[i_next]], viseme_interval, pure_phoneme_list[i_next])\n",
    "        increment = 2\n",
    "    elif viseme_list[i] in jali_sets.lip_heavy:\n",
    "        # if the viseme is a lip-heavy viseme, the it is voice simutaneously as nearby labial dental and bilabials \n",
    "        current_interval = viseme_intervals[i] \n",
    "        if not get_kth_neighbour(viseme_list, i, -1) is None:\n",
    "            if current_interval[0][0] <= viseme_intervals[i-1][-1][0] - lyric.dt and viseme_intervals[i-1][-1][0] in jali_sets.lip_rounder:\n",
    "                current_interval[0][0] = viseme_intervals[i-1][0][0]\n",
    "                current_interval[1][0] = viseme_intervals[i-1][1][0]\n",
    "        if not get_kth_neighbour(viseme_list, i, +1) is None:\n",
    "            if current_interval[-1][0] <= viseme_intervals[i+1][0][0] - lyric.dt and viseme_intervals[i+1][-1][0] in jali_sets.lip_rounder:\n",
    "                current_interval[2][0] = viseme_intervals[i+1][0][0]\n",
    "                current_interval[3][0] = viseme_intervals[i+1][1][0]\n",
    "#         if viseme_list[i] in jali_sets.lip_rounder:\n",
    "#             visemes_sing(jali_sets.lip_rounder_no_jaw_dict[viseme_list[i]], current_interval, pure_phoneme_list[i])\n",
    "        visemes_sing.add(viseme_list[i], copy.deepcopy(current_interval), pure_phoneme_list[i])\n",
    "    else:\n",
    "        visemes_sing.add(viseme_list[i], copy.deepcopy(viseme_intervals[i]), pure_phoneme_list[i])\n",
    "    i = i + increment"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d25fad9a",
   "metadata": {},
   "source": [
    "## 2.2 smooth out un-intended artifacts due to unexpected overlaps for speech visemes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0b08d8be",
   "metadata": {
    "scrolled": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "# pass 3\n",
    "# set this up\n",
    "prev_slider_dict = {}\n",
    "for i in range(0, len(list(jali_sets.vocabs))):\n",
    "    prev_slider_dict[list(jali_sets.vocabs)[i]] = -1\n",
    "viseme_list_final = copy.deepcopy(visemes_speak.viseme_list[-1])\n",
    "viseme_intervals_final = copy.deepcopy(visemes_speak.viseme_ctrl_pts[-1])\n",
    "pure_phoneme_list = copy.deepcopy(visemes_speak.pure_phoneme[-1])\n",
    "visemes_speak.new_pass()\n",
    "i = 0\n",
    "while i < len(visemes_speak.viseme_list[-2]):\n",
    "    increment = 1\n",
    "    # prev_viseme = viseme_list_final[i]\n",
    "    # if the previous instance of the current viseme is not -1\n",
    "    if prev_slider_dict[viseme_list_final[i]] != -1:\n",
    "        current_interval = copy.deepcopy(viseme_intervals_final[i])\n",
    "        prev_interval = copy.deepcopy(visemes_speak.viseme_ctrl_pts[-1][prev_slider_dict[viseme_list_final[i]]])\n",
    "        # For the same vowel/consonant if the two intervals do not overlap but the onset and off set overlap \n",
    "        if (current_interval[0][0] <= prev_interval[-1][0]):\n",
    "            # if we have weird overlap where the boundary of the previous interval interupts the curve of the current\n",
    "            if (current_interval[1][0] <= prev_interval[-1][0]):\n",
    "                temp = copy.deepcopy(prev_interval)\n",
    "                temp[-1][0] = current_interval[0][0]\n",
    "                temp[-1][1] = min(current_interval[1][1], prev_interval[-2][1]) * 0.5\n",
    "                visemes_speak.viseme_ctrl_pts[-1][prev_slider_dict[viseme_list_final[i]]] = temp\n",
    "                current_interval[0][1] = min(current_interval[1][1], prev_interval[-2][1]) * 0.5\n",
    "                visemes_speak.add(viseme_list_final[i], current_interval, pure_phoneme_list[i])\n",
    "            elif (current_interval[0][0] <= prev_interval[-2][0]):\n",
    "                temp = copy.deepcopy(current_interval)\n",
    "                temp[0][0] = prev_interval[-1][0]\n",
    "                temp[0][1] = min(current_interval[1][1], prev_interval[-2][1]) * 0.5\n",
    "                visemes_speak.viseme_ctrl_pts[-1][prev_slider_dict[viseme_list_final[i]]][-1][1] = min(current_interval[1][1], prev_interval[-2][1]) * 0.5\n",
    "                visemes_speak.add(viseme_list_final[i], temp, pure_phoneme_list[i])\n",
    "            else:\n",
    "                temp = copy.deepcopy(prev_interval)\n",
    "                \n",
    "                if pure_phoneme_list[i] in cmu_sets.lip_closer:\n",
    "                    visemes_speak.add(viseme_list_final[i], viseme_intervals_final[i], pure_phoneme_list[i])\n",
    "#                 print(temp[-1][0], current_interval[0][0])\n",
    "                else:\n",
    "                    temp[-1][0] = (current_interval[0][0] + prev_interval[-1][0]) / 2.0\n",
    "                    temp[-1][1] = min(current_interval[1][1], prev_interval[-2][1]) * 0.5\n",
    "                    visemes_speak.viseme_ctrl_pts[-1][prev_slider_dict[viseme_list_final[i]]] = temp\n",
    "                    current_interval[0][0] = (current_interval[0][0] + prev_interval[-1][0]) / 2.0\n",
    "                    current_interval[0][1] = min(current_interval[1][1], prev_interval[-2][1]) * 0.5\n",
    "                    visemes_speak.add(viseme_list_final[i], current_interval, pure_phoneme_list[i])\n",
    "                \n",
    "                    \n",
    "        # if the two interval overlaps\n",
    "        elif (current_interval[1][0] <= prev_interval[-2][0]):\n",
    "            interval = prev_interval[0:-2] + current_interval[1:]\n",
    "            visemes_speak.viseme_ctrl_pts[-1][prev_slider_dict[viseme_list_final[i]]] = copy.deepcopy(interval)\n",
    "        else:\n",
    "            visemes_speak.add(viseme_list_final[i], viseme_intervals_final[i], pure_phoneme_list[i])\n",
    "            \n",
    "    else:        \n",
    "        visemes_speak.add(viseme_list_final[i], viseme_intervals_final[i], pure_phoneme_list[i])\n",
    "        \n",
    "    prev_slider_dict[viseme_list_final[i]] = len(visemes_speak.viseme_ctrl_pts[-1]) - 1\n",
    "    i = i + increment"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21d3dbb1",
   "metadata": {},
   "source": [
    "## 2.3 Extending the onset and offset of vowels so they overlap more for singing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "71f18b23",
   "metadata": {},
   "outputs": [],
   "source": [
    "## extend vowels so if a nasal vowel follows the vowel, the jaw will be open for those \n",
    "\n",
    "viseme_list = copy.deepcopy(visemes_sing.viseme_list[-1])\n",
    "viseme_intervals = copy.deepcopy(visemes_sing.viseme_ctrl_pts[-1])\n",
    "pure_phonemes = copy.deepcopy(visemes_sing.pure_phoneme[-1])\n",
    "# visemes_sing_notM = Viseme_curve(viseme_list.deepcopy(), viseme_intervals.deepcopy())\n",
    "for i in range(1, len(viseme_list)):\n",
    "    # re-order all the clips so that \n",
    "    if viseme_list[i] in jali_sets.vowels:\n",
    "        next_nasal_vowel = -1\n",
    "        for si in range(i+1, len(pure_phonemes)):\n",
    "            if pure_phonemes[si] in set([\"N\", \"NG\"]):\n",
    "                next_nasal_vowel = si\n",
    "                break\n",
    "        if next_nasal_vowel > -1:\n",
    "            interval_i = copy.deepcopy(viseme_intervals[i])\n",
    "            interval_i.sort(key=lambda x:x[0])\n",
    "            interval_next = copy.deepcopy(viseme_intervals[next_nasal_vowel])\n",
    "            interval_next.sort(key=lambda x:x[0])\n",
    "#             if interval_i[0][0] <= interval_prev[-1][0]:\n",
    "            if abs(interval_i[-1][0]-interval_next[0][0]) <= 0.1 or (interval_i[-1][0] >= interval_next[0][0] and interval_i[-1][0] <= interval_next[-1][0]):\n",
    "                interval_i[-1][0] = interval_next[-1][0]\n",
    "#                 interval_i[-2][0] = interval_next[-3][0]\n",
    "                viseme_intervals[i] = interval_i        \n",
    "visemes_sing.viseme_list[-1] = viseme_list\n",
    "visemes_sing.viseme_ctrl_pts[-1] = viseme_intervals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f741ce93",
   "metadata": {},
   "outputs": [],
   "source": [
    "viseme_list = copy.deepcopy(visemes_sing.viseme_list[-1])\n",
    "viseme_intervals = copy.deepcopy(visemes_sing.viseme_ctrl_pts[-1])\n",
    "pure_phonemes = copy.deepcopy(visemes_sing.pure_phoneme[-1])\n",
    "for i in range(1, len(viseme_list)):\n",
    "    # re-order all the clips so that \n",
    "    if viseme_list[i] in jali_sets.voiced:\n",
    "        prev_vowel = -1\n",
    "        for si in range(i-1, -1, -1):\n",
    "            if viseme_list[si] in jali_sets.voiced:\n",
    "                prev_vowel = si\n",
    "                break\n",
    "#         if prev_vowel > -1 and viseme_list[i] != viseme_list[prev_vowel]:\n",
    "        if prev_vowel > -1:\n",
    "            interval_i = copy.deepcopy(viseme_intervals[i])\n",
    "            interval_i.sort(key=lambda x:x[0])\n",
    "            interval_prev = copy.deepcopy(viseme_intervals[prev_vowel])\n",
    "            interval_prev.sort(key=lambda x:x[0])\n",
    "            if abs(interval_i[0][0]-interval_prev[-1][0]) <= 0.3 or (\n",
    "                interval_i[0][0] >= interval_prev[0][0] and interval_i[0][0] <= interval_prev[-1][0]):\n",
    "                # there is an overlap\n",
    "                interval_prev[-1][0] = min(interval_i[1][0] + 0.12, interval_i[-2][0])\n",
    "                interval_i[0][0] = max(interval_prev[-2][0] - 0.12, interval_prev[1][0])\n",
    "                viseme_intervals[i] = interval_i\n",
    "                viseme_intervals[prev_vowel] = interval_prev\n",
    "visemes_sing.viseme_list[-1] = viseme_list\n",
    "visemes_sing.viseme_ctrl_pts[-1] = viseme_intervals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "516101c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pass 3\n",
    "# set this up\n",
    "prev_slider_dict = {}\n",
    "for i in range(0, len(list(jali_sets.vocabs))):\n",
    "    prev_slider_dict[list(jali_sets.vocabs)[i]] = -1\n",
    "viseme_list_final = visemes_sing.viseme_list[-1]\n",
    "viseme_intervals_final = visemes_sing.viseme_ctrl_pts[-1]\n",
    "pure_phoneme_list = visemes_sing.pure_phoneme[-1]\n",
    "visemes_sing.new_pass()\n",
    "i = 0  \n",
    "while i < len(visemes_sing.viseme_list[-2]):\n",
    "    increment = 1\n",
    "    prev_viseme = viseme_list_final[i]\n",
    "    # if the previous instance of the current viseme is not -1\n",
    "    if prev_slider_dict[viseme_list_final[i]] != -1:\n",
    "        current_interval = copy.deepcopy(viseme_intervals_final[i])\n",
    "        current_interval.sort(key=lambda x:x[0])\n",
    "        prev_interval = copy.deepcopy(visemes_sing.viseme_ctrl_pts[-1][prev_slider_dict[viseme_list_final[i]]])\n",
    "        prev_interval.sort(key=lambda x:x[0])\n",
    "        # For the same vowel/consonant if the two intervals do not overlap but the onset and off set overlap \n",
    "        if (current_interval[0][0] - prev_interval[-1][0]) <= vowel_blend_distance:\n",
    "            # if we have weird overlap where the boundary of the previous interval interupts the curve of the current\n",
    "            if prev_interval[2][0] - prev_interval[1][0] <= speech_vowel_threshold:\n",
    "                prev_interval[2][1] = 0.95 * prev_interval[1][1]\n",
    "            if current_interval[1][0] <= prev_interval[-1][0] and current_interval[0][0] <= prev_interval[-2][0]:\n",
    "                temp = copy.deepcopy(prev_interval)\n",
    "                temp[-1][0] = (current_interval[0][0] + prev_interval[-1][0]) / 2.0\n",
    "                temp[-1][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "                visemes_sing.viseme_ctrl_pts[-1][prev_slider_dict[viseme_list_final[i]]] = copy.deepcopy(temp)\n",
    "                current_interval[0][0] = (current_interval[0][0] + prev_interval[-1][0]) / 2.0\n",
    "                current_interval[0][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "                visemes_sing.add(viseme_list_final[i], current_interval, pure_phoneme_list[i])\n",
    "            elif (current_interval[1][0] <= prev_interval[-1][0]):\n",
    "                temp = copy.deepcopy(prev_interval)\n",
    "                temp[-1][0] = current_interval[0][0]\n",
    "                temp[-1][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "                visemes_sing.viseme_ctrl_pts[-1][prev_slider_dict[viseme_list_final[i]]] = temp\n",
    "                current_interval[0][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "                visemes_sing.add(viseme_list_final[i], current_interval, pure_phoneme_list[i])\n",
    "            elif (current_interval[0][0] <= prev_interval[-2][0]):\n",
    "                temp = copy.deepcopy(current_interval)\n",
    "                temp[0][0] = prev_interval[-1][0]\n",
    "                temp[0][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "                visemes_sing.viseme_ctrl_pts[-1][prev_slider_dict[viseme_list_final[i]]][-1][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "                visemes_sing.add(viseme_list_final[i], temp, pure_phoneme_list[i])\n",
    "            else:\n",
    "                if pure_phoneme_list[i] in cmu_sets.lip_closer:\n",
    "                    visemes_sing.add(viseme_list_final[i], viseme_intervals_final[i], pure_phoneme_list[i])\n",
    "                else:\n",
    "                    temp = copy.deepcopy(prev_interval)\n",
    "                    temp[-1][0] = (current_interval[0][0] + prev_interval[-1][0]) / 2.0\n",
    "                    temp[-1][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "                    visemes_sing.viseme_ctrl_pts[-1][prev_slider_dict[viseme_list_final[i]]] = copy.deepcopy(temp)\n",
    "                    current_interval[0][0] = (current_interval[0][0] + prev_interval[-1][0]) / 2.0\n",
    "                    current_interval[0][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "                    visemes_sing.add(viseme_list_final[i], current_interval, pure_phoneme_list[i])\n",
    "        # if the two interval overlaps\n",
    "        elif (current_interval[1][0] <= prev_interval[-2][0]):\n",
    "            interval = prev_interval[0:-2] + current_interval[1:]\n",
    "            visemes_sing.viseme_ctrl_pts[-1][prev_slider_dict[viseme_list_final[i]]] = interval\n",
    "        else:\n",
    "            visemes_sing.add(viseme_list_final[i], viseme_intervals_final[i], pure_phoneme_list[i])\n",
    "    else:        \n",
    "        visemes_sing.add(viseme_list_final[i], viseme_intervals_final[i], pure_phoneme_list[i])\n",
    "        \n",
    "    prev_slider_dict[viseme_list_final[i]] = len(visemes_sing.viseme_ctrl_pts[-1]) - 1\n",
    "    i = i + increment\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b89b900e",
   "metadata": {},
   "source": [
    "### Vibrato"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ed4f5e3d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "lyric.compute_self_vibrato_intervals()\n",
    "\n",
    "vib_ctrl_pts = []\n",
    "for k in lyric.vibrato_intervals:\n",
    "    if len(k) > 0:\n",
    "        for m in k:\n",
    "            vib_ctrl_pts.append(m)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b460b3c",
   "metadata": {},
   "source": [
    "## Vowel Modification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "09e738a3",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2086131,)\n"
     ]
    }
   ],
   "source": [
    "print(lyric.sound_arr.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "7e50be28",
   "metadata": {},
   "outputs": [],
   "source": [
    "vowel2Cardinal5 = {\"Ah_pointer\":0, \"Aa_pointer\":1, \"Eh_pointer\":1, \"Ee_pointer\":2, \n",
    "                 \"Ih_pointer\":2, \"Oo_pointer\":3, \"Oh_pointer\":3, \"Uh_pointer\":0, \n",
    "                  \"U_pointer\":4, \"Eu_pointer\":4}\n",
    "vowel2Cardinal3 = {\"Ah_pointer\":0, \"Aa_pointer\":1, \"Eh_pointer\":1, \"Ee_pointer\":1, \n",
    "                 \"Ih_pointer\":1, \"Oo_pointer\":2, \"Oh_pointer\":2, \"Uh_pointer\":0, \n",
    "                  \"U_pointer\":2, \"Eu_pointer\":2}\n",
    "\n",
    "control_direction_matrix_coarse = {0:{1:[\"Dimple\", \"Dimple\", [0, 9]], 2:[\"Pucker\", \"Pucker\", [0, 3]]},\n",
    "                                  1:{0:[\"self\", \"lipCornerPull\", [0, -3],\n",
    "                                       \"self\", \"lipStretch\", [0, -3]], 2:[\"Pucker\", \"Pucker\", [0, 3]]},\n",
    "                                  2:{0:[\"self\", \"lipPucker\", [0, -7]], 1:[\"self\", \"lipPucker\", [0, -7],\n",
    "                                                                          \"Dimple\", \"Dimple\", [0, 9]]}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "d2617c65",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20.024833700501382\n",
      "26.999665229885057\n",
      "32.57356859756097\n",
      "41.91086702201994\n"
     ]
    }
   ],
   "source": [
    "viseme_list = visemes_sing.viseme_list[-1]\n",
    "viseme_interval = visemes_sing.viseme_ctrl_pts[-1]\n",
    "\n",
    "modification_ctrl_pts = []\n",
    "modification_sliders = []\n",
    "# iterate through the vowels in the list\n",
    "dt = 0.01\n",
    "for i in range(0, len(viseme_list)):\n",
    "    if viseme_list[i] in VOWELS_SLIDERS_JALI and viseme_interval[i][-2][0] - viseme_interval[i][0][0] > 0.30:\n",
    "        ##################################################################\n",
    "        ###################### get the audio signal ######################\n",
    "        ##################################################################\n",
    "        vowel_mod_out, vowel_mod_out_coarse = vowel_mod(lyric.sound_arr_interp(np.arange(viseme_interval[i][1][0], \n",
    "                                                                                        min(viseme_interval[i][-1][0], lyric.snd.xs()[-1]), 1.0/44100.0)))\n",
    "#         print([\"A\", \"Stretcher\", \"Rounder\", \"Silence\"][vowel2Cardinal3[viseme_list[i]]])\n",
    "        if vowel_mod_out_coarse.shape[0] <= 10:\n",
    "            continue\n",
    "        xs = np.linspace(viseme_interval[i][1][0], min(viseme_interval[i][-1][0], lyric.snd.xs()[-1]), vowel_mod_out_coarse.shape[0])\n",
    "        coarse_vowel_sounds_like_interp = interp1d(xs, vowel_mod_out_coarse, axis=0)\n",
    "        \n",
    "#         for i in range(0, 4):\n",
    "#             plt.plot(vowel_mod_out_coarse[:, i], label=[\"Ah\", \"stretcher\", \"rounder\", \"silence\"][i])\n",
    "# #         plt.plot(np.argmax(out[0, :], axis=1))\n",
    "#         plt.legend()\n",
    "#         plt.show()\n",
    "        \n",
    "        # what the original sound was\n",
    "        original_vowel_shape = vowel2Cardinal3[viseme_list[i]]\n",
    "        only_peaks = np.where(vowel_mod_out_coarse > 0.75, vowel_mod_out_coarse, 0)\n",
    "        vowel_sounds_like = np.argmax(only_peaks, axis=1)\n",
    "        ##################################################################\n",
    "        ### obtain the intervals of which cardinal vowels are dominant ###\n",
    "        ##################################################################\n",
    "        cardinal_list = []\n",
    "        cardinal_intervals = []\n",
    "        current_interval_start = 0\n",
    "        current_vowel = original_vowel_shape\n",
    "        for t in range(0, vowel_sounds_like.shape[0]):\n",
    "            if vowel_sounds_like[t] == current_vowel:\n",
    "                if (t == vowel_sounds_like.shape[0]-1) and current_vowel != 3:\n",
    "                    cardinal_list.append(current_vowel)\n",
    "                    cardinal_intervals.append([current_interval_start, t])\n",
    "            else:\n",
    "                if xs[t-1] - xs[current_interval_start] >= 0.1:\n",
    "                    if t > 0 and current_vowel != 3:\n",
    "                        cardinal_list.append(current_vowel)\n",
    "                        cardinal_intervals.append([current_interval_start, t-1])\n",
    "                    current_interval_start = t\n",
    "                    current_vowel = vowel_sounds_like[t]\n",
    "        ###########################################################################\n",
    "        ######### optionally additional smoothing are added to this here ##########      \n",
    "        ###########################################################################\n",
    "        cardinal_list_new = []\n",
    "        cardinal_intervals_new = []\n",
    "        j = 0\n",
    "        while j < len(cardinal_list):\n",
    "            step = 1\n",
    "            if j == len(cardinal_list) - 1:\n",
    "                # if the cardinal vowel is the final one in the list, then it is added to the very end\n",
    "                cardinal_list_new.append(cardinal_list[j])\n",
    "                cardinal_intervals_new.append(cardinal_intervals[j])\n",
    "            elif cardinal_list[j] == cardinal_list[j+1] and xs[cardinal_intervals[j+1][0]] - xs[cardinal_intervals[j][1]] <= spike_width:\n",
    "                # if the cardinal\n",
    "                cardinal_list_new.append(cardinal_list[j])\n",
    "                cardinal_intervals_new.append([cardinal_intervals[j][0], cardinal_intervals[j+1][1]])\n",
    "                step = 2\n",
    "            elif j < len(cardinal_list) - 2:\n",
    "                if (cardinal_list[j] == cardinal_list[j+2] and xs[cardinal_intervals[j+2][0]] - xs[cardinal_intervals[j][1]] <= spike_width \n",
    "                    and cardinal_list[j+1] == original_vowel_shape):\n",
    "                    cardinal_list_new.append(cardinal_list[j])\n",
    "                    cardinal_intervals_new.append([cardinal_intervals[j][0], cardinal_intervals[j+2][1]])\n",
    "                    step = 3\n",
    "            else:\n",
    "                cardinal_list_new.append(cardinal_list[j])\n",
    "                cardinal_intervals_new.append(cardinal_intervals[j])\n",
    "            j = j + step\n",
    "        cardinal_list = cardinal_list_new\n",
    "        cardinal_intervals = cardinal_intervals_new\n",
    "        # now set pucker/stretch values based on the detected sound\n",
    "        for c in range(0, len(cardinal_list)):\n",
    "            if original_vowel_shape == cardinal_list[c] or cardinal_list[c] == 3:\n",
    "                continue\n",
    "            else:\n",
    "                max_prob = coarse_vowel_sounds_like_interp(xs[cardinal_intervals[c][0]:cardinal_intervals[c][1]+1])[:, cardinal_list[c]].max()\n",
    "                \n",
    "                slider_ct_pts = control_direction_matrix_coarse[original_vowel_shape][cardinal_list[c]]\n",
    "                for s in range(0, int(len(slider_ct_pts)/3)):\n",
    "                    ctrl_pts = []\n",
    "                    # get the name and attribute of the slider\n",
    "                    slider_name = slider_ct_pts[0 + 3*s]\n",
    "                    if slider_name == \"self\":\n",
    "                        slider_name = viseme_list[i]\n",
    "                    slider_attribute = slider_ct_pts[1 + 3*s]\n",
    "                    # add a starting keyframe and ending keyframe\n",
    "                    modification_sliders.append([slider_name, slider_attribute])\n",
    "                    # the start of this curve should be earlier, e.g. at 75% of the previous interval\n",
    "                    # however, if the detected modification is at the very beginning of the vowel, then it is\n",
    "                    # applied the same time as the beginning of the vowel\n",
    "                    slider_range = slider_ct_pts[2 + 3*s]\n",
    "                    same_start = False\n",
    "                    start = -100\n",
    "                    \n",
    "                    if c == 0 : \n",
    "                        # if the modification stat\n",
    "                        start = viseme_interval[i][0][0]-0.12\n",
    "                        same_start = True\n",
    "                    else:\n",
    "                        # otherwise I choose the starting point \n",
    "                        # find the previous c that is not the current vowel\n",
    "                        ci = -1\n",
    "                        for ccci in range(c-1, -1, -1):\n",
    "                            if original_vowel_shape != cardinal_list[ccci] and cardinal_list[ccci] != 3:\n",
    "                                ci = ccci\n",
    "                                break\n",
    "                        if ci > -1:\n",
    "#                             start_candidate = (xs[cardinal_intervals[ci][1]] - xs[cardinal_intervals[ci][0]]) * 0.6 + xs[cardinal_intervals[ci][0]]\n",
    "#                             start_candidate = min(start_candidate, xs[cardinal_intervals[c][0]]-0.12)\n",
    "                            start_candidate = xs[cardinal_intervals[ci][0]]\n",
    "                            start = max(start_candidate, viseme_interval[i][0][0])\n",
    "                            if np.abs(start - viseme_interval[i][0][0]) <= 0.2:\n",
    "                                start = viseme_interval[i][0][0]-0.12\n",
    "                                same_start = True\n",
    "                        else:\n",
    "                            start = viseme_interval[i][0][0]-0.12\n",
    "#                             start_candidate = xs[cardinal_intervals[ci][0]]-0.12\n",
    "#                             start = max(viseme_interval[i][0][0], start_candidate)\n",
    "#                             if viseme_interval[i][0][0] >= start_candidate:\n",
    "#                                 same_start = True\n",
    "#                             if np.abs(start - viseme_interval[i][0][0]) <= 0.2:\n",
    "#                                 start = viseme_interval[i][0][0]\n",
    "#                                 same_start = True\n",
    "\n",
    "                            \n",
    "                    ctrl_pts.append([start, slider_range[0]])\n",
    "                    # add the end point of the curve. This would either at the end of the \n",
    "                    # prediction interval, or at the end of the vowel. \n",
    "#                     modification_sliders.append([slider_name, slider_attribute])\n",
    "                    ci = -1\n",
    "                    for ccci in range(c+1, len(cardinal_intervals)):\n",
    "                        if original_vowel_shape != cardinal_list[ccci] and cardinal_list[ccci] != 3:\n",
    "                            ci = ccci\n",
    "                            break\n",
    "                    if ci == -1:\n",
    "                        ctrl_pts.append([viseme_interval[i][-1][0], slider_range[0]])\n",
    "                    else:\n",
    "                        ctrl_pts.append([xs[cardinal_intervals[c][1]] + 0.12, slider_range[0]])\n",
    "        \n",
    "                    # add the peaks in the middle with the decay\n",
    "                    \n",
    "#                     modification_sliders.append([slider_name, slider_attribute])\n",
    "                    # with some anticipation \n",
    "                    if same_start:\n",
    "                        ctrl_pts.append([viseme_interval[i][1][0], max_prob * (slider_range[1])])\n",
    "                    else:\n",
    "                        if slider_name == \"Dimple\":\n",
    "                            print(xs[cardinal_intervals[c][0]])\n",
    "                        ctrl_pts.append([xs[cardinal_intervals[c][0]], max_prob * (slider_range[1])])\n",
    "#                     modification_sliders.append([slider_name, slider_attribute])\n",
    "                    end_p75 = (xs[cardinal_intervals[c][1]] - xs[cardinal_intervals[c][0]]) * 0.75 + xs[cardinal_intervals[c][0]]\n",
    "                    ctrl_pts.append([end_p75, max_prob * (slider_range[1]) * 0.75])\n",
    "                    modification_ctrl_pts.append(ctrl_pts)\n",
    "modification_ctrl_pts_sing = copy.deepcopy(modification_ctrl_pts)\n",
    "modification_sliders_sing = copy.deepcopy(modification_sliders)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "8a1d2703",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1eccdbb5fa0>]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAD4CAYAAADxeG0DAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA1Q0lEQVR4nO2deZAkd3Xnv6/OPqrmUlePRtKMRseMQMggjUeYcwy2wEJLSMYHC157MfaubGJx2GE7MFg+MLvewLd3Y9cm5F0ZliWw2QWtCRbbIBsj1hbg0ehgBGJ0MDpG0kxrrp7pqyoz3/6R+cv8ZVaelTWdlZ3vE9HRXVWZVb/O/OWrl9/3fu8RM0MQBEEoL7WiByAIgiDkQwy5IAhCyRFDLgiCUHLEkAuCIJQcMeSCIAglp1HEh87NzfHu3buL+GhBEITScv/997/IzL3g84UY8t27d+PgwYNFfLQgCEJpIaKnwp4XaUUQBKHkiCEXBEEoOWLIBUEQSo4YckEQhJIjhlwQBKHkjMWQE9HNRPRtInqciN4/jvcUBEEQ0pHbkBNRHcB/BfAWANcCeCcRXZv3fQVBEIR0jCOP/JUAHmfmJwGAiP4CwG0AvjmG9/bxd986joeeOTPutw3le6/p4bsv37Yun6Xz2Yeew+PHz6FWI/zo/p24dMv0SO9z6OnT+IdHT4x5dOVg62wLP/ma3SCiXO/z1MklfObQMTAz4LyXekf11gTS/vZeC342kb2tf9/o1+y/hz+Tgq/p70WEOhG2zjSxbbaFizotbJttY8t0E7VavmNxIbjviZO474kXY7fZPNPCu1+ze2j8JxZX8eSLS3jVlReF7vflIwu4/+ipsY11nLxt32W4Ym52rO85DkN+KYBntMfPAvie4EZEdDuA2wFg165dI33Ql48s4ONfDc2HHyvMwOe+8Tz+7he/N5UxMEwLnzr4LN6+/zI06vlucn75Uw+hb1oAgL5h4X03v2Sk93n/px/GkePnkdOWlQ5VXv/G3dtw3aWbQ7e555vH8d5PHsLBX3sTOu3oS+DP//EoPvpPRy/AKNeXGgFbZ1rYNmv/zHXa7t+2sXf+nrWf3zrTzD2P0/Arn34YT59ajpyj6lxeu2MTXn2V32D/8Ef+Cc+cWsHRD/+L0H3/w+e+icdOTOb833f51ok05Klg5jsB3AkA+/fvH6mbxYduuw4fuu26sY4rjI/fdxS//leP4MkXl3BVrzP0et+w8OG/fhQ/931XY+tsC4eePoNfvfsbuHTrNL5379Dq2dRYFqNvWvj579+DTx18BscX10Z6n0dfWMSR4+fxW7e+DO96ze6Rx1NGvvPiEt74+/+Abz2/GGnIP/w3j2J1YOGpk0t42SXh2wDAwrk1XNmbxd//0hvc51QjFmaAg8+5jwGGt51O2Gv6++jvoV7Msr1hWTi9NMCppT5OLq3h1FLf+buPk+ftx996YRGnlvo4szwI/b+JgC3TTZ9x39Zp4SLX+Le9v2db2DrbQjOj4X/65DKePrUcO0fPrQ5w/Ye+iH964sUhQ/7MqRUAwOrAxFSzPrTvwLRw6ysuwX9+5w2ZxlVWxmHIjwHYqT2+zHmutLzhmnkAj+DeIwuhhvwL33wBd/3jd3BmpY8/fPv1GDge9InF1VyfO7Ds92k1atg228Lp5X7m97j7gWdxx92HMd2s4y3XXZxrPGVk17YZTDVrePSFc5HbrA5MAMC5VSP2vc6uDLBluul7LihpOM+ONNYLxY7N6eQ4w7Rwetlv9E+et43+Ke3xEwvn8c9H+zi93IcV4YJtmmrgIsfTv8jn6XtGX90BbJluuXc6r716LnJ83akmXrqji/ufOj30WqtRQ9+wsLg6CDXkFtt3IlVhHIb8nwHsIaIrYBvwdwD4sTG8b2Hs3DaDK+Zmce+RBbz7tVcMvd5u2BPnrOPRmM7sPnFuNA9aMTDt92nWCZ12A+fX4g1NGH9z+AXMthv45L/9Hsxvmso1njJSrxH2bu/i2zGGfMk5rkmGvG9amT3NMtGo19DrttHrtgF0E7c3LcbZlQFOnl9zjL1j9M/bhl8999TJZRx6+gxOL/fdayOMW19xCa7qxUsMN+zcis8cehamxahrlrnbbuCk0ce5VQPzIUM3LUZtEnWVC0RuQ87MBhG9F8DfAqgDuIuZH8k9soI5sGcOnzr4LNYM0zXcCqWrnnMMguXc1y7kNeSG7ZE36zV0pxp4/mx2D9+0GPPdNq4Om90VYc98F195bCHy9dPOF/ByP96QG6aFmVYhdeUmknqNXM96T4rtLYuxuDrwjP75viP1rGHv9i7edO32xBjUvsu34ONffQpHjp/DS3dscp/vTDVwcqmPxZVweYiZJzLAe6EYyyxl5s8D+Pw43mtSOLC3h4/d9xQOHj09dPs31bS9tPOrYzbklmfIZzN65H/14DFcNNuGYTEaFZrAYezd3sGnDz2Ls8sDbJ7xSyN6s/GVvhn7PkEvUMhGrUbYMtPClpkWrhoxdLRv11YAdhaWz5ArZ8q5BhfO2VLQNRfbDozJXClpZePeN+bkVVdehGadcO+RYc9OmYIlx6Nz7G9+Q+5IK616zZZWEm79df7kS0/gf9x3VIwPgD3b7bjGYyeG5ZXFFe+YLicY8oHJaNarfSyLZte2GWybbeHQU2d8zytDvrhqe+Rv+U9fwQ/88b3u6xajUteBGPIIZtsNfPflW/HlMEPuWHKltZrKIz8/JmmlkV0jNywLFjMMk9GoVfu07nFkpSPHzw+9duzMivv3yiDekBuWVfljWTREhH27tuCBZ/wBz+6UY8idL+YXnWtP3XFZFudeR1AmZJbGcGBvD4++cG4oG0VNFnVbpx7nzlpxsl8aNdsjXzMs9B3jnoTF9o945MClW6Yx06qHeuTPn/UMebJGzmiIR144N+zaiicXlnBGy+LqTtmS2blVv0Z+Xotb1cWQCwBwYI8t7N37mH/1mQrErzlG1rG/WOqbrpc+CmohULNeQ8fxONK+n2kxTIttL7LixqdWI1w938FjIR75c2d0Q57kkUu8YRK4YdcWAMADT59xn1NxqsWAIfeuSdHIBYdrd2zCXKc1pJNzYJWHqT3Oo5O7GnmDMOtogGnlFdNiWMzikTvsme+GeuTPnV1Fs06Y67QSg52Gaa3LCkchnldctgU1Ah542pNX1CUXTCFV6Y7MqFTWiszSGGo1wuv39PD/Hn8RlpYPG0yN1Q17Hp18oHnk3YyG3GLbkIsXabNnewfHF9dwNpCe9tyZFVy8eQqz7UZysNOSYOckMNtu4CUXb8IhzSNXmWLB9ENlyC2uVh65GPIEDuydw6mlPg4/d9Z9bsgjt8blkQ9LK2kNueFIK+KR2+xVmSvH/V7582dWsWPzNKab9WRpxZRg56Sw7/ItePCZMz6PG4j2yE2u1nUgszSB1zs6+Vc0nVw348zs89DzBDy9lZ21zNKKZdnjsD1yOa0qc+WxE36d/NiZFVy6ZRqz7QZWBsnBzioZg0nmhp1bcX7NwOPO+VTXXFAjN1yPPFhGYWMjV3wCc502XnbJJl8aoqV55Et90ye75JJWnEBNS5dWUuaSm8ywHI+8StpgFJdusb3uI5pHblqM44ur2LF5CjOtOpbWkoOdIq1MBvsu9xYGAV4hMX1dAKBJK5ZkrQgBDuzt4dBTp91UJ11ZWVwZuIa9RuORVhp1yiytqGCnVbEVbVHUaoQ92zuuBwfY58awGJc4Rj4x2GlJsHNS2H3RDLbONHHIKaDlSSuikQNiyFNxYE8PhsW474mTAPwe+blVw81a6XXbuQpn6emHSlpJm35oWQyT7Qlenekbz9XzHZ9H/pyTQ37JFtsjX46RVpjZXtkp34oTARHhhl1bXY/cDXYG7lgNy3LlzirdmYohT8F3X74Vs6067nUKMeke+bnVgavXbd80NZ70w3oNsy1/LYkklLTCqJYnEsfe7V1f5orKIb9kyzSmW41Yj1x5duKRTw77dm3BEwtLOLs8cK/B82uGL9nAsrzrs0J2XAx5GlqNGl591UW494gd8GT4PXKlkW/fNJXLIzdMb4l+vUaYbdUzBDudFEQL4pI77Jm3M1ced/LJnz9jB6J3bLZXfsZlraigmQQ7JwdVQOuBZ0777or1OJJhWe4dcpUcGjHkKTmwt4enTy3j6ItLbpEswI6aK4/g4k1TOHl+LbYGcxx6+iFg58+mXtnpLAYCvP6PVWfvdn/NlWNnVtBpN7BpqoGZVh0rA3MolVShDLkEOyeHl++0FwYdevqML3NMz1xRcSKgWl/CYshT4i3XX/BNoqU10504F2+egsXAqaXsnX0AoK+lHwJ2zeVzGYKdzLa2WyFHJJZg5srzZ1ewY/MUiAjTrTqYgdVBeC0bQ6t7I0wGnXYDe7d38cDTp31fwLohN0x2Ha0qXQcyS1Oye24Wu7bN4N4jC77butWBZ8i3Ox15TpwbLZdceeQtx5B3U5ayVdKO6fRwrND8jUXVXFGZK8+dWcUlW+w2aDNOe7Cowll6tyZhcth3+VbfwiDArq+ijLbK3gIg6YdCOAf2zuG+J066hXkAYNUw3WDnxY4hHzXgqfLIVdGrtNKK0gQtxyuvkjaYxB4tc+X44qp7jlTnnyid3HDcurp45BPFvl1bcW7V8C30six2nReTWTRyIZ4De3pY6pu4/+gp97nVgeVp5JvbAEbv3emVsbUnYNqa5HrurCXSio89WuaKYTFaDXvKT7dsjzyqJrlhqqwVOZiThKqE+OTCkvucqdUeNywGO36WpB8Kobz6qovQqJFvleeaFjDbntMj75uMVr3mTsrOVCNV+qG6lXSllerM30RUzZXHT5zzLZaaaSlpJcojF2llErlybhZbAu379LZuliatVMiOiyHPQneqiX27tuLoyWX3uTXDcuuRz7Qa6E41Rjbkhmn5DEen3XDbycXhLUtWObQVmsEJ6N2C9K4xrrQScccjwc7JhIhww84tvucsy8vUMixPWpGsFSGSA3vnhp4zNQ+g122PrpGbFpoN75Sovp1RKXIKFaW3PRFZoq9z2dZpTDVreOz4eV/8IMkjl2Dn5HKDk0+uMLXlzHqwU1q9pYSIfpSIHiEii4j2j2tQk8yBvf524MzspvwREeZzGPK+yW7qIWBLK4bFvuBqGOqLxHQqIFZo/ibidguKklYiNHLTXRAkvs6ksS9gyH3BTstLP5SslfQcBvBDAO5N2nCjcN0lm7FttuU+Zjh9Mp1J0+tO5Uo/bNX9HjmQXDjLC3Y6eeQirfjYO9/FY8fP++pvuMHOqPRDy59BJEwOr9i5GUSeBq4H+E3RyLPDzN9i5m+PazBloFYjvO5qT15hp+mxumXP45EPTH+/zU7KUrZq4loS7Azl6u0dvLC4ipWB6R6bxPRDJa2IRz5xdKea2DvfdevgmBb7NXLHsZGslQsAEd1ORAeJ6ODCwkLyDhPMLd+1A61GzecRqOu9122P3IR5YFp+aSWzRy555GFc6iwCAtJr5IYpHvkk82tvfSl+4aY9AOCTzNRaCqBa10GiISeie4jocMjPbVk+iJnvZOb9zLy/1+sl7zDB3HzdxXjg19+ELTMtMOyqg2rS9Dp2LvkoXvkgqJFnNOT6baXgoQe91AXfbtRAhMgKiCr9UPqfTiav39PDTS/dDgAwLfjyyL2slcKGt+40kjZg5pvWYyBlY7bdAMGWVkz2NPL5TY4hP7+G3XOzmd7T1sg1aWUqo7TifKFUyBHJjPrCJSLMxPTtNFyNvELWoGSo71iT/Ss7PY28OheCzNIcENnBTtYyRXpdZ3Xn4igeeV5pBU6tlepM4Kzo3vl0K7pvp0o/FI98clGG2rK09EPTcmsPiSFPCRG9jYieBfBqAP+XiP52PMMqC2R75Frn+vmuWt2ZPXNlYOSUVrRUSMGDIv6Oq0luBCpRCpOHuuZMLf3QsLxm6FUy5InSShzMfDeAu8c0ltJhzxO7xZqaVFumm2jUaKQmzH3TwqaWt/w4bd9OpQnaLa6oUmlXWdEv7lhDLumHE486lyZ7DcctrS5/la4DcTdyoDRy2wu2Z02tRpjrtEeXVrTZN92so0bJGrke7GRwpVa0ZUW/uGda0Q2YDZFWJh7lPFlDHnn10g9zeeRVh0jlkbNvFdn8pvZIHnlQIycizKaogOgt0Zfmy0noF/dMK7qWjQQ7Jx91ydlrJzyjLsFOIRMEstMP2e/p9Ub0yA2TfbVWAKe5REppBXAyWKozf1OhX8/639MxHrlba6VCXl3ZUEF9fcrrGnmVvoMr9K9eOCyLfZ5erzuaR94PVD8EbJ08rbQC2BO5Sp5IVlJr5KZ45JOO55F77dClaJaQGV1a0Q3EfLc9UhPmYK0VwOkSlFDKVl8EJA55PEGNPKkeeZVKoZYNdWbUNQioolnS6k3IAMEpmsX+C77XbcNi4ORSNq88uLITsFMQk5pLBL8wKjR/U6Hn1etfuNPNRmTRLGksUQK0U2NpAf8qph+KIc8BkZ1Hbjcs8J7vdUfrFDQwrCFD3p1KE+wMGHLxySOhoLSidXjSkcYS5YEB13j7i2YVN6b1pkL/6oXBDnb6s1bc1Z0ZDXmYRj7bSm7AbAYMkagB0dQCwU5mhNZ7l8YSk4/rsGjL8k2L3S9m8ciFVJCjregrOwFbIweye+SGFSKtZAx2egMTwggGO4HwCojqnFYpYFY29PRDfS2FtHoTMqFqrdhdeYY98iyG3HRuCYeklXYD5/vx7d6CFQ+rM33TodviYLATAJZDdPKBZVXKEJQRPdipLgG7aJb9d5VOnxjyHBDIWRbPvpzVqWY9cxPmgaPJNhsBaaXdAHN03WwAbvNnd1wVmsBZCRbNAsJL2RomSw75hKPOJbPnhZsmS9EsIRtqngTTD4HsnYKUIQ+mH6aptxKUVqo0gbPik1aa0dKKYVqSQz7huB45vLtS3xL9Cl0HMlNzovS54KTpZTbk4dX20lRAFGklHl/1Q11aaduGPCxPf2CxBDonHFcj16QVvWhWlaQxMeQ58IpmDetxWZswu9JKlCGPCXhKHnl6/Bp5tLRimlwpQ1BGVNaK7sjoS/SrdB2IIc8BEbkeefCizyqt9I3wsqkjeeRVmsEZCeaRA+HSysCyJId80nFOpaE5MlI0S8iM7ZHbgZYwaSVLE2Y1GcehkQt+/Fkr/jLBQEywU6SViUadSn+tIcs15FW6oxJDngdSrd7Cg51A+hTEcUorVfJEspI2/dCwJNhZFvSVzf4l+gUNqABkpubAaRAUKq24ueQpqyAqaWWo+uFI0kqqj6wk/gVB9rFdHoR75NJUYrJRZ0df2awXzaqSQyOGPAe2Rs7OgiD/a1mbMHt55KNIK4FxpfrEaqKfp6lmDUQR0orF0uZtwtGbSSgk/VDIjJom9oKgoLSSrQmzSj8MauTtRh3NOsUbcvHIEwivfkhEmG6Gl7IdmBLsnHT0ZhIKtUIaEI1cyEBYPXLAa8KctnBWlEYO2PJKrEYecMmr5IlkJXhsomqSS7Bz8nGDnQFphSX9MBtE9HtE9CgRPUxEdxPRljGNqxSoxhKmNWwgVBPmtMHOvhndtb0zFV8B0ZSkldQEnTS73dvwsTUtFo98wlF55KYZ8MglayUzXwRwHTO/HMARAB/IP6Ty4PbstDi0P2CWJsxGhLQC2KVsz8UFO4cWBFVnAqfB37Mz4JE3G9F55OKRTzShHjmLRp4ZZv4CMysL81UAl+UfUnmIavWmyNKEOU5a6SaUsh3SyFN9YjUJOmkz7TpWJGul1AylH0rWSi5+CsBfR71IRLcT0UEiOriwsDDGjy0Wu9Wbv/myIotH7hnyEGkloW/ncB55qo+sJGEaeZhsNZCiWROPOpV6sNMwJY88FCK6h4gOh/zcpm1zBwADwCei3oeZ72Tm/cy8v9frjWf0BaNavdm1VsI98rRNmL088hBpJSHYKdJKeoKy93SEtGJI0azSoK+jqGrRrEbSBsx8U9zrRPSTAN4K4Ps5rvvBBsSphmwvCAqZM3oTZpWOGIWbftgIl1biNHJJP4zHX/1w2CMPk1bsRV7ikU8ybrAzIo+8Sg5N3qyVmwG8D8CtzLw8niGVh7h65EC2JsxxGnlS387h5stCFMFjE5V+ODAtaSwx4Xi1VrznTM2QV8kjz+ty/BcAXQBfJKIHiegjYxhTqWC2DWmYRp6lCfMgIf1wuW9GSjTDHnl1JnBWgl+4dvphRLBTpJWJxl2ib9nXTr1Gla21kiitxMHMV49rIGVE79kZNmmyFM6KWtkJ+OutbJ5uDr0urd7i0b/YwhcE2T1R9e2kaNbko86XSiNvuIZcslaEDKienWbIEn0gWxPmpJWdACLlleEOQdWZwFkZSj9sNWAxsGb4vw0NS3p2TjpuiQytBLSkHwqZcT3ykFZvQLYmzAPTQo3Cdb2kwlnSISg9Qdkpqia5YUqwc9Lx0g+9gnNGRaUVmak5UK3eooKdQPpOQX3TCvXGAc8jPxeRgjhkyBM/rboMe+ROTfJA5srAtCT9cMJxpRXnZqpZJ5iWVcn0QzHkeYhp9aboddupencODA7Vx4Hs0kqVbinToB+NYFB6uqU8cv+xlTK25UEFO1uNmlM0S9IPhQyoVm8cUo9cMd+dSi2tBGuRK7JKK+KSRxOmkQP+vp3sLCqRolmTDxFcKaVZq7lFs6rkjQNiyHOhF+2pR1hy2yNPNuSGZUXW9khq9zYc7BSiCHpps45HvrTmGXK15FtqrZQDNf+b9ZpTNKta+jgghnwsWBG1VgDbkC+naMLcNzhRI08f7KzYLE4gqvkyoEkrA+/YqkqUkn44+RC0yqENL2ulavKizNQcuMHOkHrkirS55APTCl2eD9i1VoA4Q+5/XDVvJAtppJWBFV3ATJgsiMhdENesk7tEXwy5kBrVs9PW5MK3Sbu6My5LolmvYapZizTkFnOg5nby2KtK2IIgwG/IXY9cvhEnHoJ3R9qs18BsS2OikQupSZN+mHZR0CAm/RBw2r3FSCv6vrIgKJrgafKyVnSNXJVLkMtj0iHyG3LAvpaq5szITM2BHTHnyDK2QPomzH0zWiMH4vt2muxfhVi1SZxEnEYuHnm5IZBmyO3zNTDEIxcyQCA4zlukIU/bhNkwrcg8csBOQYyUViz2pS5KsDOa4HmaagznkUuws0RoHrmKMdmrpKt1DchMzQNBa/QavkmtRuilWN1pd6SJnnyzrSzSihBF8Pqu1QjTzboEO0sKwZ9+CNirpMWQC6kheDnHcV5wmlzyJGklrm+naflXhVZsDieixwzC7rhnWnXfEn3TzSOXy2PSIfKuwVZd98iLHNX6IzM1B0Re5bU4Ta7XSeGRGzmCnexvSybBzmjCvnCDNclVJcqq6axlxJY3g8FO0ciFjLgFemLc4DRNmO088hhppR3dJci02KfnVmwOZyLsllvVJFcojVyklXLg5pE7149o5EImCKT1B4zeLk0T5sT0w5i+nRYHNPJqzeFkfFkrwy/PtPwNmCX9sDyEpR/2DUk/FDKg63Ox0sqmKbcJcxSDJI283UDfsNAPNEAAVLBT//yKzeIMRHvkurTiGAa5tZl4CP7GEoDtFIm0IqRG9wbibuV6HWd152KcIY/3yGdjStlaFsQjT0nYsQkacjfYKR75xENErjOla+QirQip0RcjRBXNArTVnTE6eVIjg7jCWcPBTiGKsAt8utXw5ZFLsLM8hKYfGpK1ImTA75FHb5emcFaitDIV3SUomEdeNW8kCV9jiTBpJZBHLsHOEqHJmxLsHBEi+vdE9DARPUhEXyCiS8Y1sLKQJmslTb2VuFZvgCat9EOklUAh/YrN4UyEfeEG0w/dYKfkkU88etGslrYgqGp3U3ln6u8x88uZ+XoAnwPwG/mHVB70EppxHsBUs45NMU2YmdlOP0wjrUR45PoXiRjyaMLyyNWCINUizLDEIy8LRGF55FblylTkMuTMvKg9nIXdVL5SWCk0ciC+d6fdZxDppJUwjdzyN7aQBUF+9Is6amWnaTH6jjaupJWqeXVlRM8c8y8IKnJU608j7xsQ0W8D+NcAzgJ4Y8x2twO4HQB27dqV92MnAn2JftLEiau34ml8I2atsHjkaYkKdgJ2Kdt2o+4GO+O+WIXJwB/sVNUPRSMfgojuIaLDIT+3AQAz38HMOwF8AsB7o96Hme9k5v3MvL/X643vPyiQtOmHQHwT5n4Kw5Eorfg08mpN4ixE5ZEDXilbt2enSCulIFj9sIpFsxI9cma+KeV7fQLA5wH8Zq4RlQg90JI0ceIKZw2M5Gp7s61oacViBKQVIZIIaQUIMeQS7Jx4iAhqwXRTimaNBhHt0R7eBuDRfMMpF0SUyZBHNWF2VxLGeOS1GmG2VQ/d3w526uNKM/rq4E8/HH59RpNWALs2PCDBzjKgnyF1/VhcvfhGXo38w0R0DQALwFMAfjb/kMoDIbkeuWJe6915Rdt/2NNqsp2IUrbBYGfVbiuzECetqNROCXaWB/10+hbFVewayGXImfmHxzWQMqJr5EkTR88lv2Ju1vdaP6UHGFXKdijYmTz0yhIe7PT37fQaS4i0Mvl451OvyR+3rmMjIjM1F95kSZo4Xu/OYZ1ceeRxrd6AaEMeDHaKJfdDCbLTkEYuPTtLg88j17K+qhbeqNi/e+FIug3vudLKcC65kUIjB6L7dloseeRpCV+ib9+YqprkaSpaCpNBmEYOVE9eFEOegyRPT0c1YQ7zyF1pJSaPHHA88hQrO8X+RBO1RB8AVgZesLNRo8rprGVEP0UtMeTCKOhTJcl7U02Yw1IQ3fTDhPeYTSmtiAGKJm0eueSQlwP97lOPMVXNmRFDngPdJqTxAKJWd7rphwkeeTcy2AkpmhWDfrGHHZvppt+QD0wLzaqJrCXFn7WiBTsrZslltubA3509eeLMRxryDOmHa4Zb3ElhWP5qb9WawtkIu1up1QhTzZpbk9wUj7yU6I5Q1e5KxZDnwO+RJ28fJa2kTT+cbTdgWoy1QLs3y/J/kVRsDo8FvW+nXXRJLo0yoKY6kT/LSNIPhdTocyXNrVyv08appeEmzGnTD7vt8OYSJvurvVXNG0kizeGYbtZ9KztlVWc5UHO9TuS7Bqv2PVyxf3e8+KSVNIZcNWEOtHzLkn4IDLd7G6pHnjgSIYjet1OCneWjRhTI3KrW+RNDPiZSBTs73jJ9nbTph6pwll5vJaweunjk2ZlpN7A88Ay5BDvLgZrqREC9LoZcGAVdWkkT7NwU3oTZDXYmePWdkL6dbq0X8chzMdOsY3lN1VqxxCMvCWra12vk18gla0VIiz5V0jgAyiNfWAwYciNd1kq33QTgl1bMEI+8at5IEmmOhi6tSLCzPCh5s0ZU6YC/zNYc6BJGqmBnN8ojT5dHPtt2qvTp0goPLyev2iQeB9Oturey05JgZ1lQc70mWSvCqPjrXCdPnKgmzKmrH4b07VQeedUm7rixPXKvjK0UzCoH6izVaoGslYpdD2LIc+BPP0y3T1gTZjdrJeF23pVWVvVgp/27Jh55NCmOh55HblgWGlLCthSou+Ia2bVx1GWQJotsIyGzNQd+jTzdxAlbpj9wijQlTb6pZg018ksrXrDT265q3sg4mG7peeTikZcNNedVe76qnT4x5DnwaeQpjed8d2oo/XCQMkuCiIZqkpshJVfFjmdnplmHYTH6hoWBxeKRlwRXWnE9cfu3ZK0IqclS/VAR5pH3TSt1N5ruVNOXfqiCnVKPPJo0x0PvEmSYVmIqqDAhaOmHgO6RV+v8iSEfE2nnzXxIE+aBaSUuz1fMtv0NmMOCnRWbw2NBNWBeHhhSNKtEeB650sqd5yt2+sSQ5yFjrRVA7xTkeeUDg1N75FHSit8jF7Ki1yS3pS65NMqAG+x0Tpc6b1XL4pLZmoOsZWwBfxNmxcC00Gyk278z1fSlH1phKzsrNonHwYwurVgS7CwLQY9cOVSStTICRPRLRMRENDeO9ysLWRtLAF4TZj0FcWBl8cgjpBUJdkaS5njMaHVs7KwV8XHKgLcgyKuCqD+uCrlnKxHtBPBmAE/nH0658C8ISrdPqEdupNfIg307w4OdQlZUsHN5YMrKzhLhLdG3H7seecVO3zjcjj8C8D4AnLThRiNrPXLAbsLcrNOQtJI2uDbbbgQ8cufzRVrJhU9aMSXYWRaCHrk6b5J+mAEiug3AMWZ+KMW2txPRQSI6uLCwkOdjJ4as9cjVdnMdf6egTOmH7QbO9w23fK0nrWifUa05nEjaolmAFuwUaaVUKMOtHJqqOTONpA2I6B4AF4e8dAeAX4UtqyTCzHcCuBMA9u/fvyG891E0cmA4l3yQwZB3phpgtiWATrvhSSu+MrbVmsTjwMsjNyTYWULINeD246plrSQacma+Kex5IvouAFcAeMg5iJcBOEREr2TmF8Y6yhKQZeLMd9s4dkYLdprsdnJPYrbtBeU6Tg9PQIKdeXHzyF1pRTzyMkBD+eOikWeCmb/BzPPMvJuZdwN4FsC+Khlx3WBmMZ5BjzxLj8hOoG+nEWLIheyoL9LlvgQ7y4Q6S2r+69UQq4S4HbnQaq1kmDi97pSvCXPfTJ9+2A307QyrR161SZxEGr20XiO0GzUsrRmwGKKRlwQKeOLB4GdVGNtsdTzzF8f1fmUgj0auN2G2FwSlXKIf6NsZukQ/9UgEnZlW3b3TkayVcuBp4s5jqKyVggZUEBX7d8fLKHnkwHAT5kGGIk3Bvp1hzZfT6u2Cn5lWA4urAwCQYGdJ0Fu9AeKRCyOgt5nKku7kNmFWhtzIkn7o79tphkgrM20x5DppT810q+4Z8qq5dCXFvQaVRi4rO4WsBL2BtLhNmB1D3jc5vbQS6NvpFs3SxtBuiCEfhdlWHYsr9nGVYGc58GqthD+uCmLIcxD0BtISbMKcpYxtJybYedNLt2cah+BH98glC6gkULi0UrXzl5hHLiSTdc6oJswnFu1c8izph+1GHa16zdXI9SX6H/nxfW4jZyE7M60GFldsQ57UP1WYLNz0w0AWS1UQQ54DN4d1hEnT67Y1jzx9+iHgby7h1SO3dV3RdodJe3Zsj1yyVsqEOktu+uGIcmfZkas+B3kCK/PdKZxYXAMzZ6q1AtjySlweuTAaM826+8UoX4jlYCj90JVWihlPUVTs370wjLIAR3nkamVmluBap93UpJXhPHJhNFThLADSs7MkeMFN/8rOqkkrYshzoKcfZkUt0x84mnYmj1yTVsLqkQt+0qcfekqj3OGUA29Fp/931RwbMeQ58FaRjSKt2E2Yzyw7wbVMhtyTVsQjHx8+j7xq9+Ylxau14jx2M8kKGU5hVOzfHS95VpGpFMTnzqwAQOo8csDu2zlkyMWDzI1uyCXYWQ6C12BQaqkKYshzkGfSqN6dxxxD3sqkkdeHgp0ireRnWjfkVXPpSoqbpSIrO4W8jOINK49cGfLM0kpIHrkQhT/HOArxyEtIhEdetTtUMeQ58BYfZN93SFrJZMibWBmYMEzLrbUiDmQySV7ajBbslKJZ5WBoiX6OBIQyI5d/DtwI+QizRjVhPnZaGfL07+HWW+mbbvVD8ciTSTpCEuwsL8FenSKtCKnJo5GrJszPOS3fshgOvbmEBDuTSRuUFmmlfAw1lnCeF0MupCfnbZzdu9MJdmbJWlGlbFcNCXZmIOnanm6KtFI2go0kqlo0Swx5DvLkkQO2Tq6yT7LWWgECHnnFPJBRyBTslKBDKRhOP0wX2N5oyGzNQd5uJCrgCWTzyH3SitRaSSTt7bZIK+VjuLGE/bhq14MY8hzk1eN6Ti45gNT1yIGAtCIaeWqSztO0BDtLh1ft0Hlc0ctAZmsO8i4HHtUj17sEGSKtpCZZWhGNvGxESSvOjWplyGXIieiDRHSMiB50fm4Z18DKxKhGVLV8AzJmrTge+bk1I7T5shBO0hGq18j9QhWNvFwEOwRVzI6PpbHEHzHz74/hfUqHF1gZzYiqJszAaHnk51dtjVxklXjc3OIUx2mmVUffsEQjLwnBvPHtm2y5smpnTzoE5SBvYEX3yLNIK416DVPNGpb6BmpEIqukJE0sY6ZZxxkMxJCXhGD1ww/e+jJc1evgVVdeVNiYimAc94/vJaKHieguItoatRER3U5EB4no4MLCwhg+tnjcSTSOrJWMwTXVXMJiluX5KUlzllTAU3p2lgv1Jd1pN/CeN1yVyTHaCCT+t0R0DxEdDvm5DcCfArgKwPUAngfwB1Hvw8x3MvN+Zt7f6/XGNf5iCehyWVFNmIFsHjlgpyCqPHLxyONRRyfNYZptN0AkMYeyUNVmy0ESpRVmvinNGxHRnwH4XO4RlYhxLAfuddtYXDUyp7upBszmbEuMTkrSXOzTzbp44yUiKK1UlbxZKzu0h28DcDjfcMrFOLqRzHenQJQ93U2VsrUk2JmaNIdpplUXfbxEVLVIVpC8wc7fJaLrYWf7HAXwM3kHVCa8xQj5PPJmvZb51rDTbuLYmRWRVjJAKVTymVZDcshLRFWLZAXJZciZ+SfGNZAyk0efu+biLh5+9kzm/VQDZjvYWe1JnESWGtXTrbqs6iwRKl9cDLkwMuMoYv8zB67Eu1+7O/N+HQl2ZibNF+5bX74Du7bNrMNohHHgVv+s+CUghjwHFPg9Co16DY0RPMBOu2kvCLKkzkpa0nzfveGaebzhmvkLPxhhLKil+FW/K5V7yBzkrX6Yh067jr5pYdUwJY88gXHEMoTJRKQVGzEBOaCceeR56LTtm6nFlYFIKympuNO2IWG3jHPBAymYiv/746GIxQidKbtw1uLKoPK3lWmpute2EfE08mqfWzHkOXBXlRXw2R2ncNZZ8cjTI4dpw2FZ9m8x5MLIFKm9quYSZ1cGEuxMoMhYhnBhYUjWCiCGfCwUEWzsODVaFlcNMVApqfrFvhGRrBUbMeQ5KLJgj5JWTEuW6KclzcpOoVy4hrzizowY8hwUuTxYSSuAeCNpqfi1viHxpJVqn1wx5DkoNNg55a3lkhpP6ah6qdONiNPpUNIPix5AmQl28F5PZppex3eRVtIhh2njofLIq/4lLYY8B0Vq5LUauYuCxJDHY7kXe8EDEcaOrOy0EUM+BoqaQ2LI0yEBsY2LSCs2Ff/38+Eu0S8oG2LWyVwRAxWPJbffGxaWlZ0AxJDnwstaKebz1TJ98cjjUV6bHKWNhyzRtxFDPgaKMqRdJa1UfBInI6v/Nioim9mIIc+BF+ws5vNdaUUsVCyWXOwbFu/cFjuOohFDngO3sURBBkItChKPPB7LkqyVjYqrkVfckoshz4EX7CwGtUxfNPJ4VIqaBDs3LlW/2xJDPgaKmkRqdWfVvZEkpK/jxsWSxhIAxmDIiejniOhRInqEiH53HIMqC1zwQhNPWinm88uCBMQ2LurcVv1uK1fzZSJ6I4DbALyCmdeIqFJda4teVdaRYGcqZGXnxsX1yCt+cvN65O8B8GFmXgMAZj6Rf0jlwSw4iKaklapP4iTEI9+4yLm1yWvI9wJ4PRF9jYi+TEQ3Rm1IRLcT0UEiOriwsJDzYyeDoieRK62IRx6L8tqEjYd3V1zoMAonUVohonsAXBzy0h3O/tsAvArAjQA+RURXMg9fOcx8J4A7AWD//v0b4soqus2U5JGnQ7JWNi6WpB8CSGHImfmmqNeI6D0APuMY7q8TkQVgDsDGcLkTKNrR60oeeTpk0ciGpei74kkhr7TyfwC8EQCIaC+AFoAXc75n6ShsQdCUVD9Mg9Tj2LhI+qFNrqwVAHcBuIuIDgPoA3hXmKyy0SnqX5bqh+mQZdwbF0k/tMllyJm5D+DHxzSW0lH03HGllYp7I0l4wc5qX+wbmao7M2ICxkBR9yBTzRpajRrajXryxhWGxSPfsEgeuU1eaUUoECLCXe+6EXu3d4oeykQjzQc2Lp60Uuw4ikYMeQ6K6gyk87o9c0UPYeKZceq297rtgkcijBsv2Fn8tVgkYsjHQOWiuyXjwJ45/M4PfxdufcWlRQ9FGDNSa95GDHkOKj53SgMR4V/euKvoYQgXAHUNVj3gX/F/fzxUL+FSECaDpiOpVD39UAy5IAilpeG44tU242LIxwKLSi4IhdB0ivFXvTCaGPIcVP12ThCKpul45H1DDLkgCEIpaTgeuWFZBY+kWMSQj4GK39UJQmH84pv2ol4jXDE3W/RQCkXSD3PQcryBVkO+DwWhCL7vJdvxxH+8pehhFI4Y8hy8/cadePb0Cn7u+/cUPRRBECqMGPIctBt1fOCWlxY9DEEQKo5oAoIgCCVHDLkgCELJEUMuCIJQcsSQC4IglBwx5IIgCCVHDLkgCELJEUMuCIJQcsSQC4IglBziAgqFENECgKdCXpoD8OI6DycvMub1oWxjLtt4ARnzepFnzJczcy/4ZCGGPAoiOsjM+4seRxZkzOtD2cZctvECMub14kKMWaQVQRCEkiOGXBAEoeRMmiG/s+gBjICMeX0o25jLNl5AxrxejH3ME6WRC4IgCNmZNI9cEARByIgYckEQhJKzLoaciO4iohNEdFh77i+J6EHn5ygRPRix71Ei+oaz3cH1GK/zuTuJ6EtE9E0ieoSIft55fhsRfZGIHnN+b43Y/13ONo8R0bsKHO/vEdGjRPQwEd1NRFsi9l/34xwz5g8S0TFtfoT28iKim4no20T0OBG9v+AxT+x8JqIpIvo6ET3kjPm3nOevIKKvOcfvL4moFbH/B5xtvk1EP1DwmD/hjOOwY1eaEfub2vn4bIHj/SgRfUcby/UR++ezF8x8wX8AHACwD8DhiNf/AMBvRLx2FMDceowz8Lk7AOxz/u4COALgWgC/C+D9zvPvB/A7IftuA/Ck83ur8/fWgsb7ZgAN5/nfCRtvUcc5ZswfBPDLCfvWATwB4EoALQAPAbi2qDEHtpmo+QyAAHScv5sAvgbgVQA+BeAdzvMfAfCekH2vdY5tG8AVzjGvFzjmW5zXCMAnw8bs7HN+Qo7xRwH8SMK+ue3FunjkzHwvgFNhrxERAXg77JMyMTDz88x8yPn7HIBvAbgUwG0APuZs9jEAPxiy+w8A+CIzn2Lm0wC+CODmIsbLzF9gZsPZ7KsALruQ48hCzDFOwysBPM7MTzJzH8BfwD43F5SkMU/ifGab887DpvPDAL4PwP92no+ay7cB+AtmXmPm7wB4HPaxv6BEjZmZP++8xgC+jgmZzzHHOA257cUkaOSvB3CcmR+LeJ0BfIGI7iei29dxXC5EtBvADbC/Zbcz8/POSy8A2B6yy6UAntEeP4v0Bio3gfHq/BSAv47YrdDjHDLm9zpy0F0R8lWhxxiIPM4TOZ+JqO7IPSdgG4onAJzRvuSjjl9hxzk4Zmb+mvZaE8BPAPibiN2niOggEX2ViH7wgg8WseP9bWcu/xERtUN2zX2MJ8GQvxPx3svrmHkfgLcA+HdEdGB9hmVDRB0AnwbwC8y8qL/meAUTlb8ZNV4iugOAAeATEbsWdpxDxvynAK4CcD2A52FLFRNFzLyYyPnMzCYzXw/bg30lgJesx+fmIThmIrpOe/lPANzLzF+J2P1ytpfB/xiAPyaiqy7saCPH+wHYx/pG2NLJr1yIzy7UkBNRA8APAfjLqG2Y+Zjz+wSAu7EOt3UK51v/0wA+wcyfcZ4+TkQ7nNd3wP72DXIMwE7t8WXOcxeUiPGCiH4SwFsB/Cvny2eIoo5z2JiZ+bhzUVgA/ixiLIUcYyD2OE/0fHY+9wyALwF4NYAtzpiB6ONX2HFWaGO+GQCI6DcB9AD8Ysw+6jg/CeAfYN85rQv6eB0pjpl5DcCf4wLN5aI98psAPMrMz4a9SESzRNRVf8MO3B0O23bcOFrnfwfwLWb+Q+2lzwJQUeV3AfirkN3/FsCbiWirIwu82Xlu3cdLRDcDeB+AW5l5OWLfQo5zzJh3aJu9LWIs/wxgj5N50QLwDtjn5oISMy+ACZ3PRNQjJ1uJiKYBvAm2tv8lAD/ibBY1lz8L4B1E1CaiKwDsga1NFzHmR4no38DWlN/pfNGH7btVSRhENAfgtQC+WdB4ldNHsGMQYec7v73IEhkd9Qf2rebzAAaw9Z+fdp7/KICfDWx7CYDPO39fCTti/hCARwDcsR7jdT77dbBlk4cBPOj83ALgIgB/B+AxAPcA2OZsvx/Af9P2/ynYgaHHAby7wPE+Dlt/U899ZFKOc8yYPw7gG87znwWwIzhm5/EtsLNGnih6zJM8nwG8HMADzpgPw8moccbzdWeO/C8Abef5WwF8SNv/DucYfxvAWwoes+GMRR179bx7/QF4jTN/HnJ+/3SB4/17ZwyHAfxPeJktY7UXskRfEASh5BQtrQiCIAg5EUMuCIJQcsSQC4IglBwx5IIgCCVHDLkgCELJEUMuCIJQcsSQC4IglJz/D0Zm2UL20hsDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = []\n",
    "y = []\n",
    "xs = []\n",
    "ys = []\n",
    "for i in range(0, len(modification_sliders_sing)):\n",
    "    if modification_sliders_sing[i][0] == \"U_pointer\":\n",
    "        modification_ctrl_pts_sing[i].sort(key=lambda x:x[0])\n",
    "        for l in range(0, len(modification_ctrl_pts_sing[i])):\n",
    "            x.append(modification_ctrl_pts_sing[i][l][0])\n",
    "            y.append(modification_ctrl_pts_sing[i][l][1])\n",
    "#             xs.append(modification_ctrl_pts[i][l][0])\n",
    "#             ys.append(modification_ctrl_pts[i][l][1])\n",
    "# y = np.array(y)\n",
    "# plt.plot(np.where(y!=0, y, np.nan))\n",
    "y_interp = interp1d(x, y)\n",
    "plt.plot(np.arange(18, 35, 0.01), y_interp(np.arange(18, 35, 0.01)))\n",
    "# plt.plot(np.array(ys))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "aea24984",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[1.2923125, 0], [8.445769122517406, -0.1953793466091156], [8.47582973793415, -0.1465345099568367], [8.485849943073065, 0]], [[17.0795, 0], [20.033976206191014, 0], [17.703500000000055, 8.84707760810852], [19.478981333811486, 6.63530820608139]], [[19.443500000000377, 0], [19.913976206191013, -1.0164846926927567], [19.98956086140263, -0.7623635195195675], [20.134755746473168, 0]], [[19.913976206191013, 0], [20.024833700501382, -6.932718336582184], [20.962083425125417, -5.199538752436638], [21.274500000000096, 0]], [[19.913976206191013, 0], [21.274500000000096, 0], [20.024833700501382, 8.913495004177094], [20.962083425125417, 6.68512125313282]], [[20.014119051547752, 0], [21.95021683453238, 0], [21.059500000000053, -0.4318057894706726], [21.439066832897353, -0.32385434210300446]], [[20.014119051547752, 0], [21.95021683453238, 0], [21.059500000000053, -0.4318057894706726], [21.439066832897353, -0.32385434210300446]], [[21.154500000000095, 0], [21.83021683453238, -6.235128939151764], [22.026093597432318, -4.676346704363823], [22.172563430256634, 0]], [[21.154500000000095, 0], [22.172563430256634, 0], [21.83021683453238, 8.016594350337982], [22.026093597432318, 6.012445762753487]], [[21.94668409895937, 0], [22.063151363386357, -0.09496329352259636], [22.142560861859305, -0.07122247014194727], [22.179618627813348, 0]], [[21.9755, 0], [23.01155820044116, 0], [22.207658170417798, -1.0214355736970901], [22.72058319293532, -0.7660766802728176]], [[21.9755, 0], [23.085500000000025, 0], [22.207658170417798, -6.488565564155579], [23.03956641589398, -4.866424173116684]], [[21.9755, 0], [23.085500000000025, 0], [22.207658170417798, 8.342441439628601], [23.03956641589398, 6.256831079721451]], [[22.255539542604446, 0], [24.183500000000002, 0], [22.965500000000024, -0.4317993074655533], [23.04162500000002, -0.32384948059916496]], [[22.255539542604446, 0], [24.183500000000002, 0], [22.965500000000024, -0.4317993074655533], [23.04162500000002, -0.32384948059916496]], [[25.662125, 0], [25.942125, -1.0075317174196243], [26.018125, -0.7556487880647182], [26.163458333333335, 0]], [[25.662125, 0], [25.942125, -6.942371845245361], [26.53999166666667, -5.206778883934021], [26.702125000000002, 0]], [[25.662125, 0], [26.702125000000002, 0], [25.942125, 8.925906658172607], [26.53999166666667, 6.694429993629456]], [[25.930124999999997, 0], [27.468125, 0], [26.999665229885057, 8.885477721691132], [27.351010057471264, 6.664108291268349]], [[26.738124999999997, 0], [27.598125000000003, 0], [27.254125000000002, -0.45554615557193756], [27.33230681818182, -0.34165961667895317]], [[26.738124999999997, 0], [27.598125000000003, 0], [27.254125000000002, -0.45554615557193756], [27.33230681818182, -0.34165961667895317]], [[28.770125000000085, 0], [29.762125000000005, 0], [29.030125, 8.948858320713043], [29.617250000000006, 6.711643740534782]], [[29.114124999999998, 0], [31.495335256410257, -0.11973939463496208], [31.57087371794872, -0.08980454597622156], [31.606125000000002, 0]], [[31.8678125, 0], [32.5258125, 0], [32.1078125, -0.5347073525190353], [32.225375, -0.4010305143892765]], [[31.8678125, 0], [32.5258125, 0], [32.1078125, -0.5347073525190353], [32.225375, -0.4010305143892765]], [[32.08381250000001, 0], [32.4598125, -1.0164846926927567], [32.537373475609755, -0.7623635195195675], [32.683227134146335, 0]], [[32.4598125, 0], [32.57356859756097, -6.957819879055023], [32.80625152439024, -5.218364909291267], [32.8838125, 0]], [[32.4598125, 0], [32.8838125, 0], [32.57356859756097, 8.945768415927887], [32.80625152439024, 6.709326311945915]], [[32.4058125, 0], [33.42508909574465, 0], [32.7638125, -0.4777660667896271], [33.16976994680849, -0.3583245500922203]], [[32.4058125, 0], [33.42508909574465, 0], [32.7638125, -0.4777660667896271], [33.16976994680849, -0.3583245500922203]], [[32.7638125, 0], [33.72381249999996, 0], [33.488918882978695, -0.6907568871974945], [33.66508909574464, -0.5180676653981209]], [[32.7638125, 0], [33.72381249999996, 0], [33.488918882978695, -0.6907568871974945], [33.66508909574464, -0.5180676653981209]], [[32.727812500000006, 0], [34.7398125, 0], [33.60381249999996, 8.912220418453217], [34.483705357142846, 6.684165313839912]], [[33.83381249999985, 0], [34.961312500000005, 0], [34.6198125, -0.45935386419296265], [34.69742613636364, -0.344515398144722]], [[33.83381249999985, 0], [34.961312500000005, 0], [34.6198125, -0.45935386419296265], [34.69742613636364, -0.344515398144722]], [[34.84131250000001, 0], [35.89697883885546, 0], [35.179812500000004, -0.4859554320573807], [35.25665175059166, -0.3644665740430355]], [[34.84131250000001, 0], [35.89697883885546, 0], [35.179812500000004, -0.4859554320573807], [35.25665175059166, -0.3644665740430355]], [[35.139924330902254, 0], [36.80555535525708, 0], [36.724083948017345, -0.0066214799880981445], [36.78518750344715, -0.004966109991073608]], [[35.139924330902254, 0], [36.80555535525708, 0], [36.724083948017345, -0.0066214799880981445], [36.78518750344715, -0.004966109991073608]], [[37.835963915748145, 0], [38.5978125, 0], [38.3638125, -0.4356362968683243], [38.443585227272735, -0.3267272226512432]], [[37.835963915748145, 0], [38.5978125, 0], [38.3638125, -0.4356362968683243], [38.443585227272735, -0.3267272226512432]], [[38.477812500000006, 0], [39.155812499999996, 0], [38.78654934905287, -0.45554615557193756], [38.86567716711297, -0.34165961667895317]], [[38.477812500000006, 0], [39.155812499999996, 0], [38.78654934905287, -0.45554615557193756], [38.86567716711297, -0.34165961667895317]], [[38.730023030658046, 0], [39.5238125, 0], [39.0358125, -0.5125167220830917], [39.11368484042553, -0.3843875415623188]], [[38.730023030658046, 0], [39.5238125, 0], [39.0358125, -0.5125167220830917], [39.11368484042553, -0.3843875415623188]], [[41.18395434912646, 0], [41.921250776962346, 0], [41.91086702201994, 7.930158019065857], [41.91865483822674, 5.947618514299393]], [[41.420751095585146, 0], [42.119249999999994, 0], [41.80125077696235, -0.6906414180994034], [41.952300407905234, -0.5179810635745525]], [[41.420751095585146, 0], [42.119249999999994, 0], [41.80125077696235, -0.6906414180994034], [41.952300407905234, -0.5179810635745525]], [[41.72548716201776, 0], [42.45324999999999, 0], [42.05325, -0.45159628987312317], [42.132197368421046, -0.3386972174048424]], [[41.72548716201776, 0], [42.45324999999999, 0], [42.05325, -0.45159628987312317], [42.132197368421046, -0.3386972174048424]], [[41.999249999999996, 0], [43.60026931012415, 0], [42.33324999999999, -0.03345220908522606], [43.454892776714324, -0.025089156813919544]], [[41.999249999999996, 0], [43.60026931012415, 0], [42.33324999999999, -0.03345220908522606], [43.454892776714324, -0.025089156813919544]], [[43.37876317648484, 0], [43.65282973731097, 0], [43.490419923488076, 2.95536732673645], [43.61222728385525, 2.2165254950523376]], [[42.56324999999988, 0], [44.15725, 0], [43.50282973731101, -1.1268892139196396], [43.57951961184487, -0.8451669104397297]], [[43.53282973731097, 0], [44.495912865817445, 0], [44.03725, -0.45550301671028137], [44.11543117030979, -0.34162726253271103]], [[43.53282973731097, 0], [44.495912865817445, 0], [44.03725, -0.45550301671028137], [44.11543117030979, -0.34162726253271103]], [[43.91725000000001, 0], [46.89027347834793, -0.08984784688800573], [46.96570429672384, -0.0673858851660043], [47.06125, 0]]]\n"
     ]
    }
   ],
   "source": [
    "# smooth the vowel modificatoin sliders\n",
    "\n",
    "vm_ctpt_temp = copy.deepcopy(modification_ctrl_pts_sing)\n",
    "vm_slider_temp = copy.deepcopy(modification_sliders_sing) # these are references\n",
    "# pass 3\n",
    "# set this up\n",
    "prev_slider_dict = {}\n",
    "for i in range(0, len(vm_slider_temp)):\n",
    "    prev_slider_dict[vm_slider_temp[i][0]+vm_slider_temp[i][1]] = -1\n",
    "modification_ctrl_pts = copy.deepcopy(modification_ctrl_pts_sing) # output is stored here\n",
    "modification_sliders = copy.deepcopy(modification_sliders_sing)\n",
    "i = 0  \n",
    "new_ctrl_pts = []\n",
    "new_sliders = []\n",
    "while i < len(modification_ctrl_pts):\n",
    "    increment = 1\n",
    "    # if the previous instance of the current viseme is not -1\n",
    "    if prev_slider_dict[vm_slider_temp[i][0]+vm_slider_temp[i][1]] != -1:\n",
    "        prev_i = prev_slider_dict[vm_slider_temp[i][0]+vm_slider_temp[i][1]]\n",
    "        current_interval = copy.deepcopy(vm_ctpt_temp[i])\n",
    "        current_interval.sort(key=lambda x:x[0])\n",
    "        prev_interval = copy.deepcopy(vm_ctpt_temp[prev_i])\n",
    "        prev_interval.sort(key=lambda x:x[0])\n",
    "        \n",
    "        # a simplified detection and merging operation:\n",
    "        if current_interval[0][0] < prev_interval[-1][0]:\n",
    "            temp = copy.deepcopy(prev_interval)\n",
    "            prev_interval = 0\n",
    "        \n",
    "        # For the same vowel/consonant if the two intervals do not overlap but the onset and off set overlap \n",
    "        elif (current_interval[0][0] - prev_interval[-1][0]) <= vowel_blend_distance:\n",
    "            # if we have weird overlap where the boundary of the previous interval interupts the curve of the current\n",
    "#             if prev_interval[2][0] - prev_interval[1][0] <= speech_vowel_threshold:\n",
    "#                 prev_interval[2][1] = 0.95 * prev_interval[1][1]\n",
    "            if current_interval[1][0] <= prev_interval[-1][0] and current_interval[0][0] <= prev_interval[-2][0]:\n",
    "                temp = copy.deepcopy(prev_interval)\n",
    "                temp[-1][0] = (current_interval[0][0] + prev_interval[-1][0]) / 2.0\n",
    "                temp[-1][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "                modification_ctrl_pts[prev_i] = copy.deepcopy(temp)\n",
    "                current_interval[0][0] = (current_interval[0][0] + prev_interval[-1][0]) / 2.0\n",
    "                current_interval[0][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "#                 modification_ctrl_pts[i] = copy.deepcopy(current_interval)\n",
    "                new_ctrl_pts.append(copy.deepcopy(current_interval))\n",
    "            elif (current_interval[1][0] <= prev_interval[-1][0]):\n",
    "                temp = copy.deepcopy(prev_interval)\n",
    "                temp[-1][0] = current_interval[0][0]\n",
    "                temp[-1][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "#                 visemes_sing.viseme_ctrl_pts[-1][prev_slider_dict[viseme_list_final[i]]] = temp\n",
    "                modification_ctrl_pts[prev_i] = copy.deepcopy(temp)\n",
    "                current_interval[0][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "#                 visemes_sing.add(viseme_list_final[i], current_interval, pure_phoneme_list[i])\n",
    "                modification_ctrl_pts[i] =  copy.deepcopy(current_interval)\n",
    "            elif (current_interval[0][0] <= prev_interval[-2][0]):\n",
    "                temp = copy.deepcopy(current_interval)\n",
    "                temp[0][0] = prev_interval[-1][0]\n",
    "                temp[0][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "                modification_ctrl_pts[prev_i][-1][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "#                 visemes_sing.viseme_ctrl_pts[-1][prev_slider_dict[viseme_list_final[i]]][-1][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "#                 visemes_sing.add(viseme_list_final[i], temp, pure_phoneme_list[i])\n",
    "                modification_ctrl_pts[i] =  copy.deepcopy(temp)\n",
    "            else:\n",
    "                if pure_phoneme_list[i] in cmu_sets.lip_closer:\n",
    "                    pass\n",
    "#                     visemes_sing.add(viseme_list_final[i], viseme_intervals_final[i], pure_phoneme_list[i])\n",
    "                else:\n",
    "                    temp = copy.deepcopy(prev_interval)\n",
    "                    temp[-1][0] = (current_interval[0][0] + prev_interval[-1][0]) / 2.0\n",
    "                    temp[-1][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "#                     visemes_sing.viseme_ctrl_pts[-1][prev_slider_dict[viseme_list_final[i]]] = copy.deepcopy(temp)\n",
    "                    modification_ctrl_pts[prev_i] = copy.deepcopy(temp)\n",
    "                    current_interval[0][0] = (current_interval[0][0] + prev_interval[-1][0]) / 2.0\n",
    "                    current_interval[0][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "#                     visemes_sing.add(viseme_list_final[i], current_interval, pure_phoneme_list[i])\n",
    "                    modification_ctrl_pts[i] =  copy.deepcopy(current_interval)\n",
    "    else:\n",
    "        new_sliders.append(modification_sliders[i])\n",
    "        new_ctrl_pts.append(modification_ctrl_pts[i])\n",
    "        \n",
    "    prev_slider_dict[vm_slider_temp[i][0]+vm_slider_temp[i][1]] = i\n",
    "    i = i + increment\n",
    "modification_ctrl_pts_sing = copy.deepcopy(modification_ctrl_pts)\n",
    "modification_sliders_sing = copy.deepcopy(modification_sliders)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "68b472c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "viseme_list = visemes_speak.viseme_list[-1]\n",
    "viseme_interval = visemes_speak.viseme_ctrl_pts[-1]\n",
    "\n",
    "modification_ctrl_pts = []\n",
    "modification_sliders = []\n",
    "# iterate through the vowels in the list\n",
    "dt = 0.01\n",
    "for i in range(0, len(viseme_list)):\n",
    "    if viseme_list[i] in VOWELS_SLIDERS_JALI and viseme_interval[i][-2][0] - viseme_interval[i][0][0] > 0.30:\n",
    "        ##################################################################\n",
    "        ###################### get the audio signal ######################\n",
    "        ##################################################################\n",
    "        vowel_mod_out, vowel_mod_out_coarse = vowel_mod(lyric.sound_arr_interp(np.arange(viseme_interval[i][1][0], \n",
    "                                                                                        min(viseme_interval[i][-1][0], lyric.snd.xs()[-1]), 1.0/44100.0)))\n",
    "#         print([\"A\", \"Stretcher\", \"Rounder\", \"Silence\"][vowel2Cardinal3[viseme_list[i]]])\n",
    "        if  vowel_mod_out_coarse.shape[0] <= 10:\n",
    "            continue\n",
    "        xs = np.linspace(viseme_interval[i][1][0], min(viseme_interval[i][-1][0], lyric.snd.xs()[-1]), vowel_mod_out_coarse.shape[0])\n",
    "        coarse_vowel_sounds_like_interp = interp1d(xs, vowel_mod_out_coarse, axis=0)\n",
    "        \n",
    "#         for i in range(0, 4):\n",
    "#             plt.plot(vowel_mod_out_coarse[:, i], label=[\"Ah\", \"stretcher\", \"rounder\", \"silence\"][i])\n",
    "# #         plt.plot(np.argmax(out[0, :], axis=1))\n",
    "#         plt.legend()\n",
    "#         plt.show()\n",
    "        \n",
    "        # what the original sound was\n",
    "        original_vowel_shape = vowel2Cardinal3[viseme_list[i]]\n",
    "        only_peaks = np.where(vowel_mod_out_coarse > 0.75, vowel_mod_out_coarse, 0)\n",
    "        vowel_sounds_like = np.argmax(only_peaks, axis=1)\n",
    "        ##################################################################\n",
    "        ### obtain the intervals of which cardinal vowels are dominant ###\n",
    "        ##################################################################\n",
    "        cardinal_list = []\n",
    "        cardinal_intervals = []\n",
    "        current_interval_start = 0\n",
    "        current_vowel = original_vowel_shape\n",
    "        for t in range(0, vowel_sounds_like.shape[0]):\n",
    "            if vowel_sounds_like[t] == current_vowel:\n",
    "                if (t == vowel_sounds_like.shape[0]-1) and current_vowel != 3:\n",
    "                    cardinal_list.append(current_vowel)\n",
    "                    cardinal_intervals.append([current_interval_start, t])\n",
    "            else:\n",
    "                if xs[t-1] - xs[current_interval_start] >= 0.1:\n",
    "                    if t > 0 and current_vowel != 3:\n",
    "                        cardinal_list.append(current_vowel)\n",
    "                        cardinal_intervals.append([current_interval_start, t-1])\n",
    "                    current_interval_start = t\n",
    "                    current_vowel = vowel_sounds_like[t]\n",
    "        ###########################################################################\n",
    "        ######### optionally additional smoothing are added to this here ##########      \n",
    "        ###########################################################################\n",
    "        cardinal_list_new = []\n",
    "        cardinal_intervals_new = []\n",
    "        j = 0\n",
    "        while j < len(cardinal_list):\n",
    "            step = 1\n",
    "            if j == len(cardinal_list) - 1:\n",
    "                cardinal_list_new.append(cardinal_list[j])\n",
    "                cardinal_intervals_new.append(cardinal_intervals[j])\n",
    "            elif cardinal_list[j] == cardinal_list[j+1] and xs[cardinal_intervals[j+1][0]] - xs[cardinal_intervals[j][1]] <= spike_width:\n",
    "                cardinal_list_new.append(cardinal_list[j])\n",
    "                cardinal_intervals_new.append([cardinal_intervals[j][0], cardinal_intervals[j+1][1]])\n",
    "                step = 2\n",
    "            elif j < len(cardinal_list) - 2:\n",
    "                if (cardinal_list[j] == cardinal_list[j+2] and xs[cardinal_intervals[j+2][0]] - xs[cardinal_intervals[j][1]] <= spike_width \n",
    "                    and cardinal_list[j+1] == original_vowel_shape):\n",
    "                    cardinal_list_new.append(cardinal_list[j])\n",
    "                    cardinal_intervals_new.append([cardinal_intervals[j][0], cardinal_intervals[j+2][1]])\n",
    "                    step = 3\n",
    "            else:\n",
    "                cardinal_list_new.append(cardinal_list[j])\n",
    "                cardinal_intervals_new.append(cardinal_intervals[j])\n",
    "            j = j + step\n",
    "        cardinal_list = cardinal_list_new\n",
    "        cardinal_intervals = cardinal_intervals_new\n",
    "#         print(cardinal_list, cardinal_intervals)\n",
    "        # now set pucker/stretch values based on the detected sound\n",
    "        for c in range(0, len(cardinal_list)):\n",
    "            if original_vowel_shape == cardinal_list[c] or cardinal_list[c] == 3:\n",
    "                continue\n",
    "            else:\n",
    "                max_prob = coarse_vowel_sounds_like_interp(xs[cardinal_intervals[c][0]:cardinal_intervals[c][1]+1])[:, cardinal_list[c]].max()\n",
    "                \n",
    "                slider_ct_pts = control_direction_matrix_coarse[original_vowel_shape][cardinal_list[c]]\n",
    "                for s in range(0, int(len(slider_ct_pts)/3)):\n",
    "                    ctrl_pts = []\n",
    "                    # get the name and attribute of the slider\n",
    "                    slider_name = slider_ct_pts[0 + 3*s]\n",
    "                    if slider_name == \"self\":\n",
    "                        slider_name = viseme_list[i]\n",
    "                    slider_attribute = slider_ct_pts[1 + 3*s]\n",
    "                    slider_range = slider_ct_pts[2 + 3*s]\n",
    "                    # add a starting keyframe and ending keyframe\n",
    "                    modification_sliders.append([slider_name, slider_attribute])\n",
    "                    # the start of this curve should be earlier, e.g. at 75% of the previous interval\n",
    "                    # however, if the detected modification is at the very beginning of the vowel, then it is\n",
    "                    # applied the same time as the beginning of the vowel\n",
    "                    same_start = False\n",
    "                    if c == 0 : \n",
    "                        start = viseme_interval[i][0][0]\n",
    "                        same_start = True\n",
    "                    else:\n",
    "                        # otherwise I choose the starting point \n",
    "                        # find the previous c that is not the current vowel\n",
    "                        ci = -1\n",
    "                        for ccci in range(c-1, -1, -1):\n",
    "                            if original_vowel_shape != cardinal_list[ccci] and cardinal_list[ccci] != 3:\n",
    "                                ci = ccci\n",
    "                                break\n",
    "                        if ci > -1:\n",
    "#                             start_candidate = (xs[cardinal_intervals[ci][1]] - xs[cardinal_intervals[ci][0]]) * 0.6 + xs[cardinal_intervals[ci][0]]\n",
    "#                             start_candidate = min(start_candidate, xs[cardinal_intervals[c][0]]-0.12)\n",
    "                            start_candidate = xs[cardinal_intervals[ci][0]]-0.12\n",
    "                            if viseme_interval[i][0][0] >= start_candidate:\n",
    "                                same_start = True\n",
    "                            start = max(start_candidate, viseme_interval[i][0][0])\n",
    "                            if np.abs(start - viseme_interval[i][0][0]) <= 0.2:\n",
    "                                start = viseme_interval[i][0][0]\n",
    "                                same_start = True\n",
    "                        else:\n",
    "                            start_candidate = xs[cardinal_intervals[ci][0]]-0.12\n",
    "#                             start = max(viseme_interval[i][0][0], start_candidate)\n",
    "                            start = viseme_interval[i][0][0]\n",
    "#                             if np.abs(start - viseme_interval[i][0][0]) <= 0.2:\n",
    "#                                 start = viseme_interval[i][0][0]\n",
    "#                                 same_start = True\n",
    "                    ctrl_pts.append([start, slider_range[0]])\n",
    "                    # add the end point of the curve. This would either at the end of the \n",
    "                    # prediction interval, or at the end of the vowel. \n",
    "#                     modification_sliders.append([slider_name, slider_attribute])\n",
    "                    if c < len(cardinal_list) - 1:\n",
    "                        ctrl_pts.append([xs[cardinal_intervals[c][1]] + 0.12, slider_range[0]])\n",
    "                    else:\n",
    "                        ctrl_pts.append([viseme_interval[i][-1][0]+0.12, slider_range[0]])        \n",
    "                    # add the peaks in the middle with the decay\n",
    "#                     modification_sliders.append([slider_name, slider_attribute])\n",
    "                    # with some anticipation \n",
    "#                     print(slider_name, max_prob, slider_range[1])\n",
    "                    if same_start:\n",
    "                        ctrl_pts.append([viseme_interval[i][1][0], max_prob * (slider_range[1])])\n",
    "                    else:\n",
    "                        ctrl_pts.append([xs[cardinal_intervals[c][0]], max_prob * (slider_range[1])])\n",
    "#                     modification_sliders.append([slider_name, slider_attribute])\n",
    "                    end_p75 = (xs[cardinal_intervals[c][1]] - xs[cardinal_intervals[c][0]]) * 0.75 + xs[cardinal_intervals[c][0]]\n",
    "                    ctrl_pts.append([end_p75, max_prob * (slider_range[1]) * 0.75])\n",
    "                    if slider_name == \"Pucker\":\n",
    "                        print(slider_name, ctrl_pts)\n",
    "                        print(cardinal_list)\n",
    "                    \n",
    "                    modification_ctrl_pts.append(ctrl_pts)\n",
    "modification_ctrl_pts_speak = copy.deepcopy(modification_ctrl_pts)\n",
    "modification_sliders_speak = copy.deepcopy(modification_sliders)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "711a085d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'U_pointerlipPucker': -1, 'DimpleDimple': -1, 'Eh_pointerlipCornerPull': -1, 'Eh_pointerlipStretch': -1, 'Oo_pointerlipPucker': -1, 'Ee_pointerlipCornerPull': -1, 'Ee_pointerlipStretch': -1}\n"
     ]
    }
   ],
   "source": [
    "vm_ctpt_temp = copy.deepcopy(modification_ctrl_pts_speak)\n",
    "vm_slider_temp = copy.deepcopy(modification_sliders_speak) # these are references\n",
    "# pass 3\n",
    "# set this up\n",
    "prev_slider_dict = {}\n",
    "for i in range(0, len(vm_slider_temp)):\n",
    "    prev_slider_dict[vm_slider_temp[i][0]+vm_slider_temp[i][1]] = -1\n",
    "\n",
    "print(prev_slider_dict)\n",
    "modification_ctrl_pts = copy.deepcopy(modification_ctrl_pts_speak) # output is stored here\n",
    "modification_sliders = copy.deepcopy(modification_sliders_speak)\n",
    "i = 0  \n",
    "while i < len(modification_ctrl_pts):\n",
    "    increment = 1\n",
    "    # if the previous instance of the current viseme is not -1\n",
    "    if prev_slider_dict[vm_slider_temp[i][0]+vm_slider_temp[i][1]] != -1:\n",
    "        prev_i = prev_slider_dict[vm_slider_temp[i][0]+vm_slider_temp[i][1]]\n",
    "        current_interval = copy.deepcopy(vm_ctpt_temp[i])\n",
    "        current_interval.sort(key=lambda x:x[0])\n",
    "        prev_interval = copy.deepcopy(vm_ctpt_temp[prev_i])\n",
    "        prev_interval.sort(key=lambda x:x[0])\n",
    "        # For the same vowel/consonant if the two intervals do not overlap but the onset and off set overlap \n",
    "        if (current_interval[0][0] - prev_interval[-1][0]) <= vowel_blend_distance:\n",
    "            # if we have weird overlap where the boundary of the previous interval interupts the curve of the current\n",
    "#             if prev_interval[2][0] - prev_interval[1][0] <= speech_vowel_threshold:\n",
    "#                 prev_interval[2][1] = 0.95 * prev_interval[1][1]\n",
    "            if current_interval[1][0] <= prev_interval[-1][0] and current_interval[0][0] <= prev_interval[-2][0]:\n",
    "                temp = copy.deepcopy(prev_interval)\n",
    "                temp[-1][0] = (current_interval[0][0] + prev_interval[-1][0]) / 2.0\n",
    "                temp[-1][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "                modification_ctrl_pts[prev_i] = copy.deepcopy(temp)\n",
    "                current_interval[0][0] = (current_interval[0][0] + prev_interval[-1][0]) / 2.0\n",
    "                current_interval[0][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "                modification_ctrl_pts[i] =  copy.deepcopy(current_interval)\n",
    "            elif (current_interval[1][0] <= prev_interval[-1][0]):\n",
    "                temp = copy.deepcopy(prev_interval)\n",
    "                temp[-1][0] = current_interval[0][0]\n",
    "                temp[-1][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "#                 visemes_sing.viseme_ctrl_pts[-1][prev_slider_dict[viseme_list_final[i]]] = temp\n",
    "                modification_ctrl_pts[prev_i] = copy.deepcopy(temp)\n",
    "                current_interval[0][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "#                 visemes_sing.add(viseme_list_final[i], current_interval, pure_phoneme_list[i])\n",
    "                modification_ctrl_pts[i] =  copy.deepcopy(current_interval)\n",
    "            elif (current_interval[0][0] <= prev_interval[-2][0]):\n",
    "                temp = copy.deepcopy(current_interval)\n",
    "                temp[0][0] = prev_interval[-1][0]\n",
    "                temp[0][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "                modification_ctrl_pts[prev_i][-1][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "#                 visemes_sing.viseme_ctrl_pts[-1][prev_slider_dict[viseme_list_final[i]]][-1][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "#                 visemes_sing.add(viseme_list_final[i], temp, pure_phoneme_list[i])\n",
    "                modification_ctrl_pts[i] =  copy.deepcopy(temp)\n",
    "            else:\n",
    "                if pure_phoneme_list[i] in cmu_sets.lip_closer:\n",
    "#                     visemes_sing.add(viseme_list_final[i], viseme_intervals_final[i], pure_phoneme_list[i])\n",
    "                    pass\n",
    "                else:\n",
    "                    temp = copy.deepcopy(prev_interval)\n",
    "                    temp[-1][0] = (current_interval[0][0] + prev_interval[-1][0]) / 2.0\n",
    "                    temp[-1][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "#                     visemes_sing.viseme_ctrl_pts[-1][prev_slider_dict[viseme_list_final[i]]] = copy.deepcopy(temp)\n",
    "                    modification_ctrl_pts[prev_i] = copy.deepcopy(temp)\n",
    "                    current_interval[0][0] = (current_interval[0][0] + prev_interval[-1][0]) / 2.0\n",
    "                    current_interval[0][1] = min(current_interval[1][1], prev_interval[-2][1]) * vowel_blend_magnitude\n",
    "#                     visemes_sing.add(viseme_list_final[i], current_interval, pure_phoneme_list[i])\n",
    "                    modification_ctrl_pts[i] =  copy.deepcopy(current_interval)        \n",
    "    prev_slider_dict[vm_slider_temp[i][0]+vm_slider_temp[i][1]] = i\n",
    "    i = i + increment\n",
    "modification_ctrl_pts_speak = copy.deepcopy(modification_ctrl_pts)\n",
    "modification_sliders_speak = copy.deepcopy(modification_sliders)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ce7afab",
   "metadata": {},
   "source": [
    "## Larynx movements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "7bbbab84",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABSSUlEQVR4nO2deZwdVZn3f0/Vvd1ZutPZt+6ELATITqDZGURFFgUBRSe4oYODo+DyOvPOiPOOIjqOjiM6jjrIaAZHZRkRxqhogAAysoR0IPvaSYB0Z+lOOuns6eWe94+qurdu1TlV59x7bi+H8/18+nP71q2qe7q66qmnnuf3PIcYY7BYLBaLuTj9PQCLxWKxVBZr6C0Wi8VwrKG3WCwWw7GG3mKxWAzHGnqLxWIxnEx/D4DH2LFj2bRp0/p7GBaLxTJoWLVq1X7G2DjeZwPS0E+bNg1NTU39PQyLxWIZNBDR66LPbOjGYrFYDMcaeovFYjEca+gtFovFcKyht1gsFsOxht5isVgMJ9XQE9EUInqGiDYS0QYi+ixnHSKi7xFRMxGtJaJzQp/dQkTb/J9bdP8BFovFYklGRl7ZA+CvGWOvEFEtgFVE9CRjbGNonWsAzPJ/LgDw7wAuIKLRAL4MoBEA87ddyhg7qPWvsFgsFouQVI+eMbaHMfaK//sRAJsA1EdWux7AfzGPlwCMJKJJAK4C8CRjrMM37k8CuFrrXxDh16tbsf/oqdT1/rB+L3YfOpG6Xi7H8PDKN3DsVI+O4VlKJJdjaG47ikdfacHDK99Apdtr/2bNbhw81qW83bZ9R/CnbfsrMKI43b05PLzyDXT35lLXfXLjPuzp9M734109eGRVS+oxfGRVS0nHoBy6e3N4YMUbZX3vupZO/O+2do2jivOH9Xux7/BJ5e027z2MFTsOhPazB20l7EcVpRg9EU0DsAjAishH9QB2hd63+MtEy3n7vo2Imoioqb29tH/S8a4efPah1fiXZVsS1zvV04tP/WIVvrd8W+o+n9++H3/3q3V47NXWksZkUYcxhl0dx/G7tXvwT49vwuL7XsSCrzyBK+75Iz7/32vwd79ah9+u3VOx7z9yshuffvBV/OwlYf2JkB89twO3/awJJ7t7KzCyYppeO4i/+9U6PL4u/Vjc/sAruO+5HQA8o/83v1yDjXsOC9dvP3IKf/PLNfja7zZpG28a3b05fObBV/HFx9bhV6+0lLSPg8e68LH7X8ZnHnwVuVzlnIHbH3gFP3ymWXm7f1vejM8+tBqA57x86hev4Ef+/6WSSFfGElENgF8B+BxjTHyGlAhj7D4A9wFAY2NjSf+hXv8fu3TNbvy/a+egppr/5+3tPIkcA1bs7Ejd5/JNbQCADbs7SxmSRYK2Iyexdlcn1rYcwpqWTqxr7USH79FVuQ5mT6rFjYvqMb+hDgsa6vC3j6zFV36zAX82ayxGDqvSPp7gPNrWdlR5257eHI539eL55v14++wJuodW/F05z5N/alMbrj+b6z8VjWvn/mMAgGOnvJvQro7jmDu5jrt+cAwee7UFn3rrTMwcV6Nr2Fy6e3P49AOv4g8b9iLrErbsPVLSfu7+7UbsP+qdO5v3HsGcySN0DhOA54j05hhWt6jbhO7eHPYePom2wycxengVcqxvbIuUoSeiLDwj/wvG2KOcVVoBTAm9b/CXtQK4PLL82VIGKkNwAz/e1YvfrtmNxedP5a7X6odsdu4/hn2HT2LCiCHc9RhjWL55HwBgfav2e9ubks7j3VjbeghrWzqxZpf3utd/dHUIOGNCLa6YPR7zG0ZiYUMdzpxYi+qMW7SPb7xnAa77/p/w9cc34Z9vWqh9jMF51FyCoQ+2XbZhb8UNffBdz25pQ1dPDlUZ8QN6jiFv6IOnjTc6jiesz/Lb/etT2/C9mxdpGnWcrp4cPv3gK1i2YR/+4do5eGZzG7bsUzf0yzftw2OvtmLxeVPw0MpdeHHHgYoY+uC4b9p9GKd6emPnp8y2a1s6cdkZXluajbsPgzEGItI91Dyphp68b/8JgE2MsXsEqy0FcAcRPQQvGdvJGNtDRMsAfJ2IRvnrXQngTg3j5hN6Dnho5S6hod99qBATe2nHAaE3tK3tKHZ1nMC42mps2Xsk9WKyFHO8qwfrWw8XPPWWQ3jtQMG4TB87HOdPH40FDXVYOGUk5k4egWFV6b7HnMkj8Jd/NgP3/nE7blhUj4tnjtU67iB2vb39KHpzDK4jfwEGp+BTm9rQ05tDxq3c+RKM88jJHqx8rQOXnJ58HFoOnkBXTw4nJAx98HdMqhuC36zdjdvfejrOnFirZdxhunpyuOOBV/DExn348nVz8LFLpmP3oRP4xYrXkcsxOJLHvvNEN7742DqcNbEWd18/Dyt2duDF7ftx66XTtY85OO5dvTls3nMEC6eMVNkaALC25RD+7Azv/3X4ZA9aDp7AlNHDNI+0gIxHfwmADwNYR0Sr/WVfBDAVABhj9wJ4HMA7ATQDOA7gY/5nHUT0VQAr/e3uZoylx0tKhPkHceroYVi96xA27z2MsybG7+hBEramOoOXdnQIDf1Tmzxv/hOXzcDXfrcJW/cdwbx6/qPum51TPb3YvOdIyKh3YlvbkbwHM7luCOY31OF9jVOwsGEk5tfXoW5YtuTv+9wVs/D79XvwxUfX4Q+fuwxDsvJeVRqBkevqyaHl4HGcNma4/La+Eeg41oWm1w/iwhljtI0r9l2h35/atE9o6IMx9eYYdh08jlN5Qy8WIwTbfOySafje8mZ896mt+PcPnatn4D5dPTnc/sAreHLjPtx13Rx89BLPKJ85oRYnu3N4o+M4po2VO/Zf/90m7D/ahR9/5DxUZRxcNHMMfrN6d0VutuHjvqblkJKhD/Lfa1s7Ec6Fb9h9uH8NPWPsTwASb6vMOytuF3y2BMCSkkanSHDgblhUj3uf3Y6HXt6Fu949N7be7kMnMLamGgsa6rBi54HY5wFPb2rDvPoRePvsCfja7zZhw+5Oa+jhGYxtbUe8uLofhtm05zC6e71/wOjhVVjQUIer5k3EwoY6zG+ow/hafnisVIZkXfzTjfPxgR+vwPeWb8PfXn2Wtn2HL8Bt+46qGXp4XvCBY114YsO+ihr6wOKMranCU5v24UvXzuE+/of/np3tx/IefUuSR+9vM2pYFf7i0un43vJt2LC7UxjTV6WrJ4dP/eIVPLVpH77y7rm45eJp+c+CJ4fNe49IGfrntrbj4aZd+NTlMzG/wRvfxTPH4IEVb2D97sM4W8njTid8PFfvOoSPXKSwrf+6NhLf37jnMK6eN7H8wQkYkG2KSyU4iGOGV+GqeRPx2Kut+MI1Z8W8vdZDJ1A/cggunDEaT29uQ9vhkxgfidN3HOvCK28cxB1vm4XTRg9DbXUG61sP48/P66M/ZoDAGMNrB45jbYtn0Ne2HML61sN5Y1FbncG8+jr8xaXTsbBhJBY01KF+5NCKxhsDLj59LN53bgPue24Hrl0wWVs8loV8tm1tR3HFHIVYOwOGV2cwZ9IILNuwF/9w7eyKHYtgnG8/awIebtqFrfuOcsMrYQ/0tQPHcLLbS+K2HDyRGpoiItx66XTc//xOfOfJrfjxLeVfAKd6enH7L17BU5vacPf1c/GRi6YVfT5rQg2IgC17j6Qav6OnenDno+swc9xwfObts/LLgxvsC9v36zf0oSO6etchtW1DT3wtBws32o0VTsiaZej9g0gELD5vCn6zZjeWbdgbC83sPnQCZ0yoxQXTvZNhxc4OXLdwctE6z2xuQ44BV8weD8chzJk8AusNV94wxrCn82TeoAevh096NQTVGQdzJ4/An583BQun1GFBw0hMHzNcOo5aCf7+XbPxzJY23PnoWjz6qUuU4ulCQpZRNSHLwEAArpo7Ecs3t2HD7sMVewoMPMsr5niG/qlN+/iGPuSC7th/DN09nqHv6s1h3+GTmDxyqHDfBKBuaBa3XTYD//LEVqzedagsw3mqpxef+vkrWL65DV+9YR4+fOFpsXWGVWUwdfQwbJVIyH7z95uxu/MEHvmri4scurE11ThzQi1e3H4An7r89JLHyyM4NjXVGexoP4bOE92oGyoXhmTwlGRdvTms3lWwJxt2V1bsYZShz4VOzotmjMGU0UPx0Mu7igw9Ywy7D53E5WeOx9zJI/w4/YGYoX96cxvG11Zjnv+oOq++Dr9Y8XrFE2x9yYGjp7C2tbNI2hgUm2UcwpkTa/GuBZOxsMEz6rMm1CA7wP72kcOq8KXr5uIzD76K+194TUvyLVdk6NXUH7mc52i8ffZ4OAQ8sWFvxQx9MM4JI7ww5FOb9uH2t8aNWvjveW3/MYweXpCkvtFxnGvocyGnCQA+esl0/ORPO3HPk1vxX39xfknjPdXTi0/+/BU8vbkNX7thHj7EMfIBZ06oxea9ycbvxe0H8LOXXsfHL52Oc08bFfv8oplj8NDKN5SVMWkEhn7hlDo833wA61o6ceksOUFAjgEzx9egue0I1vhPA2NrqrGn8yQ6jnUV/W90YpShzz9SEcFxCIvPm4pvLduC1/Yfy8f6Dh3vxonuXkweORQZ10HjtFExPX1XTw5/3NqOaxdMynur8+pH4GR3Dtvbj1VEfVBpjpzsxrrWzryXvmZXZ15mSgTMHFeDy84Y6yVKG+owZ9IIrQnOSnLdgkl47JUWfPuJLbhq7gQ0jCovqRWcR0OyDprbjipJ3zyPnjCmphqN00bjiY378PkrzyxrPMLvCowxCFfMnoDvPLUV7UdOYVxtdWxMATv3H8OwqgxqqjM4eqoHuzqOc/MIwRbBn11TncEn3jIT3/j9ZjS91oHGaaOVxnqyuxef/PkqPLOlHf944zx88AKxkQeAsybWYvnmNpzs7uWeh8e7evB3v1qL08YMw18Lju/FM8fg/hdew5pdnTh/utp4kwiO58KGkXi++QBW7zoobegZY6jOODhr4gisaTkEwLMtz25px8bdh6X3o8rAcs/KxT87g6f3m85tgOsQHm4qFOcGxq1+pBeTv3DGGDS3HUX7kULbhJd3duDoqZ4iHXTg2a9vHfjhm5PdvVj1+kH85/M78X8eXo23fftZzL/rCXzgP1bgG7/fjHWtnTh76kh88Z1n4cG/vBBrv3wlnvr8W3DP+8/GLRdPwzlTRw0aIw94ceSv3TgfAPD//md92e0Rgs1nja/Fsa5e7OmUL1FnrGAcr5o7EZv3HsHrB46VNR7hd/mvRMCiqSPBGLCjPR5qCv6eYVUu9nSexMHjXZgxbjgc8oqmuPv2N3JCN7iPXHQaxtZU4dtPbFUa58nuXnziZ56R//qN81ONPACcMbEWvX7bCx7ffmIr3ug4jm++dwGGVvHP1QtmjIFDXpxeJ8HxrBuaxdiaKrQeUmth4BAwbezwvPpvqq+2CVpUVALDPHoP8kVCE0YMwVvPHI9HVrXg8+84A1nXyR/c4HH1Av9Ov2LnAVy7wAvfLN+8D9UZB5eG5GozxtVgSNbB+t2deO+5DX30F6XT3ZvDlr1HiuLqW/cdQY//vD6+thoLGkbihrPrscAPwVTq8bA/qR85FH9z5Zm4+7cbsXTN7tRK0SSC82jWhBqsa+1Ec9tRbnhDtG3g/V85ZwK++tuNWLZhL267bGbJ4xF+V+h+FuQmkqr+Z4wbjvWth7F17xHMb6jDpLqhQi09bzfDqjL45OWn46u/3YgXtu+Xql8IjPwft7bjG++ZL6xtiXKW/9TMkzSver0DS57fiY9cdFqiqqluaBbz6uvwwvYD+NwVUl8rRfgG6xApORaeI0BwqVB9HPzvKtm9ySxDX4jc5Fl83hQ8tWkfntnchivnTowZ+nn1dRhe5WLFjg5cu2CyVw27qQ2XnD62yFNwHcKcSSOwoR8rZHM5hh37j2KNH1Nf29qJjbsP45SfXKsbmsWChjp84qwZWNAwEgsbRmJinV5Z40Dmloun4ddrduPu32zEZbPGYVSJN7Tgwp013jM229qO5qsY07ctaJGnjB6GuZNHYNmGfRUx9IFpcIjgUmDo4+YiWDRjbA3Wtx7GkVM9GJJ1MXX0MOw6yPciC9dSccjqgxdMxX3Pbcc9T2zFRX81JjGkdbK7F3/5X034U/N+fPO98/Hn58kZeQCYNmY4qlwn1grhZHcv/u8jazG5bqiUpPaimWOw5E87caKrV+j5qxIOmTlE3GMu3NZP1jtEeWcs/7+rYG8eowx9PoEUWnb5meMwYUQ1Hlq5yzP0nSdRlXEwxjcCWdfBudNG4yW/o1xz21G80XEct102I7b/+fV1eGRVi1LFXqkwxtBy8ATWtBzCupZOrPFljUf9LprDqlzMq6/Dhy88DQumeO0Cpo4e1ieyxoGK6xC+8Z75uO7f/oR/fHwT/uV9pbVHCK7bMTVVGDUsq6S88eL5hfdXzpmI7y7firYjJ7XXEoQdm8Ar7OUYi+C6CGvSh2ZdjKupxtNb2gT7jl9LgFe/cMdbT8c//HoDntu2H28R3ACLjPx7FuD9503hrici4zqYOb4GmyOG/rtPbcOO9mP42a3nC3tZhbl45lj86I870PR6B/5sltzNOo1c5Lir2OcgWe84lDfsrpv+NFYuRhn6aAIJ8E6Y9507BT98thl7Ok/4GvpinfeFM0bjn/+wBfuPnsLyzd6J//bZ42P7n1tfh5+++DpeO3AMMzQ3eWo7fDLfJoDb2GvyCNy4qD7fLmDmuBo9UkLDmD1pBG67bAZ++Ox23HB2fUnJrbC0cNb4WiXljRe6Kby/ap6XJH1qYxs+cIG8RytD2OBQkkfvv9ZUu5gwohr7Dp9CddbB1DHD0H7kFNfb5V1LAe8/bwru/eMO3PPEFlw2a2zMuSj25Bfg/Y1qRj7grImePDJgza5DuO+57Vh83hRpo33etFHIOIQXth/QZugRetoh4h9z8abMfxJA3qPPOOL/nS7MMvShR6ow72+cgu8/04xfNrVg96ETmDyy2LMK9PQv7+zA8k37MHfyCEyqi8dkg4TsutbOsgz9oeNdeQVMUmOvIPxy5sRa22NHgc+8fRYeX7cHX3xsHZZ97jLlR/ZAVUFEmDm+Br9fv0daecMYKzr/zpxQi9PGDMOyDXu1G/r8OEGhGD0vdFNYb9qY4dh3+BSGZl00jPLO8V0Hj+OMCbWRbZDfJkp1xsWn33Y6vvDoOizf1FZUUHaiyzPyz2/fj2/dtBA3lZHPOnNiLR57tRWdx7sxtMrF3z6yFuNrh+CL75otvY9hVRksmjoSL2wXV8CrUjjuQYxeYVtWiO0H/6sgdFPJORYMM/Tea/R6nDpmGC45fQweXrkLXb05XB553FzQUIehWRd/WL8Xq14/iDs4WmTAS85VuQ427D4snew7dqoHG3YXGnutbTmE1yONvS6YMdo36nWYI9nYyyJmSNbF198zHx/4jxX47vKtuPMaecMAFM4jh4BZ42vw4PFuHDjWhbE11ckbwnP2wg9aRISr5k7Efz6/E4dPdmPEkNL7+4jG6RkO7/ccZw6SsHc+Y9xwrNjZkY/RA57yJmbo8/F//ne/99wG/PDZ7bjnya1421leUeGJrl7c+tOVeHHHAfzLTQvLFi2c6Y9py74j+FPzfmzZdwRLPtqofAwvmjkW3396m7bjHz3uvHCZcFt42xBRfjtHIpFeLkZaFJ7ntfi8qfj0g68CQExBkfX19EvX7AYAYXvZrOvgrEm1QolltLHX2pZDaG47WtTYa0HDSK+ytGEk5tXXSVfUWdS4eOZYvL+xAT/+3524bsFkpaKlsGE8fbz35NbcdlTO0EdjN/DUN/c9twPPbmnHuyOFeeUQjNOhggyyNyEZS0SY7sfph2SdvKHnKW9ETlNA1nXw2bfPwl//cg2WbdiLy88cnzfy337fQrznnPKVaUG9ymOvtuKXTbvwnkX1eNtZ6q2fL5oxBt9bvg0rd3ZoaR1dUPdBORkLX3XjUMGwJyXSdWGUoQ/HVqNcOXcCRg3L4uDxbtRzpHIXzhiD/922H+NqqzE/wSjMnVyH3631uuI1tx/F2l1eonRtSyc27y009hrjN/a6Zt4kLJxSh/n1I2OFLJbK8sV3zsbTm9tx56Pr8NinLpauaA6HOmZN8Az9trajUg3KGOLn3zlTR2FsTTWWbdir19CzgsnJh254bmHoupjmN2gbmnUxengVhle5iYY+qZ/hDYvq8YNnm71q2Rdfx4qdB3DP+xfixkV65MeT6oagdkgGD778BsbWVONL180paT+Lpo5EdcbBC9sP6DH0+aphrzBTzc4z39AXjqv16BWJlm2Hqc64eM85DfjJn3aiflTc0Ad6+redOT5RUTO/vg4PvvwG5t21LN8cqrY6g/kNdbj10hn5bo191djLImbksCrc9e45uOOBV/GLFW8UdUhMIpzknDhiCGqqM2iWnAgjqroBvAv5HXMmYOnqVq3l+MUhBLGxCF8XM8YFHr0LIsKU0cO4RVNJ11KA6xA+d8UZ+MyDr2J7+1Hc8/6zccOi0usXohARzppYi5WvHcTXbphb8mxiQ7IuGqeN0hanD46x44duVDzxHCsOtQGFZKyN0UuSpBQAgL+4dDr2Hz3F7R+9cMpIvO/chlRj8PbZ43HF5vGYMnpYvgCpvxt7WcS8a/4k/NPIzXjljYPShj5cukJEGFdbjY7j3XJbMr4PvKDBcxA6jnVxE/2lEE4KBg8r3NCN/xp49B+4YGq+LmDUsCocPiGe+D7trL52/iSseq0DF80cg6vnTVL7AyT40IVeUVS5+z59XA3+p3W3ljEVtVpR1dH7yfqwvUhKpOvCLEMvUN0E1I8cin9dvIj7WdZ18C0J3fWEEUO0tGq19A1EhKFVLnp61aoXg20BNa8teDSPkqRzL5XwOCm/LEF1Q4SM6+DrfruIYFzdvfEMrqhgKorjEL5y/Tz1wUtSToVzmLBuvWxCoTAiNR19kMIpCt0kPI3pQmYqwSUArgXQxhiL/UeJ6P8C+GBof7MBjPNnl3oNwBEAvQB6GGONugbOI82jt7w5yfptYWUJJzm9V3kjwRhfqZKppKFHIaHH23/07wkj0oGnqW4GGxmHuE87pRC2Mw6pVbQGLRDCx7UvPHqZ7NT9AK4WfcgY+xZj7GzG2Nnw5oP9Y2S6wLf6n1fUyHtj8V5tbNwSJuvyvVYRUQ25V/0ob+h5T5QV8ej9V4cocf8swQNyBB5pmupmsOE4pO3Yh88PlXMDKCTrucnYCrr0qYaeMfYcANl5Xm8G8GBZIyoDUdm25c1N1nXUDH2+YAr+K0F2cwZ+kL4yoZvCOIOx8mxOOJYfRRSWKsT1zbiaXFUZZALh80M1dAM/WR92RguVsVqGx0VbuSURDYPn+f8qtJgBeIKIVhHRbSnb30ZETUTU1N7eXtIYbOjGwiPrErp7FJQRvlEPTiPXkVdE5ATJ2HxoRePjOa97ZbKOnjMugUeaC8eFDMDV6NGXrboBBmToRpbrADwfCdtcyhg7B8A1AG4nostEGzPG7mOMNTLGGseNK60nRVLZtuXNi3qMvtijd0ghvsvEBhWAUlI4/asK4ywk9BIMPee6ICJ+Na1Zdj4fotIhYYx2ryxFRx/uU9UXyVidhn4xImEbxlir/9oG4DEApc1BJolpCSSLHqpcBz08ayYgrrqRfzxnYEXx14BKeG3hceaNBTcZK9bEiz3S4Foy42KS6dcvS6hOTb0FAiu0QAgIPPxK6ui1GHoiqgPwFgC/Di0bTkS1we8ArgSwXsf3iTAtgWTRQ9Z1lEI3AcFppKKsYGkefUWSsck5gHDvnigiHbhp11Lh+Mvf8NMoyCtVpbvFqptok7NKICOvfBDA5QDGElELgC8DyAIAY+xef7UbATzBGAvPmTYBwGP+nSsD4AHG2B/0DT2OTNm25c1HplTVTZFHL6uj54dIMo7nU+lUVoRDMvmmZtxkbGG9KI6gn7ppydjCE0/5+wqfHy6pxf55OvqCoS9/bCJSDT1j7GaJde6HJ8MML9sBYGGpAysFmbJty5uPqlJj9P57FWlejtMCwduH96rXow/F6BNCQ/mbi4JHH2xjyrWUSUhWqxIcL4e8/2u3UjEeiyVjCep97VUxssm5IeemRROq8spwrxsgiJ/KbStaL/Dodcorw/Y7KRkbIJJX8iWZ4m0GI05CaEuVsLpPvQVCQZYZEOyngnbeLENvC6YsPLIZUva6gIKhVyqYAv/8y/ei0fl8HpLGFCpjE1YTFEwlF1npGGj/4wahLR2GPqS6ISKoCKmCZH1YdZNvW9yfBVODCau6sfBQL5jyCMfo5eWVjHv+uRXw6MNx9CA0pNrOQJiMNVR1oyN0U+zRq6llCjNMFZYFT2Q6ayyimGXoDVMKWPRQpWroIyFtNXllSsFUBZKx4YlHuPLKhOtCGJYyTUevsc1AUaGaaugGBf19QFDZbEM3kpimFLDowVPdqFxFQegm8OhV5ZViHb3WZGyoK2VS5W2i6kbo0SO/bxMIjo+e41847qKCM+GWfh/rYh09+eon69FLYVrZtkUPWddBb45Je9PR00glRp/zVRVRKlIw5b96eu7g+/ljAgQevcN/yjBNwaYzGVtOC4RgvoJw6MYhvb14eBhl6K2dt/DI+plQ2fBNVHUTnsg5jb4smAqPMymhl2Q/RE25TLuWMhpvtMX1C6otEIormYESm6MpYpShN61s26KHKkVDH53AxlW4mIsV+AUS53Qtkdg4hT3Xi0NRReMi4k9W4r+acinp7B5aXL+gluBlfrK+ODFOg6MFwkDBJmMtPLK+tk62oViwVn7iEYWLObiQo2Qq4NHnCT15JLUz4KtuRI3QxDeHwYhMnYEs4acdZR19sJ1T7NE7irF+Vcwy9P6rTcZawmQzqh69/0u+YErtYk4K3fRqvJqjBtwl/kxYSdeFKCxlmIw+dPzL31fYoVQO3TBO6AbqsX5VjDL0ppVtW/SQ9UXmsm0QCi0QCjp62ZCLl4wVh250GJoAxlMHqSZjBYbKVI9eTzK2cBtU70fPaYEQqHdsjF4O07wQix6yGe+MkJZYRkKAIgPK3TQlGVsJjz6v9xf05ElKrLpOWg97M6jEDF9B/YJy98qIvDLomWNj9JKYVrZt0UPJqhv/vUpTM09VEV9eiYKpqDpIJANNLph6c8wZm29BoTNGX4KOHuAXTFW6TbFZht6qbiwcAkPf1aMWugkSZo5AmcLdljG+uiVICFeie2U4xJTQzoDnAZGg9L6QkDbjWtIZugl3N1XX0cdVN4Hh1zj5WAyjDL1pZdsWPQTySlkjGyuY0tgCoTIzTHmvXoMy8Xoi1Q1XXlnJevx+ID8fgFaPXq2YDig88bnFlt62KVahoP21pt5SQDV0E9WQK2mlWd+1QAgobqcs3r9oXIkTjxhyKQVN33S3KVZNonqVsRSbSlBUz6CLVENPREuIqI2IuNMAEtHlRNRJRKv9ny+FPruaiLYQUTMRfUHnwHmYVrZt0UOgo++WDd1Ekj0qoZvUFgiVLphKamfA2YdYe1+878FO/olKo+omqEZWMdDBxDRR1c1A0NHfD+DqlHX+lzF2tv9zNwAQkQvgBwCuATAHwM1ENKecwaZhmlLAooeMqyiv5IZE9CRjK9UCARC3upXpXhk1VuYlY/0YfQUKpsqdStAZCKEbxthzADpK2Pf5AJoZYzsYY10AHgJwfQn7kca0x02LHgotEGSNdbEHLApvcLdlfEfDcfRPLhGXV/L72iRdF4WKUfltBiOO1tBZsUevHLoh4iZjB4OO/iIiWkNEvyeiuf6yegC7Quu0+Mu4ENFtRNRERE3t7e0lDcK0Ig+LHgo6ejWPPjCCpGCggxmEeLgOVWjOWD90I6pyTbguRF01o8dgsKMzdFPk0Su3F2Z+t9GIvHIQ6OhfAXAaY2whgH8D8D+l7IQxdh9jrJEx1jhu3LiSBmJl9BYeysnYqD5dQeMcFMTwEMXQSyWqpkntLc/ZR6G9ccTQC9uzDU70NjXzKL0FQuHGA5TWM0eVsg09Y+wwY+yo//vjALJENBZAK4ApoVUb/GUVw3r0Fh7qoRuPvD5d2BWSsy0TJzBFHnepRFVmIu8yXOATpTAzlWgbLUPtd3TOB1DcplhRRx9sF7a8Jah3VCnb0BPRRPLPICI639/nAQArAcwioulEVAVgMYCl5X5fEjYZa+Gh7tEXq7eUphL0VRU8dIduoq5kauiGswtH4NGH+7mYgM5eQ8GxKaUFQqC6ic0wVeFkbCZtBSJ6EMDlAMYSUQuALwPIAgBj7F4ANwH4JBH1ADgBYDHzzqweIroDwDIALoAljLENFfkrfEzzQix6yATySsUWCAFKUwlCbBpVi2vSyLHic50EiUG5ZCx/XKZcS/nKWI0efd4TV5pKsBCqKYyt8qGbVEPPGLs55fPvA/i+4LPHATxe2tDUCQ6TKQkkix5UWyAUVBXeOxUDLWpq5u3H0Z6MDX+V6wjaFIdCDVGEqhvDno511jGEW0+ImsIJt/VbZMR19PHwmU7Mqow1rGzbogflGH1MdeOFbmTOr2TVjX55ZXEXxOTiJ1ELBCA+LtP6RmltKheKHJQ08UhER1+osB3AydiBhGnaX4seCjNMldgCwX+VuQ6TPPqMdo8+Msm0w2+MlV/EM/Qp8kpTrqWKtEBACUlUFrRAKCwiqE1XWQpmGXrDyrYtenAd78JSlleG5owF5B7RvYuef/45lfDoQ9/lCnIJuYTrQhS6yUWOwWBHZ2VsPhnrUMgJkNtvkIx1Ix69oxgCUsUwQ++9muKFWPRARMi6DrpUK2MDj17JSIhVN/o9elZ0TxGGERKui8DQx1sgFB+DwY7O0E20BYLKfoNkffGcsVTxZKxZht5/NeXktOgj61DpE4/kjWH6tqIWCEBlCqbC3yWcYcp/TZJXRm9ipmW7tOro/Vei8H4lt2VBjL6wrKQQkCJmGfpIEs1iCchmnBJ09H4hUmAMJa5EL27eRwVTEc1+0KAsvp73mlgwFd0uuJZ4GdxBiNbK2JC+UlRZLNzWT9YXt0BQ74KpilmG3rCybYs+sq68oQ8IyysBuYu5LwumGCu+qbiCCt6Cgia+D0cgOzTtWnJ0Gnr/NayekZ6uwI/dFKluoFaUVwpGGfpo21aLJaDKdZTllcFpFHhfMjrnviyYin6XWF7pvYraFIfXkdlmMKJ1hq+iGD2U9uv9z6I6evUpCVUxytBHJ4ywWAKyrnyMPt4V0lsupbrJ8eeMBSrj0cd09AmqG951IaoYNVV1o+P4F1ogENIqi6MET3zFHj3ZGH0pmOKFWPShErqJ93mXV90krSGqXC0VXmVs0kTfvOsitXulIddSoXmbRtUNqT3tBduG1TrBfnTPVRDFKENvWtm2RR8Z10FXj6zW2XsNNzXzlsvIbpJaIBB6NNa5R1siC8voE66LwNMVzjBV9igHBjqbmoVSsUpPe8G2DhV3ryylwlYVswy9YWXbFn1UqYRuWPF5JGrly90W4vPPVWyAlfpdjMnF6BOui/QZpsy4lkQy0lII1xiIKouTtuWFbtQnMFHDLENvWALJog+l0E3kvd8qR151I/gs42r26FEsfxQle2WSsTE1imEFU0SkLXQW3gMJbpRJ24aTuN4+1CcwUcUoQ29aAsmiDyV5ZcQwBhezjDQv2jo4jDd5t9wQZIgVTAl0+knXBQnCUtGiMRNwSX4CmSTCT3yqLRC8cFtUR29VN0qYVrZt0YdXMCVf1AKEC6YUKmMhVt1kHEKvVo+++LscwSTmSdeFK/jbTJytTVevofATkkqf+/AEMG40dNPfqhsiWkJEbUS0XvD5B4loLRGtI6IXiGhh6LPX/OWriahJ58B5VPA4WQY5KjH6qDerFrpJ1tH3aHTpczGPnj/GpG90BH+biUJll/TIW8MqJlchdMO7QQDe/40GgEd/P4CrEz7fCeAtjLH5AL4K4L7I529ljJ3NGGssbYgK+MfJlLJtiz4yTgnyyojqRlpemaC60VowFQkTiacS9F758kp+WMrEfJeoF5Aq4e6meXmqZHsMAH4LhMLyvojRy8ww9RwRTUv4/IXQ25fgTQLeL5hWtm3RRymhm6jqRioOyxJUN7rnjOWEbvhDtKobQN+NNlxjoNbwrmCfip3Rys8ZqztGfyuA34feMwBPENEqIrotaUMiuo2Imoioqb29vaQvN9ELsegh65L0VIKxgqm81yuxLcSqG+0FU5zQjapH7wpuYibmu3Q1lStqU6wS1vNfY90r+0BHn+rRy0JEb4Vn6C8NLb6UMdZKROMBPElEmxljz/G2Z4zdBz/s09jYWNJfbFU3FhFVpcgr/dNIJUafpLqpTAuE4v3zwktJ10WhV0t839425qDLo89P5BJqgSAT1uNtBxTaFOso5hKhxaMnogUAfgzgesbYgWA5Y6zVf20D8BiA83V8nwjTyrYt+si6CpN+5B+xvRNJRV7p6eiTCqY0GnoUfxcRccNLSdeFMEYfUR6ZgO75AIpDN3KJ+vC2AYFMc0C3KSaiqQAeBfBhxtjW0PLhRFQb/A7gSgBc5Y4uTPRCLHrIug66JUM30RYIIgkiDwaxo+EVTGlW3agkYzn7EOnATbyWHE1ec1HoRrFgCihW6+TfV7gyNjV0Q0QPArgcwFgiagHwZQBZAGCM3QvgSwDGAPihf/fv8RU2EwA85i/LAHiAMfaHCvwNeUxMIFn0kHUJXaW2QAgmlpaVVwrOP91x2GiM3hXp6P1X7sQjghmSwgoRU3A11TGEn3ZU2hQHq4S7XgJ9o6OXUd3cnPL5xwF8nLN8B4CF8S0qiIEJJIseSmmBEE3Gyhpp0emX0R2jj6huiPgyv6TEquhvM1HY4OUwyt9P2KNXm6sgpLqJ/N8Ggo5+0GBi2bZFD1nXQY7Jxtm916iOPi2+nqZUcR0HvVp7IHBCNwnGIil0E58ztoLuZT+hqxVwsWfu/a7i0RMBFLG8tteNAiaWbVv0kM1454SMV1/w6L1tZCeATlN9uY6e7okB0XyAqANiLuG6ECUTjfXoNU48EsTWw8uktgPFPPrBpqPvV6KP3BZLQJWvkZQy9JFMZHBNphkJGY9er7yyWHXjzTDFW8975Xv0/NADCxklU3AdR09Ts9DvKslYkY4+iNlbQy+JiV6IRQ/ZvKEvPXSTJn9LczRcTU21AmKqG0fQ6ybhuhD2ujHwWtJ2/MMhmJJCN3GP3k4lqIBV3VhEZFyV0E2x6kY2dJOP3Qp6LWn36BGtjE2eSjC5BUI0Ri/eZrCirU1xkepGXkdfPKl4VHUzwHX0AwkTy7Ytegg8epk2CPEWCN5rmpFIS2DmOx1qMvaMsUgXRC+hJ2pnwEPY68ZAYYP+pmaqoZuCfeK3QCh7aEIMM/Teq0knp0UPSjF6/zWmukkz9CnhjuCpQpdXz4Cikz3VaHPllfC3Kd4oZ6DTpKvXTbCLsOpGdlIagOfRi/sU6cIsQ29g2bZFDyXF6H0rKi+vLN4uiqoePxUWLZjyXlXaGaQVTJl0LWnz6MOeuYLqJqwKLDqsVDjOlQrfmGXorUdvEZAtIUafb4EgG6NP6bWUcXR79PE2xUBCYpWzD+FNrJKi7n7C1aRs4YVuFEL0+eRrgEMkfY6VilmG3n81KYFk0UM2oyKvLH4vL6/01xd8HhhiXY/ouVw8GQskzBalELqJavRNQJeOPr8HEh8/7nYh1U2YIHQju59SMMvQGygJs+gh66iEbviqG1l5pcjRyGg29AzFydgg2SuaLUpp4hFmnsOkqwVCuLspCY4fdzPBxEheKEdzWC+CUYa+kgUHlsGNUuhGoKNPVd2kFkwFoRs9jcej/egLmu7IegkzrwnDPQkTqAxWdE38En5CynviChVT0fNDNQRUCkYZ+gDDHBGLBoLQjUwHy/yF7L/KSujSPs/HYTVNMBH9usL+owoa/5eE0E30aSVpApXBiqNLdZMrPPGptUDwXqPJ+qIbhvXo0zGxbNuih7y8UkVHT4Hqxnuf6rUJYrABlfHoQ6EbkdFJuC5E0yR6LZDNuo5cR0/YLHzfLFVHHybc+8YmYyWwMXqLiEBeKaN4iYY6ZL22tMnpCwVTqUOQpDi8QoIQU1IyVlTCz6LaTQMQTbWoSlEXylKSsZHl5KjtpxTMMvT+q2lJJEv5qMToozNMFbxeOdWNoANCqGBKj6XnzTAF8BqUBeOKD0w0OTiY+O8YrHhN33R69GotEET2qShGX6F5Y6UMPREtIaI2IuJOBUge3yOiZiJaS0TnhD67hYi2+T+36Bo4D6ujt4hQaYFQmMCG/NeixeLN/NekGaYAfV5bvAUCuPsvhDSTxhTZN0wM3ejy6AuGRil0IzBQ4ZmqdLaxDiPr0d8P4OqEz68BMMv/uQ3AvwMAEY2GN/XgBfAmBv8yEY0qdbBpmFi2bdGDUmUsol0hJUM3Keef/oKpeD96gFcZC+G4RE8rjDHjriNdLRACHBJXI/MQhm6gVmFbClKGnjH2HICOhFWuB/BfzOMlACOJaBKAqwA8yRjrYIwdBPAkkm8YZZHmUVnevKjKK3mFSGneVtoMZ7oLpqLjFE1injQhSjDTUbzXjXlPxrrkleGJXFT078KCKdL/tBdFV4y+HsCu0PsWf5loeQwiuo2Imoioqb29vbRRWB29RYBSZWy0tYDk4zkTCaV99BdMFX+XaBJzUcgAEN8coooeE9CejIVqCwR+CC08WbjxOnrG2H2MsUbGWOO4ceNK2wfMSyBZ9FClELqJerPK8krBx67u0A1j/CcPwf6TQjdvhoIpfU3NPFT174X5CjhjGySqm1YAU0LvG/xlouUVwUQvxKKHrNJUgsXKCHl5pYdI9SUqaCoVFlHGpM3/yhtXvo8PZxvTLiVdMfpwl9KSphLkFkwNDh39UgAf8dU3FwLoZIztAbAMwJVENMpPwl7pL6sIOWaeF2LRg+t4rWGlu1eGTiRK8ZTz26XUcWj36CMhpmD/cR19uuqGdw8zzWnS19SskHTP69+lkrHigimV/ZRCRmYlInoQwOUAxhJRCzwlTRYAGGP3AngcwDsBNAM4DuBj/mcdRPRVACv9Xd3NGEtK6pZFVIVgsYTJuo5UC4R4n3e5+Gk+SSf4XP8MU4LulZE/MVoXUDQmYdsE81Q3umZxCt/QS2mBECXs0VcqRi9l6BljN6d8zgDcLvhsCYAl6kNTx8SybYs+qlwH3T3q8kr5qQQ9hPJK3TNMMf44xf3o4wMrbMPZt5ZRDhy0tUAItZRQC7kU12cEePJK+PsZ2DH6AYGJZdsWfWRdkozRsyKjKD+VYMEA8JCVacriJUw5oRvhDFPxfYjbJjADQzfOgGyBEFbdWEMvA7OqG4uYrOtItR+IthZQnUpQ5GxkfLetV0tTdN+L5I1T4NGLcIifwDXtWnIdPWGz8L9ZSwsEKtxwB3oydkDgnfeGnZ0WbWRdB10yoRuh6kbue0SqG5HOvWSiqpuUeHGSGog/K5VZ15JLpCVsFi58ku2DVLxd8fJwCwTr0UuQy5mXQLLoQzp0g6g+3XstdyrBvEevU3UTDt2I+taktGYgTpLSRHmlo0neGoTCHBIf86TtktRP1tBLEHmStViKyLqO/AxTEXklccIbUdJ6Lan0RZFBlIyN7j+1NQPFjV+0GMsEXEE+QpVcyKMXtZDgbpcLtot/JlJM6cIsQ28LpiwJyBp6gJ8wkyyMTTD0uj16flOzuNEOxiVOEvPi+qZdStp6DYWOVSktEHi3XBu6UcDEsm2LPrIZB12Sk4NHjaJDCnPGCs5C7d0rheqgyHoSE6LwtjEt36WieU8ifIMtpQVCkkdvfK8bHZjohVj0UeUSeiQnHokqTnheb5Q0j15XjDggNvGIsKlZ8riI+Eod41Q3ConTJMI1BqW0LuAlxa2OXhEburGIkI7RczTkMrMTpYVIKtOPPl3vX7gBidRA8b8tum8T0DU5e/j8UEmiJiXrVdodl4JRht7Esm2LPrwWCHIXZPQ08iSIadslh0h0F0xB0L2Sm1hNuC54+YdKGZz+RNQLSJXwE590Z1MkJ+sHS1OzAYGJZdsWfWRdQrfEVILRJCf896nyytC6PPL96CUTwmlExymsjE25LhxO6AYGhkGd/BNVecc/3GqlpO6VXEMf7Nt69KmYWLZt0YeSvDJiGl2HUi/CpJ4yQEj1oc+hL/qmQjl+ZL2U64Ln0fNudoMd0eTpqoRbrai1QBAn661Hr4CJCSSLPuTllfFQh0PpsxMl9ZQBwjNM6fLo+W2KuVLJhP3w8g9RRY8JiJLVyoSOp2yNhb+Zv038s2CZzjltw5hl6AHY4I1FhGfo5Qpb+Kqb5O0KE3zwPy+EVlKHIEV0nIUqzXhiVdT+AOCHbhjMc5pkexalEQuZSTgBQHKyXqVnTimYZehtMtaSQFVGpQVCXEcv3dRM4Gy42j364u8STZASrfSNIm6BYNbFJMphqMIYK7pxyve5F7uiqv2UVDHM0Ft/3iJGJUYftXG8xl9RUlsgUAUKpjjJ2HjoJrmQkJd/MHG2Np2qm2huRGXikaRkbL/KK4noaiLaQkTNRPQFzuffIaLV/s9WIjoU+qw39NlSjWOPYQumLElkHLnQDc8BdoikQy5CeaU/naHO6eKK5ZXea/SBIS2xyqv65eSjBz265uyNPu04RHItEBKS9ZXW0afOMEVELoAfAHgHgBYAK4loKWNsY7AOY+z/hNb/NIBFoV2cYIydrW3ECZhYtm3RRzZDUlMJ8sIWjpMeP00rmAK8hGzlZpjie6xpiVVu6MHAp2NdT1S87qblzBnr7SOI0Zc1NCEyHv35AJoZYzsYY10AHgJwfcL6NwN4UMfgVLGqG0sSVdKhG77qJr0FAvPXFa8jo96RJcf4M0zxJhFJ8ui5LRDAEhO4gxFdTc2iOQ/Z/6mMjr4/Qzf1AHaF3rf4y2IQ0WkApgN4OrR4CBE1EdFLRHSD6EuI6DZ/vab29naJYcUxsWzboo+s64AxucInvrwyef9pPWUAz6PXNcMUQ6FHChDy6DmhmySjzYvRmxgGFamSSqHIo3fKD90MNh39YgCPMMZ6Q8tOY4w1AvgAgO8S0Uzehoyx+xhjjYyxxnHjxpX05SaWbVv0kfWF1GlePS/Uwa0ejW7nvyaGSRx9Hn2se2VCU7M0HT2/mtYsS69VdRN6bJM5N4DkOguVwqtSkDH0rQCmhN43+Mt4LEYkbMMYa/VfdwB4FsXxe70Y6IVY9JF1vZMjLU4v8ujT4rC5gssmJOPEjWqpRBOmIi12LsXS8+SVJvaNSptqUZao6kYmrAckNzUbCDr6lQBmEdF0IqqCZ8xj6hkiOgvAKAAvhpaNIqJq//exAC4BsDG6rS7S1AWWNzd5jz6l3424qZlkMjZhHVejoY8mTJPa8CZ79Jy4vobhDTRcQWhLlWhFMu9Gyd1OomCqUqGbVNUNY6yHiO4AsAyAC2AJY2wDEd0NoIkxFhj9xQAeYsVnzGwAPyKiHLybyjfCah3dmFi2bdFHIXQjE6MvPo9ISl4ZPJonx8P1zjAVDt3wjQVvIpXomGzBlDxRR4B3o+RulxC6kZ2XuFRSDT0AMMYeB/B4ZNmXIu/v4mz3AoD5ZYxPCS/p1FffZhlsBKGbtBg9L2zhKsgrk85BlxMPL5VoUZOoZW7adUHcMTHjriVthh7xrqEy+0w6P2w/egVyBnohFn1UZbzTPVVLzwndyDU180h6qnRdjR59RE4sqvxM8865vW4MzHfpamrGK5hSaVPMC6QVpLFlDU2IUYbexJnrLfoIQjc9qaEbwQxTKRdh4EknyysdjTNMsZjBAeJGO62dAa+yM6rRNwFtTc0ix1O+BUJ66MZ69BJEVQgWSxh5eSXPo5doaua/piU+9ckr4+oPgB+6Sbr5uBzViInCBq0x+ojaSepfKqG6GSw6+v4lRS9seXOTkZVXcsIWKqqbZHmlo69gihV/l6gDIkvxgHizZ5nYIFDXVI7RVivqOvr4kR0IOvpBg4ll2xZ9VMnKKzk9k/gJy/h2QN8VTEW/S6zcSJ8zNjok795g1rWkt6lZ4b2jmIwdqDr6QYOJCSSLPmTllTmeRy/zeC6hutFZMJVjxcqYYLYjXmI1sf+Ow29tbJrqJiNIVqsSeZBS7l7Jc0Zt6EYBExNIFn3IyitF3SulWyAkKVx06ug5NyRelWZaOwNRZadpV5LOpmbFSXDZ0I2HTcaWifXoLUnIJmOBuEpFRl6ZNvEIoLsFQtyxcTmFXWntDHgN23IpRVaDkaTKYRWi3U1lWyAkrUPWo5enQsfIYgiBjj61MlboKSfvX6oFgsaCKe44OYVd0VBDFG4LBAOTsZUqmFJvgRD/TFTspguzDL0tmLIkEMRoU0M34Bl6BXllkpRRewuE4mXCTpSJBVOCcI9hl5KupmaxrqGSLRCQkKwX1UDowihDb2LZtkUfQegmXV4ZV2/JySvF8rnwfnp0TQ7OkU26nBATS3nWdRziTD9ocuimvP1Eb7DKLRA4VlfUp0gXRhl6E70Qiz4KoZu0XjfxsIWcvNJfN2Ed10mfwEQWnjKGiFNGz/jGJUDYAkHPMAcMon79qngqpsLRkQ7dBOtzPfpg39ajT8WqbixJSLcpBmIeg4y8Usaj95Kxmjx68Au7ojektOvizRK6yc8wVabbzGsmV34LBBu6kYZ34lssAYG8Mq3XDK9nkuOke4IyyVhPXpkyUEl4bbl5Iaa064KXaOYpegY7Gd+lL39ycMQmfFHR0dsWCGVi4uOmRR+yMXpATp8eJUlVEVBpj56EOnoxDu/mYKBHH4Ruyla2RI6nDh39gGiBQERXE9EWImomoi9wPv8oEbUT0Wr/5+Ohz24hom3+zy06Bx+F98htsQQUQjcS8srIMpmpBJNisPn96C6YiizjyTe9G0JS6Ea9EdpgRNTGWRVe11C5ZKw4i1NogVDW0ISkTjxCRC6AHwB4B4AWACuJaClnpqiHGWN3RLYdDeDLABrhnTur/G0Pahl9BBPLti36cB2CQzLySpHqJnn/LCEGG6C1YIpT1OR5l7z1xPvhhm44yqPBTr6pmfYZptT+p1EbFbZb/amjPx9AM2NsB2OsC8BDAK6X3P9VAJ5kjHX4xv1JAFeXNtR0bOjGkkbWddJVNzleSESlBYJ4HZ78sVT4BVPxJ4+0gilufxwtIxxY6GxqFr4JyrTHCLYD4k9X4f31Z4y+HsCu0PsWf1mU9xLRWiJ6hIimKG4LIrqNiJqIqKm9vV1iWHFM1P5a9FLlOuk6ekFrgdTQTeDRJ80w5ZC+NsWc7+ImY1PaGfAURSbO1pbX0Zd5o422lFBtgRA9quH9DXTVzW8ATGOMLYDntf9UdQeMsfsYY42MscZx48aVNAjr0VvSyGac9BmmOC6wUguEtMpYbR59PCTD61uTmozlhR44yqPBjqPLo4+8L7cFQnBTlXlqLBUZQ98KYErofYO/LA9j7ABj7JT/9scAzpXdVicmKgUsesm6JNcCIbJMSl7pv/ZpC4TIMm5iNeW64LYphpnXUkbDjTbaUsJVVd1E/mvBtrJPBqUgY+hXAphFRNOJqArAYgBLwysQ0aTQ23cD2OT/vgzAlUQ0iohGAbjSX1YRbOjGkkbGSQ/dgBf7JkqtWizopJMLprTNGSsYZ9xoJ2vieR6pqU/HjpbjHy2YkjPQomR9sKkr+WRQCqmqG8ZYDxHdAc9AuwCWMMY2ENHdAJoYY0sBfIaI3g2gB0AHgI/623YQ0Vfh3SwA4G7GWEcF/g5/rGaenBZ9VGUciYlHGDJU7APJKCuCnjLJk3zonngkHqPnNzVLGBPxOl6ap7oB5HItaUSPJ1G8VxB3u/z6xct7QzeASnn0qYYeABhjjwN4PLLsS6Hf7wRwp2DbJQCWlDFGaRhL7ulhsWRdkmqBwGstoCNGr7cfPaRyCWk6em4jNEPDoK6GymSGiOpG1kALVDfh0E2ldPRGmUUTy7YtepGRV/JaCxAn9h2lcLH3TcGUV6EZ0dELpgVMllfGvdwc7y5iANJGOQGe6kZmlyLVDcuFxmb70adjqhdi0UfWddAtUeEa8+gVEmV959HHVTfcytjU0A1ncvCUIqvBio5keDRELNPCGhCHboqTsWUNTYhZhh7W0FuSqXKd9NAN52JzJNQaMk3NtBdMRZZxe90gPUb/ZpgzFtAjb/UedgpHh0hOmy9K1oe7Wg50Hf2AwMSybYteMpLyyuh5JKWjR8EzE+E6DhjT84guziXEQzfJY+LH6E28lmR6FqURDYVJd68UJOuD4ThOurKrVIwy9JV67LGYg3SMPpbklJhKUKpgynvV4dVzVTccBQhPbx+GK6/khIVMQJe8NWyspbtXClI4NnSjSJq6wGLJug66ZCYHjyyTicPmBI/mxfvxLjkdcXp+6CZ+E8mlxG548sqcofkuXi8gVXIs3r1SSUcvLJjS4wDwMMrQm1i2bdFLVUYmdBMvvJMpc5fpXhl49LqKpmIzYfGamqVcF6LulSYq2LTE6CM32HJ19HkHQaIor1SMMvQ2GWtJI+s66EkN3fD7vAPJ4RuZS1SXR1/wDosReZfJLRD4CVwD7TxXlaRKVMXkqnavjCwPzile2E0XZhl6QxNIFn14MXqJ0A0nRg+kXNCSBVOAnp7ovO9yOJOPpzc189Yp8iZTthms8G5qqkTrddRbIFBkebAfq7qRgsGGbizJZF2SaFMcvxiDzodJj/0yqhtHl6EP9hdLxqq3M+D1Qucpj0xAl0cfNjTS3Sv917jqpnADsMlYCXgTRlgsYeQrY4uRmepNtgUCUL6hF1VZ8nryyPS6Ce/T28ZM1Y2Oydm9m2Bon5ybK3c7QbI+cB4cR24/pWCUoeef+hZLgaxkwZQodJNkoKVUN5omv0gM3cTaGSQnVvN92kNjyhkaunEdlD05ezRRLSuLzBUsfWx/hf1YQ5+KqV6IRR9SMXreDFMcY8jbDkjvRw+g7FmmCt8VLeyKP3VEQw1R8qGbkP0zteW36zixHIYq8WSsWjhIpLqxOnoFzDs1LTqpcgnduVziIzLPoyeOMeRtB6S0QJCI9csg2pxb5ZoyJn7oxsxryZUofEsjqu6TbV0gVN3YFghqWNWNJY2s67UgSPLAeDJdV0J1k/9ExqPXpKOTmXiEN5FKdBsgbuhNtPR6mprFQzcqLRBibYrz3Sttm2Ipou1DLZYoGb9iKSl8w5tMW0Z1g1CsVUTB0EsNN+2r+D15osnYElQ3vH2bgKOhqVzUEVBtgSBS3fS7vJKIriaiLUTUTERf4Hz+eSLaSERriWg5EZ0W+qyXiFb7P0uj2+rEFkxZ0sj6rnmSxJIXtuB5vbHt/FeZ0E1PmR69SHUjalAmpbrJhT16M6XKvMphVbzjWYKO3n+N5n/CN21tcxVESJ1hiohcAD8A8A4ALQBWEtFSxtjG0GqvAmhkjB0nok8C+GcAf+5/doIxdrbeYfMxtWzboo+qTODRJxh68JKc6TH6wICkzeYE6NPRx3MJ8TGWrLox8FLS0wIh0r1SYvYxoDgWH6Y39CTYn8nY8wE0M8Z2MMa6ADwE4PrwCoyxZxhjx/23LwFo0DtMORhgZFzRoo+sK2HoOd5s0KOmbI/e1VUZG3j00YIp9X70xAndmDpbmw6vmRu6kdin6NTJh276WUdfD2BX6H2Lv0zErQB+H3o/hIiaiOglIrpBtBER3eav19Te3i4xLA6GKgUs+ggMfU9SjB4JqpsUtQ4424bR1gIhP67i5fx+9Mn7Cp4ywkYmLdwzWNExw1c0tKeqf48e177Q0UtNDi4LEX0IQCOAt4QWn8YYayWiGQCeJqJ1jLHt0W0ZY/cBuA8AGhsbS/prGcxMIFn0IRuj5/V5B1Lklf5rn4Ru8jeV4u8iild+pl0X+WKwsKHn7NsEdMzZG60xkG6BIEjWh7tX9mfophXAlND7Bn9ZEUR0BYC/B/BuxtipYDljrNV/3QHgWQCLyhhvIlZ1Y0lDKnTD6ZnkyIRupNoU6zH0+QZqsf1zHv9TrgturxtDPXqVuX9FeI5A4X3we1rYJa17ZX+rblYCmEVE04moCsBiAEXqGSJaBOBH8Ix8W2j5KCKq9n8fC+ASAOEkrlZMLfKw6CNv6HvEF1Quh9iJFBjDxKZmfVgwJUrs8eSDae0Mgn28WVQ35fcZAsJH1OXcKMXbcXT0IdVNpXT0qaEbxlgPEd0BYBkAF8ASxtgGIrobQBNjbCmAbwGoAfBL/494gzH2bgCzAfyIiHLwbirfiKh1tGJq2bZFHzKhGyCe5Cw0NUuK7UuobvLySk0x+shyblOzlOsiGFOkS7GRHr2sQiaJaKuVcEdSNyqSD28n6MbVFzp6qRg9Y+xxAI9Hln0p9PsVgu1eADC/nAGqYD16SxpVsqobkUevqQWCjgmqgfhNxXXiXmHadcF7WjFVquySnmkcw0cm/0QkG7oRFEyRbWomR7SQwWKJks1Iqm4iy5TklRIxem0efeyGFDdkafF2nqEy2aPXPcOUTAtrQJysL4Ru0sM/pWKUoQfMPDkt+pDT0ceVEUSFx/Ok7QDZFgiVUd2IpgVMcoB4YSlT+0ZpmXgk0lJCavYxQJgUL4Ruyq/aFWGUoc8ZmkCy6EMmRs9Tb7kSXhtD+kWqbypBfryXp8VOS6wWWjDH928aGVfT5OAcjz41dAN+CM32o1fEVEmYRR9y8krehB7eq5TqJjF0o2lycPC/i+exyva66c0Ve/QmXks6vOZoSwmZ9hj57TgHNd+9UkOiWIRZht7Qsm2LPmRDN1HfS6qpmaAtQRjtBVPRcXKMRdp1wav6ZZx9m4CWXjdA0ekhG7oRJcULvW76twXCoMFUL8SijyB0k6SjB8SqmyRvUMqj19XrRjCbFb8TpVw/+iJ5paHFh1o6REYMNq8pnGAz7jEtDt2UNzQRZhl6WNWNJZm8vDLhOZvnefHi2LHt/NfEeLhE4ZUMIimnK/LOE8NJ4G9T1ggHJlraFKPYzvCawnG3Y/ynpGLVjfXoUzHVC7Hoo1AZmxyjj6tuvFeZpmYyqhtd8srYxCOcylveRCpheIoixgr7MgldbYpLaoHAeVIEojr6soYmxDBDb6YXYtFHNt+PXnxFJalukrxBUVuCov1oKpjKby8MMRWWyRZMhYdkqoJNR+gm2lJCtgWCKIQW9uhtjF4CU4s8LPoI5I3KM0yphG76wKMPkCnsStfR++tFtjHR0mtpUxxpKSHTBwkQVxtbeaUippZtW/QhPfFINCQiczFLXKS6JgcXFkwJ2xmI4c4ZK4gnD3YCVVI5nnPUEeA1hRNul1YwZUM36ViP3pKG6xBch1J19FFkJHQy51+hYCploCmIGmTlFTTh0E3KuBxejF4QTx7syIZZkmCRTLVKCwSuvNL/X5FNxsphatm2RS9ZlxJj9OCcR7LyyrTzr2BU9Xj0TuQK5k4ikjIubuiGFfdcN4UgtFVu+KaoBYJEHyRA/H+wLRAUqdTd0GIWWddRroyVk1emJzD1efQesTljOS0W0sbFyz+YWjAlq3lPIjY5uHQLBH5WPLjBurYyVhJbMGWRoCrF0PMUJ8F5leQJ5iTOP8chEJXv0QsnHsn3li+MkzeRStE2nLCUqbO16ahMjv6fpQ29QP1UmJDEhm6kMNULsegl41JiZSwvaeZyDCh3O4nzzyU9jbV4iJLGSePiJ3CNFN1oUT1FW0pwk9m87US9bkKhm0oFJaQMPRFdTURbiKiZiL7A+byaiB72P19BRNNCn93pL99CRFdpHHsMWzBlkSE9dFOa6kb0aB7FdUiDvLJQZFO0b47RSbsuhLNnGXgxyeRa0oh3r/ReS22BMCAqY4nIBfADANcAmAPgZiKaE1ntVgAHGWOnA/gOgG/6286BN8fsXABXA/ihv7+KwGBmAsmilyrXUdfRy3htkl6wljJ8fr0Uf/5XznphokVWohbIJpBx02/YaUQNNkWOn3C7yPkR7UtUSR29zFSC5wNoZoztAAAiegjA9Sie5Pt6AHf5vz8C4Pvk/fXXA3iIMXYKwE4iavb396Ke4RcjagNqsYTJug6e3dKOd9zzR+7np3pynIpT7/Uff7cR/7Z8G3e79qOnpJxg1yH8clULnt3SrjLs+BghDjF9+Ccr8jUDbUeSxxV89g+/Xo9v/mGz1ExZg5Xgpvb+e19MnN81iY5jXUBR6MZ7/cTPmzAkI/Zj9x0+iYwb9637ogWCjKGvB7Ar9L4FwAWidfzJxDsBjPGXvxTZtp73JUR0G4DbAGDq1KkyY49x9dyJmD2ptqRtLW8ebr10Op7d2ib8/IyJtXjX/ElFy6aOHoYPXTjVv8j5zJpQg7mT61K///a3no61LYekxyui8bRROH/66KJlF80cgxsX1eNUT29+2RkTanHjogbhfk4fX4Obz5+CzhPd+WVnTazFVXMnlj3GgcZbzhiH68+enBi6S8M7ngUzdu5po/Cec+pxsrs3YSvv/Fg0ZVT+/Z3XzMa61k685YxxAIAFDXUVa4FAaTsmopsAXM0Y+7j//sMALmCM3RFaZ72/Tov/fju8m8FdAF5ijP3cX/4TAL9njD2S9J2NjY2sqamp5D/KYrFY3mwQ0SrGWCPvM5lkbCuAKaH3Df4y7jpElAFQB+CA5LYWi8ViqSAyhn4lgFlENJ2IquAlV5dG1lkK4Bb/95sAPM28R4WlABb7qpzpAGYBeFnP0C0Wi8UiQ2qM3o+53wFgGQAXwBLG2AYiuhtAE2NsKYCfAPiZn2ztgHczgL/ef8NL3PYAuJ0xlhzIslgsFotWUmP0/YGN0VssFosa5cboLRaLxTKIsYbeYrFYDMcaeovFYjEca+gtFovFcAZkMpaI2gG8DmAsgP39PBwedlxq2HGpMRDHNRDHBNhxhTmNMTaO98GANPQBRNQkyiL3J3ZcathxqTEQxzUQxwTYccliQzcWi8ViONbQWywWi+EMdEN/X38PQIAdlxp2XGoMxHENxDEBdlxSDOgYvcVisVjKZ6B79BaLxWIpE2voLRaLxXAGrKFPm5C8vyCi14hoHRGtJqJ+67xGREuIqM2f9CVYNpqIniSibf7rqKR99OG47iKiVv+YrSaid/bxmKYQ0TNEtJGINhDRZ/3l/Xq8EsbV38drCBG9TERr/HF9xV8+nYhW+Nfkw37b8oEwrvuJaGfoeJ3dl+Pyx+AS0atE9Fv/fb8eqxiMsQH3A68d8nYAMwBUAVgDYE5/j8sf22sAxg6AcVwG4BwA60PL/hnAF/zfvwDgmwNkXHcB+Jt+PFaTAJzj/14LYCu8ie779XgljKu/jxcBqPF/zwJYAeBCAP8NYLG//F4Anxwg47ofwE39dbz88XwewAMAfuu/79djFf0ZqB59fkJyxlgXgGBCcosPY+w5eL3/w1wP4Kf+7z8FcENfjgkQjqtfYYztYYy94v9+BMAmeHMX9+vxShhXv8I8jvpvs/4PA/A2AME0oP1xvETj6leIqAHAuwD82H9P6OdjFWWgGnrehOT9fgH4MABPENEqf0LzgcQExtge//e9ACb052Ai3EFEa/3QTp+HlAKIaBqARfC8wQFzvCLjAvr5ePmhiNUA2gA8Ce8J+xBjrMdfpV+uyei4GGPB8fpH/3h9h4iq+3hY3wXwtwCCGcfHYAAcqzAD1dAPZC5ljJ0D4BoAtxPRZf09IB7Me2bsd2/H598BzARwNoA9AL7dH4MgohoAvwLwOcbY4fBn/Xm8OOPq9+PFGOtljJ0Nb57n8wGc1ddj4BEdFxHNA3AnvPGdB2A0gL/rq/EQ0bUA2hhjq/rqO0thoBr6ATupOGOs1X9tA/AYvItgoLCPiCYBgP/a1s/jAQAwxvb5F2gOwH+gH44ZEWXhGdNfMMYe9Rf3+/HijWsgHK8AxtghAM8AuAjASCIKph/t12syNK6r/RAYY4ydAvCf6NvjdQmAdxPRa/BCzG8D8K8YQMcKGLiGXmZC8j6HiIYTUW3wO4ArAaxP3qpPCU/SfguAX/fjWPIExtTnRvTxMfNjpj8BsIkxdk/oo349XqJxDYDjNY6IRvq/DwXwDnj5g2cA3OSv1h/HizeuzaGbNcGLhffZ8WKM3ckYa2CMTYNnp55mjH0Q/XysYvRnJjjpB8A74akQtgP4+/4ejz+mGfAUQGsAbOjPcQF4EN5jfTe8GOCt8GKDywFsA/AUgNEDZFw/A7AOwFp4xnVSH4/pUnhhmbUAVvs/7+zv45Uwrv4+XgsAvOp//3oAX/KXzwDwMoBmAL8EUD1AxvW0f7zWA/g5fGVOX/8AuBwF1U2/Hqvoj22BYLFYLIYzUEM3FovFYtGENfQWi8ViONbQWywWi+FYQ2+xWCyGYw29xWKxGI419BaLxWI41tBbLBaL4fx/clKkyxWVP30AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# viseme_list = visemes_sing.viseme_list[-1]\n",
    "# viseme_ctrl_pts = visemes_sing.viseme_ctrl_pts[-1]\n",
    "larynx_movements_ct_pts = []\n",
    "# extended_vowel_intervals = []\n",
    "# extended_vowel_list = []\n",
    "in_between = False\n",
    "pitch_change_intervals = []\n",
    "for i in range(0, len(extended_vowel_list)):\n",
    "    viseme = extended_vowel_list[i]\n",
    "    if len(viseme) > 13 and viseme[-13:] == \"_pitch_change\":\n",
    "        viseme = viseme[:-13]\n",
    "        in_between = True\n",
    "    else:\n",
    "        in_between = False\n",
    "        \n",
    "    if not in_between:\n",
    "        if len(pitch_change_intervals) == 0:\n",
    "            curve = extended_vowel_intervals[i]\n",
    "            val = 2\n",
    "            throat_curve = generate_basic_viseme_curve(curve[0], curve[1], val, sustain=0.75, decay = 0.95, onset=0.12, offset=0.12)\n",
    "            larynx_movements_ct_pts.append(throat_curve)\n",
    "            \n",
    "        else:\n",
    "            throat_curve = []\n",
    "            # add the onset\n",
    "            throat_curve.append([pitch_change_intervals[0][0]-0.12, 0])\n",
    "            for m in range(0, len(pitch_change_intervals)):\n",
    "                val = 2\n",
    "                throat_curve.append([pitch_change_intervals[m][0], val])\n",
    "                throat_curve.append([(pitch_change_intervals[m][1] - \n",
    "                                      pitch_change_intervals[m][0]) * 0.8 + pitch_change_intervals[m][0], val*0.9])\n",
    "            throat_curve.append([pitch_change_intervals[-1][1]+0.12, 0])\n",
    "            pitch_change_intervals = []\n",
    "            in_between = False\n",
    "            larynx_movements_ct_pts.append(throat_curve)\n",
    "                \n",
    "    else:\n",
    "        curve = extended_vowel_intervals[i]\n",
    "        pitch_change_intervals.append(curve)\n",
    "        \n",
    "        \n",
    "\n",
    "x = []\n",
    "y = []\n",
    "larynx_movements_ct_pts.sort(key=lambda x:x[0])\n",
    "for i in range(0, len(larynx_movements_ct_pts)):\n",
    "    for k in range(0, len(larynx_movements_ct_pts[i])):\n",
    "        x.append(larynx_movements_ct_pts[i][k][0])\n",
    "        y.append(larynx_movements_ct_pts[i][k][1])\n",
    "plt.plot(np.array(x), np.array(y))\n",
    "plt.show()\n",
    "\n",
    "## remove overlap artifacts\n",
    "i = 0  \n",
    "while i < len(larynx_movements_ct_pts):\n",
    "    increment = 1\n",
    "    prev_viseme = viseme_list_final[i]\n",
    "    # if the previous instance of the current viseme is not -1\n",
    "    if i >= 1:\n",
    "        current_interval = copy.deepcopy(larynx_movements_ct_pts[i])\n",
    "        prev_interval = copy.deepcopy(larynx_movements_ct_pts[i-1])\n",
    "        # For the same vowel/consonant if the two intervals do not overlap but the onset and off set overlap \n",
    "        if (current_interval[0][0] <= prev_interval[-1][0]):\n",
    "            # if we have weird overlap where the boundary of the previous interval interupts the curve of the current\n",
    "            if (current_interval[1][0] <= prev_interval[-1][0] and current_interval[0][0] <= prev_interval[-2][0]):\n",
    "                larynx_movements_ct_pts[i-1] = copy.deepcopy(prev_interval[:-1])\n",
    "                current_interval[0][1] = min(current_interval[1][1], prev_interval[-2][1]) * 0.95\n",
    "                current_interval[0][0] = (current_interval[0][0] + prev_interval[-2][0]) / 2\n",
    "                larynx_movements_ct_pts[i] = current_interval\n",
    "            elif (current_interval[1][0] <= prev_interval[-1][0]):\n",
    "                larynx_movements_ct_pts[i-1] = copy.deepcopy(prev_interval[:-1])\n",
    "                current_interval[0][1] = min(current_interval[1][1], prev_interval[-2][1]) * 0.95\n",
    "                current_interval[0][0] = (current_interval[0][0] + prev_interval[-2][0]) / 2\n",
    "                larynx_movements_ct_pts[i] = current_interval\n",
    "            elif (current_interval[0][0] <= prev_interval[-2][0]):\n",
    "                larynx_movements_ct_pts[i-1][-1][1] = min(current_interval[1][1], prev_interval[-2][1]) * 0.95\n",
    "                larynx_movements_ct_pts[i-1][-1][0] = (current_interval[0][0] + prev_interval[-2][0]) / 2\n",
    "                larynx_movements_ct_pts[i] = copy.deepcopy(current_interval[1:])\n",
    "            else:\n",
    "                larynx_movements_ct_pts[i-1] = copy.deepcopy(prev_interval[:-1])\n",
    "                current_interval[0][0] = (current_interval[0][0] + prev_interval[-2][0]) / 2.0\n",
    "                current_interval[0][1] = min(current_interval[1][1], prev_interval[-2][1]) * 0.95\n",
    "                larynx_movements_ct_pts[i] = current_interval\n",
    "        # if the two interval overlaps\n",
    "        elif (current_interval[1][0] <= prev_interval[-2][0]):\n",
    "            interval = prev_interval[0:-2] + current_interval[1:]\n",
    "            larynx_movements_ct_pts[i-1] = copy.deepcopy(interval)\n",
    "#         else:\n",
    "#             visemes_speak.add(viseme_list_final[i], viseme_intervals_final[i], pure_phoneme_list[i])\n",
    "    else:        \n",
    "#         visemes_speak.add(viseme_list_final[i], viseme_intervals_final[i], pure_phoneme_list[i])\n",
    "        pass\n",
    "    i = i + increment"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa4c1a21",
   "metadata": {},
   "source": [
    "## Output 1 Marking all the boundary points accordingly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "ad642095",
   "metadata": {},
   "outputs": [],
   "source": [
    "M_Pnot_viseme_list = []\n",
    "M_Pnot_viseme_ctrl_pts = []\n",
    "M_P_viseme_list = []\n",
    "M_P_viseme_ctrl_pts = []\n",
    "Mnot_Pnot_viseme_list = []\n",
    "Mnot_Pnot_viseme_ctrl_pts = []\n",
    "Mnot_P_viseme_list = []\n",
    "Mnot_P_viseme_ctrl_pts = []\n",
    "\n",
    "for i in range(0, len(visemes_sing.viseme_list[-1])):\n",
    "    M_Pnot_viseme_list.append(visemes_sing.viseme_list[-1][i])\n",
    "    Mnot_P_viseme_list.append(visemes_sing.viseme_list[-1][i])\n",
    "    Mnot_Pnot_viseme_list.append(visemes_sing.viseme_list[-1][i])\n",
    "    M_P_viseme_list.append(visemes_sing.viseme_list[-1][i])\n",
    "    ########################################################################\n",
    "    # M and Pnot => boundary from visemes_sing + internal from visemes_speak\n",
    "    ########################################################################\n",
    "    interval = []\n",
    "    for item in visemes_speak.viseme_ctrl_pts[-1][i]:\n",
    "        if item[2] == \"internal\":\n",
    "            interval.append([item[0], item[1]])\n",
    "    for item in visemes_sing.viseme_ctrl_pts[-1][i]:\n",
    "        if item[2] == \"bound\":\n",
    "            interval.append([item[0], item[1]])\n",
    "    M_Pnot_viseme_ctrl_pts.append(interval)\n",
    "    ##########################################################################\n",
    "    # M not and P => boundary from visemes_speak + internal from visemes_sing#\n",
    "    ##########################################################################\n",
    "    interval = []\n",
    "    for item in visemes_sing.viseme_ctrl_pts[-1][i]:\n",
    "        if item[2] == \"internal\":\n",
    "            interval.append([item[0], item[1]])\n",
    "    for item in visemes_speak.viseme_ctrl_pts[-1][i]:\n",
    "        if item[2] == \"bound\":\n",
    "            interval.append([item[0], item[1]])\n",
    "    \n",
    "    Mnot_P_viseme_ctrl_pts.append(interval)\n",
    "    ##########################################################################\n",
    "    # M not and P not => boundary from visemes_speak + internal from visemes_speak\n",
    "    ##########################################################################\n",
    "    interval = []\n",
    "    for item in visemes_speak.viseme_ctrl_pts[-1][i]:\n",
    "        interval.append([item[0], item[1]])\n",
    "    Mnot_Pnot_viseme_ctrl_pts.append(interval)\n",
    "    ##########################################################################\n",
    "    # M and P => boundary from visemes_speak + internal from visemes_sing\n",
    "    ##########################################################################\n",
    "    interval = []\n",
    "    for item in visemes_sing.viseme_ctrl_pts[-1][i]:\n",
    "        interval.append([item[0], item[1]])\n",
    "    M_P_viseme_ctrl_pts.append(interval)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "9b7b3953",
   "metadata": {},
   "outputs": [],
   "source": [
    "output ={\"M_Pnot_viseme\":[M_Pnot_viseme_list, M_Pnot_viseme_ctrl_pts],\n",
    "         \"M_P_viseme\":[M_P_viseme_list, M_P_viseme_ctrl_pts],\n",
    "         \"Mnot_Pnot_viseme\":[Mnot_Pnot_viseme_list, Mnot_Pnot_viseme_ctrl_pts],\n",
    "         \"Mnot_P_viseme\":[Mnot_P_viseme_list, Mnot_P_viseme_ctrl_pts],\n",
    "#         \"brow\":[brow_movement, brow_ctrl_points, finer_brow_raise_ctrl_points, finer_brow_furrow_ctrl_points],\n",
    "         \n",
    "#         \"blink\":[eye_movement, eye_ctrl_points],\n",
    "        \"vowel_mod_M\": [modification_sliders_sing, modification_ctrl_pts_sing],\n",
    "        \"vowel_mod_M_not\": [modification_sliders_speak, modification_ctrl_pts_speak],\n",
    "        \"throat\": larynx_movements_ct_pts,\n",
    "        \"jaw\":[[0, 6]],\n",
    "        \"lip\":[[0, 6]], \n",
    "        \"vib\":vib_ctrl_pts}\n",
    "jsonoutput = json.dumps(output)\n",
    "with open(os.path.join(dir, file_name_template+ '.json'), 'w') as outfile:\n",
    "    json.dump(jsonoutput, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9914aa22",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58eb83ee",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67a8e726",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Tags",
  "kernelspec": {
   "display_name": "visemenet",
   "language": "python",
   "name": "visemenet"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}