Super_Piano_4.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Super_Piano_4.ipynb",
      "provenance": [],
      "private_outputs": true,
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/asigalov61/SuperPiano/blob/master/Super_Piano_4.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M2ApNyMrTFNS"
      },
      "source": [
        "# Super Piano 4\n",
        "## Google TransformerXL\n",
        "\n",
        "### Huge thanks and all the credit for this colab go out to Aniket Singh Rajpoot on whose repo and code it is based: https://github.com/AniketRajpoot/DeepMusicGeneration"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H-8sDtgWUr5j"
      },
      "source": [
        "# Setup and Prep"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pgErROWE11qg",
        "cellView": "form"
      },
      "source": [
        "#@title Import Modules\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "import music21\n",
        "import os\n",
        "#import midifile \n",
        "# pre_process\n",
        "import numpy as np\n",
        "from keras.callbacks import EarlyStopping, ModelCheckpoint, TensorBoard\n",
        "from enum import Enum\n",
        "import matplotlib.pyplot as plt\n",
        "from typing import *\n",
        "import math\n",
        "import time\n",
        "import pickle\n",
        "#import modules\n",
        "#import XL_class"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4JRlvMEHDGAy",
        "cellView": "form"
      },
      "source": [
        "#@title Define Main Constants\n",
        "#specifying data paths \n",
        "path = '/content'\n",
        "\n",
        "BPB = 4 # beats per bar\n",
        "TIMESIG = f'{BPB}/4' # default time signature\n",
        "PIANO_RANGE = (21, 108)\n",
        "NOTE_RANGE = (1,127)\n",
        "VALTSEP = -1 # separator value for numpy encoding\n",
        "VALTCONT = -2 # numpy value for TCONT - needed for compressing chord array\n",
        "\n",
        "SAMPLE_FREQ = 4\n",
        "NOTE_SIZE = 128\n",
        "DUR_SIZE = (10*BPB*SAMPLE_FREQ)+1 # Max length - 8 bars. Or 16 beats/quarternotes\n",
        "MAX_NOTE_DUR = (8*BPB*SAMPLE_FREQ)\n",
        "\n",
        "\n",
        "#tokenizing\n",
        "BOS = 'xxbos'\n",
        "PAD = 'xxpad'\n",
        "EOS = 'xxeos'\n",
        "#MASK = 'xxmask' # Used for BERT masked language modeling. \n",
        "#CSEQ = 'xxcseq' # Used for Seq2Seq translation - denotes start of chord sequence\n",
        "#MSEQ = 'xxmseq' # Used for Seq2Seq translation - denotes start of melody sequence\n",
        "#S2SCLS = 'xxs2scls' # deprecated\n",
        "#NSCLS = 'xxnscls' # deprecated\n",
        "SEP = 'xxsep'\n",
        "IN = 'xxni'     #null instrument\n",
        "\n",
        "SPECIAL_TOKS = [BOS, PAD, EOS, SEP,IN]\n",
        "\n",
        "NOTE_TOKS = [f'n{i}' for i in range(NOTE_SIZE)] \n",
        "DUR_TOKS = [f'd{i}' for i in range(DUR_SIZE)]\n",
        "NOTE_START, NOTE_END = NOTE_TOKS[0], NOTE_TOKS[-1]\n",
        "DUR_START, DUR_END = DUR_TOKS[0], DUR_TOKS[-1]\n",
        "\n",
        "MTEMPO_SIZE = 10\n",
        "MTEMPO_OFF = 'mt0'\n",
        "MTEMPO_TOKS = [f'mt{i}' for i in range(MTEMPO_SIZE)]\n",
        "\n",
        "SEQType = Enum('SEQType', 'Mask, Sentence, Melody, Chords, Empty')\n",
        "\n",
        "ACCEP_INS = dict()\n",
        "ACCEP_INS['Piano'] = 0 \n",
        "ACCEP_INS['Acoustic Bass'] = 1\n",
        "ACCEP_INS['Acoustic Guitar'] = 2 \n",
        "ACCEP_INS['Violin'] = 3 \n",
        "ACCEP_INS['Flute'] = 4 \n",
        "ACCEP_INS['Contrabass'] = 5 \n",
        "ACCEP_INS['Trumpet'] = 6"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OesYYo1qDPX9",
        "cellView": "form"
      },
      "source": [
        "#@title Check GPU\n",
        "gpu_info = !nvidia-smi\n",
        "gpu_info = '\\n'.join(gpu_info)\n",
        "if gpu_info.find('failed') >= 0:\n",
        "  print('Select the Runtime > \"Change runtime type\" menu to enable a GPU accelerator, ')\n",
        "  print('and then re-execute this cell.')\n",
        "else:\n",
        "  print(gpu_info)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SRpRWOGgDajP",
        "cellView": "form"
      },
      "source": [
        "#@title Check Memory\n",
        "\n",
        "from psutil import virtual_memory\n",
        "ram_gb = virtual_memory().total / 1e9\n",
        "print('Your runtime has {:.1f} gigabytes of available RAM\\n'.format(ram_gb))\n",
        "\n",
        "if ram_gb < 20:\n",
        "  print('To enable a high-RAM runtime, select the Runtime > \"Change runtime type\"')\n",
        "  print('menu, and then select High-RAM in the Runtime shape dropdown. Then, ')\n",
        "  print('re-execute this cell.')\n",
        "else:\n",
        "  print('You are using a high-RAM runtime!')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "slksmnTo2HiN",
        "cellView": "form"
      },
      "source": [
        "#@title Functions 1\n",
        "from enum import Enum\n",
        "import music21\n",
        "\n",
        "PIANO_TYPES = list(range(24)) + list(range(80, 96)) # Piano, Synths\n",
        "PLUCK_TYPES = list(range(24, 40)) + list(range(104, 112)) # Guitar, Bass, Ethnic\n",
        "BRIGHT_TYPES = list(range(40, 56)) + list(range(56, 80))\n",
        "\n",
        "PIANO_RANGE = (21, 109) # https://en.wikipedia.org/wiki/Scientific_pitch_notation\n",
        "\n",
        "\n",
        "#Using enums in python\n",
        "class Track(Enum):\n",
        "    PIANO = 0 # discrete instruments - keyboard, woodwinds\n",
        "    PLUCK = 1 # continuous instruments with pitch bend: violin, trombone, synths\n",
        "    BRIGHT = 2\n",
        "    PERC = 3\n",
        "    UNDEF = 4\n",
        "    \n",
        "ype2inst = {\n",
        "    # use print_music21_instruments() to see supported types\n",
        "    Track.PIANO: 0, # Piano\n",
        "    Track.PLUCK: 24, # Guitar\n",
        "    Track.BRIGHT: 40, # Violin\n",
        "    Track.PERC: 114, # Steel Drum\n",
        "}\n",
        "\n",
        "# INFO_TYPES = set(['TIME_SIGNATURE', 'KEY_SIGNATURE'])\n",
        "INFO_TYPES = set(['TIME_SIGNATURE', 'KEY_SIGNATURE', 'SET_TEMPO'])\n",
        "\n",
        "def file2mf(fp):\n",
        "    mf = music21.midi.MidiFile()\n",
        "    if isinstance(fp, bytes):\n",
        "        mf.readstr(fp)\n",
        "    else:\n",
        "        mf.open(fp)\n",
        "        mf.read()\n",
        "        mf.close()\n",
        "    return mf\n",
        "\n",
        "def mf2stream(mf): return music21.midi.translate.midiFileToStream(mf)\n",
        "\n",
        "def is_empty_midi(fp):\n",
        "    if fp is None: return False\n",
        "    mf = file2mf(fp)\n",
        "    return not any([t.hasNotes() for t in mf.tracks])\n",
        "\n",
        "def num_piano_tracks(fp):\n",
        "    music_file = file2mf(fp)\n",
        "    note_tracks = [t for t in music_file.tracks if t.hasNotes() and get_track_type(t) == Track.PIANO]\n",
        "    return len(note_tracks)\n",
        "\n",
        "def is_channel(t, c_val):\n",
        "    return any([c == c_val for c in t.getChannels()])\n",
        "\n",
        "def track_sort(t): # sort by 1. variation of pitch, 2. number of notes\n",
        "    return len(unique_track_notes(t)), len(t.events)\n",
        "\n",
        "def is_piano_note(pitch):\n",
        "    return (pitch >= PIANO_RANGE[0]) and (pitch < PIANO_RANGE[1])\n",
        "\n",
        "def unique_track_notes(t):\n",
        "    return { e.pitch for e in t.events if e.pitch is not None }\n",
        "\n",
        "def compress_midi_file(fp, cutoff=6, min_variation=3, supported_types=set([Track.PIANO, Track.PLUCK, Track.BRIGHT])):\n",
        "    music_file = file2mf(fp)\n",
        "    \n",
        "    info_tracks = [t for t in music_file.tracks if not t.hasNotes()]\n",
        "    note_tracks = [t for t in music_file.tracks if t.hasNotes()]\n",
        "    \n",
        "    if len(note_tracks) > cutoff:\n",
        "        note_tracks = sorted(note_tracks, key=track_sort, reverse=True)\n",
        "        \n",
        "    supported_tracks = []\n",
        "    for idx,t in enumerate(note_tracks):\n",
        "        if len(supported_tracks) >= cutoff: break\n",
        "        track_type = get_track_type(t)\n",
        "        if track_type not in supported_types: continue\n",
        "        pitch_set = unique_track_notes(t)\n",
        "        if (len(pitch_set) < min_variation): continue # must have more than x unique notes\n",
        "        if not all(map(is_piano_note, pitch_set)): continue # must not contain midi notes outside of piano range\n",
        "#         if track_type == Track.UNDEF: print('Could not designate track:', fp, t)\n",
        "        change_track_instrument(t, type2inst[track_type])\n",
        "        supported_tracks.append(t)\n",
        "    if not supported_tracks: return None\n",
        "    music_file.tracks = info_tracks + supported_tracks\n",
        "    return music_file\n",
        "\n",
        "def get_track_type(t):\n",
        "    if is_channel(t, 10): return Track.PERC\n",
        "    i = get_track_instrument(t)\n",
        "    if i in PIANO_TYPES: return Track.PIANO\n",
        "    if i in PLUCK_TYPES: return Track.PLUCK\n",
        "    if i in BRIGHT_TYPES: return Track.BRIGHT\n",
        "    return Track.UNDEF\n",
        "\n",
        "def get_track_instrument(t):\n",
        "    for idx,e in enumerate(t.events):\n",
        "        if e.type == 'PROGRAM_CHANGE': return e.data\n",
        "    return None\n",
        "\n",
        "def change_track_instrument(t, value):\n",
        "    for idx,e in enumerate(t.events):\n",
        "        if e.type == 'PROGRAM_CHANGE': e.data = value\n",
        "\n",
        "def print_music21_instruments():\n",
        "    for i in range(200):\n",
        "        try: print(i, music21.instrument.instrumentFromMidiProgram(i))\n",
        "        except: pass"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ew5brC192Jc-",
        "cellView": "form"
      },
      "source": [
        "#@title Functions 2\n",
        "\n",
        "\n",
        "def file2stream(fp):\n",
        "    if isinstance(fp, music21.midi.MidiFile): return music21.midi.translate.midiFileToStream(fp)\n",
        "    return music21.converter.parse(fp)\n",
        "\n",
        "def npenc2stream(arr,rev_uniq_ins,bpm=120):\n",
        "    \"Converts numpy encoding to music21 stream\"\n",
        "    chordarr = npenc2chordarr(np.array(arr)) # 1.\n",
        "    return chordarr2stream(chordarr,rev_uniq_ins,bpm=bpm) # 2.\n",
        "\n",
        "# 2.\n",
        "def stream2chordarr(s, note_size=NOTE_SIZE, sample_freq=SAMPLE_FREQ, max_note_dur=MAX_NOTE_DUR):\n",
        "    \"Converts music21.Stream to 1-hot numpy array\"\n",
        "    # assuming 4/4 time\n",
        "    # note x instrument x pitch\n",
        "    # FYI: midi middle C value=60\n",
        "    \n",
        "    # (AS) TODO: need to order by instruments most played and filter out percussion or include the channel\n",
        "    highest_time = max(s.flat.getElementsByClass('Note').highestTime, s.flat.getElementsByClass('Chord').highestTime)\n",
        "    maxTimeStep = round(highest_time * sample_freq)+1\n",
        "    score_arr = np.zeros((maxTimeStep, len(s.parts), NOTE_SIZE))\n",
        "\n",
        "    def note_data(pitch, note):\n",
        "        return (pitch.midi, int(round(note.offset*sample_freq)), int(round(note.duration.quarterLength*sample_freq)))\n",
        "    ins=dict()\n",
        "    for idx,part in enumerate(s.parts):\n",
        "        notes=[]\n",
        "        iterate = False\n",
        "        for elem in part.flat:\n",
        "            if isinstance(elem,music21.instrument.Instrument):\n",
        "                if elem.instrumentName in ACCEP_INS.keys():\n",
        "                    ins[idx] = elem.instrumentName \n",
        "                    iterate = True\n",
        "                else :\n",
        "                    break\n",
        "            if isinstance(elem, music21.note.Note):\n",
        "                notes.append(note_data(elem.pitch, elem))\n",
        "            if isinstance(elem, music21.chord.Chord):\n",
        "                for p in elem.pitches:\n",
        "                    notes.append(note_data(p, elem)) \n",
        "        # sort notes by offset (1), duration (2) so that hits are not overwritten and longer notes have priority\n",
        "        \n",
        "        notes_sorted = sorted(notes, key=lambda x: (x[1], x[2])) \n",
        "        if(iterate == True):\n",
        "            for n in notes_sorted:\n",
        "                if n is None: continue\n",
        "                pitch,offset,duration = n\n",
        "                if max_note_dur is not None and duration > max_note_dur: duration = max_note_dur\n",
        "                score_arr[offset,idx, pitch] = duration\n",
        "                score_arr[offset+1:offset+duration, idx, pitch] = VALTCONT      # Continue holding not\n",
        "        \n",
        "    return score_arr,ins\n",
        "\n",
        "def chordarr2npenc(chordarr, skip_last_rest=True):\n",
        "    # combine instruments\n",
        "    result = []\n",
        "    wait_count = 0\n",
        "    for idx,timestep in enumerate(chordarr):\n",
        "        flat_time = timestep2npenc(timestep)\n",
        "        if len(flat_time) == 0:\n",
        "            wait_count += 1\n",
        "        else:\n",
        "            # pitch, octave, duration, instrument\n",
        "            if wait_count > 0: result.append([VALTSEP, wait_count,-2])\n",
        "            result.extend(flat_time)\n",
        "            wait_count = 1\n",
        "    if wait_count > 0 and not skip_last_rest: result.append([VALTSEP, wait_count,-2])\n",
        "    return np.array(result,dtype = int)\n",
        "\n",
        "#   return np.array(result, dtype=int).reshape(-1, 2) # reshaping. Just in case result is empty\n",
        "\n",
        "# Note: not worrying about overlaps - as notes will still play. just look tied\n",
        "# http://web.mit.edu/music21/doc/moduleReference/moduleStream.html#music21.stream.Stream.getOverlaps\n",
        "def timestep2npenc(timestep, note_range=NOTE_RANGE, enc_type='full'):\n",
        "    # inst x pitch\n",
        "    notes = []\n",
        "    for i,n in zip(*timestep.nonzero()):\n",
        "        d = timestep[i,n]\n",
        "        if d < 0: continue # only supporting short duration encoding for now\n",
        "        if n < note_range[0] or n >= note_range[1]: continue # must be within midi range\n",
        "        notes.append([n,d,i])\n",
        "        \n",
        "    notes = sorted(notes, key=lambda x: x[0], reverse=True) # sort by note (highest to lowest)\n",
        "    \n",
        "    if enc_type is None: \n",
        "        # note, duration\n",
        "        return [n[:2] for n in notes] \n",
        "    if enc_type == 'parts':\n",
        "        # note, duration, part\n",
        "        return [n for n in notes]\n",
        "    if enc_type == 'full':\n",
        "        # note_class, duration , instrument\n",
        "        return [[n, d, i] for n,d,i in notes] \n",
        "\n",
        "###################Decoding Phase##########################################################\n",
        "\n",
        "# 1.\n",
        "def npenc2chordarr(npenc,note_size=NOTE_SIZE):\n",
        "    num_instruments = 1 if npenc.shape[1] <= 2 else npenc.max(axis=0)[-1]\n",
        "    max_len = npenc_len(npenc)\n",
        "    # score_arr = (steps, inst, note)\n",
        "    score_arr = np.zeros((max_len, num_instruments + 1, note_size))\n",
        "    \n",
        "    idx = 0\n",
        "    for step in npenc:\n",
        "        n,d,i = (step.tolist()+[0])[:3] # or n,d,i\n",
        "        if n < VALTSEP: continue # special token\n",
        "        if n == VALTSEP:\n",
        "            idx += d\n",
        "            continue\n",
        "        score_arr[idx,i,n] = d\n",
        "    return score_arr\n",
        "\n",
        "def npenc_len(npenc):\n",
        "    duration = 0\n",
        "    for t in npenc:\n",
        "        if t[0] == VALTSEP: duration += t[1]\n",
        "    return duration + 1\n",
        "\n",
        "\n",
        "# 2.\n",
        "def chordarr2stream(arr,rev_uniq_ins,sample_freq=SAMPLE_FREQ, bpm=120):\n",
        "    duration = music21.duration.Duration(1. / sample_freq)\n",
        "    stream = music21.stream.Score()\n",
        "    stream.append(music21.meter.TimeSignature(TIMESIG))\n",
        "    stream.append(music21.tempo.MetronomeMark(number=bpm))\n",
        "    stream.append(music21.key.KeySignature(0))\n",
        "    for inst in range(arr.shape[1]):\n",
        "        p = partarr2stream(arr[:,inst,:],inst,rev_uniq_ins,duration)\n",
        "        stream.append(p)\n",
        "    stream = stream.transpose(0)\n",
        "    return stream\n",
        "\n",
        "# 2b.\n",
        "def partarr2stream(partarr,inst,rev_uniq_ins,duration):\n",
        "    \"convert instrument part to music21 chords\"\n",
        "#    part = music21.stream.Part()\n",
        "#    part.append(music21.instrument.Piano())\n",
        "#    part_append_duration_notes(partarr, duration, part) # notes already have duration calculated\n",
        "    l = len(rev_uniq_ins) \n",
        "    inst = inst%l\n",
        "    part = music21.stream.Part()\n",
        "    if(rev_uniq_ins[inst] == 'Piano'):\n",
        "        part.append(music21.instrument.Piano())\n",
        "    elif(rev_uniq_ins[inst] == 'Trumpet'):\n",
        "        part.append(music21.instrument.Trumpet())\n",
        "    #elif(rev_uniq_ins[inst] == 'Flute'):\n",
        "        #part.append(music21.instrument.Flute)\n",
        "    elif(rev_uniq_ins[inst] == 'Trumpet'):\n",
        "        part.append(music21.instrument.Trumpet())\n",
        "    elif(rev_uniq_ins[inst] == 'Violin'):\n",
        "        part.append(music21.instrument.Violin())\n",
        "    elif(rev_uniq_ins[inst] == 'Acoustic Bass'):\n",
        "        part.append(music21.instrument.AcousticBass())\n",
        "    elif(rev_uniq_ins[inst] == 'Cello'):\n",
        "        part.append(music21.instrument.Contrabass())\n",
        "    elif(rev_uniq_ins[inst] == 'Acoustic Guitar'):\n",
        "        part.append(music21.instrument.AcousticGuitar())\n",
        "    else:\n",
        "        part.append(music21.instrument.Piano())\n",
        "    part_append_duration_notes(partarr, duration, part)\n",
        "    \n",
        "\n",
        "    return part\n",
        "\n",
        "def part_append_duration_notes(partarr, duration, stream):\n",
        "    \"convert instrument part to music21 chords\"\n",
        "    for tidx,t in enumerate(partarr):\n",
        "        note_idxs = np.where(t > 0)[0] # filter out any negative values (continuous mode)\n",
        "        if len(note_idxs) == 0: continue\n",
        "        notes = []\n",
        "        for nidx in note_idxs:\n",
        "            note = music21.note.Note(nidx)\n",
        "            note.duration = music21.duration.Duration(partarr[tidx,nidx]*duration.quarterLength)\n",
        "            notes.append(note)\n",
        "        for g in group_notes_by_duration(notes):\n",
        "            if len(g) == 1:\n",
        "                stream.insert(tidx*duration.quarterLength, g[0])\n",
        "            else:\n",
        "                chord = music21.chord.Chord(g)\n",
        "                stream.insert(tidx*duration.quarterLength, chord)\n",
        "    return stream\n",
        "\n",
        "from itertools import groupby\n",
        "#  combining notes with different durations into a single chord may overwrite conflicting durations. Example: aylictal/still-waters-run-deep\n",
        "def group_notes_by_duration(notes):\n",
        "    \"separate notes into chord groups\"\n",
        "    keyfunc = lambda n: n.duration.quarterLength\n",
        "    notes = sorted(notes, key=keyfunc)\n",
        "    return [list(g) for k,g in groupby(notes, keyfunc)]\n",
        "\n",
        "\n",
        "# Midi -> npenc Conversion helpers\n",
        "def is_valid_npenc(npenc, note_range=PIANO_RANGE, max_dur=DUR_SIZE, \n",
        "                   min_notes=32, input_path=None, verbose=True):\n",
        "    if len(npenc) < min_notes:\n",
        "        if verbose: print('Sequence too short:', len(npenc), input_path)\n",
        "        return False\n",
        "    if (npenc[:,1] >= max_dur).any(): \n",
        "        if verbose: print(f'npenc exceeds max {max_dur} duration:', npenc[:,1].max(), input_path)\n",
        "        return False\n",
        "    # https://en.wikipedia.org/wiki/Scientific_pitch_notation - 88 key range - 21 = A0, 108 = C8\n",
        "    if ((npenc[...,0] > VALTSEP) & ((npenc[...,0] < note_range[0]) | (npenc[...,0] >= note_range[1]))).any(): \n",
        "        print(f'npenc out of piano note range {note_range}:', input_path)\n",
        "        return False\n",
        "    return True\n",
        "\n",
        "# seperates overlapping notes to different tracks\n",
        "def remove_overlaps(stream, separate_chords=True):\n",
        "    if not separate_chords:\n",
        "        return stream.flat.makeVoices().voicesToParts()\n",
        "    return separate_melody_chord(stream)\n",
        "\n",
        "# seperates notes and chords to different tracks\n",
        "def separate_melody_chord(stream):\n",
        "    new_stream = music21.stream.Score()\n",
        "    if stream.timeSignature: new_stream.append(stream.timeSignature)\n",
        "    new_stream.append(stream.metronomeMarkBoundaries()[0][-1])\n",
        "    if stream.keySignature: new_stream.append(stream.keySignature)\n",
        "    \n",
        "    melody_part = music21.stream.Part(stream.flat.getElementsByClass('Note'))\n",
        "    melody_part.insert(0, stream.getInstrument())\n",
        "    chord_part = music21.stream.Part(stream.flat.getElementsByClass('Chord'))\n",
        "    chord_part.insert(0, stream.getInstrument())\n",
        "    new_stream.append(melody_part)\n",
        "    new_stream.append(chord_part)\n",
        "    return new_stream\n",
        "    \n",
        " # processing functions for sanitizing data\n",
        "\n",
        "def compress_chordarr(chordarr):\n",
        "    return shorten_chordarr_rests(trim_chordarr_rests(chordarr))\n",
        "\n",
        "def trim_chordarr_rests(arr, max_rests=4, sample_freq=SAMPLE_FREQ):\n",
        "    # max rests is in quarter notes\n",
        "    # max 1 bar between song start and end\n",
        "    start_idx = 0\n",
        "    max_sample = max_rests*sample_freq\n",
        "    for idx,t in enumerate(arr):\n",
        "        if (t != 0).any(): break\n",
        "        start_idx = idx+1\n",
        "        \n",
        "    end_idx = 0\n",
        "    for idx,t in enumerate(reversed(arr)):\n",
        "        if (t != 0).any(): break\n",
        "        end_idx = idx+1\n",
        "    start_idx = start_idx - start_idx % max_sample\n",
        "    end_idx = end_idx - end_idx % max_sample\n",
        "#     if start_idx > 0 or end_idx > 0: print('Trimming rests. Start, end:', start_idx, len(arr)-end_idx, end_idx)\n",
        "    return arr[start_idx:(len(arr)-end_idx)]\n",
        "\n",
        "def shorten_chordarr_rests(arr, max_rests=8, sample_freq=SAMPLE_FREQ):\n",
        "    # max rests is in quarter notes\n",
        "    # max 2 bar pause\n",
        "    rest_count = 0\n",
        "    result = []\n",
        "    max_sample = max_rests*sample_freq\n",
        "    for timestep in arr:\n",
        "        if (timestep==0).all(): \n",
        "            rest_count += 1\n",
        "        else:\n",
        "            if rest_count > max_sample:\n",
        "#                 old_count = rest_count\n",
        "                rest_count = (rest_count % sample_freq) + max_sample\n",
        "#                 print(f'Compressing rests: {old_count} -> {rest_count}')\n",
        "            for i in range(rest_count): result.append(np.zeros(timestep.shape))\n",
        "            rest_count = 0\n",
        "            result.append(timestep)\n",
        "    for i in range(rest_count): result.append(np.zeros(timestep.shape))\n",
        "    return np.array(result)\n",
        "\n",
        "# sequence 2 sequence convenience functions\n",
        "\n",
        "def stream2npenc_parts(stream, sort_pitch=True):\n",
        "    chordarr = stream2chordarr(stream)\n",
        "    _,num_parts,_ = chordarr.shape\n",
        "    parts = [part_enc(chordarr, i) for i in range(num_parts)]\n",
        "    return sorted(parts, key=avg_pitch, reverse=True) if sort_pitch else parts\n",
        "\n",
        "def chordarr_combine_parts(parts):\n",
        "    max_ts = max([p.shape[0] for p in parts])\n",
        "    parts_padded = [pad_part_to(p, max_ts) for p in parts]\n",
        "    chordarr_comb = np.concatenate(parts_padded, axis=1)\n",
        "    return chordarr_comb\n",
        "\n",
        "def pad_part_to(p, target_size):\n",
        "    pad_width = ((0,target_size-p.shape[0]),(0,0),(0,0))\n",
        "    return np.pad(p, pad_width, 'constant')\n",
        "\n",
        "def part_enc(chordarr, part):\n",
        "    partarr = chordarr[:,part:part+1,:]\n",
        "    npenc = chordarr2npenc(partarr)\n",
        "    return npenc\n",
        "\n",
        "def avg_tempo(t, sep_idx=VALTSEP):\n",
        "    avg = t[t[:, 0] == sep_idx][:, 1].sum()/t.shape[0]\n",
        "    avg = int(round(avg/SAMPLE_FREQ))\n",
        "    return 'mt'+str(min(avg, MTEMPO_SIZE-1))\n",
        "\n",
        "def avg_pitch(t, sep_idx=VALTSEP):\n",
        "    return t[t[:, 0] > sep_idx][:, 0].mean()   "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6sQ5xy212MlQ",
        "cellView": "form"
      },
      "source": [
        "#@title Functions 3\n",
        "def embedding_lookup(lookup_table, x):\n",
        "    return tf.compat.v1.nn.embedding_lookup(lookup_table, x)\n",
        "\n",
        "\n",
        "def normal_embedding_lookup(x, n_token, d_embed, d_proj, initializer,\n",
        "                            proj_initializer, scope='normal_embed', **kwargs):\n",
        "    emb_scale = d_proj ** 0.5\n",
        "    with tf.compat.v1.variable_scope(scope):\n",
        "        lookup_table = tf.compat.v1.get_variable('lookup_table', [n_token, d_embed], initializer=initializer)\n",
        "        y = embedding_lookup(lookup_table, x)\n",
        "        if d_proj != d_embed:\n",
        "            proj_W = tf.compat.v1.get_variable('proj_W', [d_embed, d_proj], initializer=proj_initializer)\n",
        "            y = tf.einsum('ibe,ed->ibd', y, proj_W)\n",
        "        else:\n",
        "            proj_W = None\n",
        "        ret_params = [lookup_table, proj_W]\n",
        "    y *= emb_scale\n",
        "    return y, ret_params\n",
        "\n",
        "\n",
        "def normal_softmax(hidden, target, n_token, params, scope='normal_softmax', **kwargs):\n",
        "    def _logit(x, W, b, proj):\n",
        "        y = x\n",
        "        if proj is not None:\n",
        "            y = tf.einsum('ibd,ed->ibe', y, proj)\n",
        "        return tf.einsum('ibd,nd->ibn', y, W) + b\n",
        "\n",
        "    params_W, params_projs = params[0], params[1]\n",
        "\n",
        "    with tf.compat.v1.variable_scope(scope):\n",
        "        softmax_b = tf.compat.v1.get_variable('bias', [n_token], initializer=tf.zeros_initializer())\n",
        "        output = _logit(hidden, params_W, softmax_b, params_projs)\n",
        "        nll = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=target, logits=output)\n",
        "    return nll, output\n",
        "\n",
        "\n",
        "def positional_embedding(pos_seq, inv_freq, bsz=None):\n",
        "    sinusoid_inp = tf.einsum('i,j->ij', pos_seq, inv_freq)\n",
        "    pos_emb = tf.concat([tf.sin(sinusoid_inp), tf.cos(sinusoid_inp)], -1)\n",
        "    if bsz is not None:\n",
        "        return tf.tile(pos_emb[:, None, :], [1, bsz, 1])\n",
        "    else:\n",
        "        return pos_emb[:, None, :]\n",
        "\n",
        "\n",
        "def positionwise_FF(inp, d_model, d_inner, dropout, kernel_initializer,\n",
        "                    scope='ff', is_training=True):\n",
        "    output = inp\n",
        "    with tf.compat.v1.variable_scope(scope):\n",
        "        output = tf.keras.layers.Dense(d_inner, activation=tf.nn.relu, \n",
        "                                       kernel_initializer=kernel_initializer, name='layer_1')(inp)\n",
        "        output = tf.keras.layers.Dropout(dropout, name='drop_1')(output, training=is_training)\n",
        "        output = tf.keras.layers.Dense(d_model, activation=tf.nn.relu, \n",
        "                                       kernel_initializer=kernel_initializer, name='layer_2')(output)\n",
        "        output = tf.keras.layers.Dropout(dropout, name='drop_2')(output, training=is_training)\n",
        "        output = tf.keras.layers.LayerNormalization(axis=-1)(output + inp)\n",
        "    return output\n",
        "\n",
        "\n",
        "def _create_mask(qlen, mlen, same_length=False):\n",
        "    attn_mask = tf.ones([qlen, qlen])\n",
        "    mask_u = tf.linalg.band_part(attn_mask, 0, -1)\n",
        "    mask_dia = tf.linalg.band_part(attn_mask, 0, 0)\n",
        "    attn_mask_pad = tf.zeros([qlen, mlen])\n",
        "    ret = tf.concat([attn_mask_pad, mask_u - mask_dia], 1)\n",
        "    if same_length:\n",
        "        mask_l = tf.matrix_band_part(attn_mask, -1, 0)\n",
        "        ret = tf.concat([ret[:, :qlen] + mask_l - mask_dia, ret[:, qlen:]], 1)\n",
        "    return ret\n",
        "\n",
        "\n",
        "def _cache_mem(curr_out, prev_mem, mem_len=None):\n",
        "    if mem_len is None or prev_mem is None:\n",
        "        new_mem = curr_out\n",
        "    elif mem_len == 0:\n",
        "        return prev_mem\n",
        "    else:\n",
        "        new_mem = tf.concat([prev_mem, curr_out], 0)[-mem_len:]\n",
        "    return tf.stop_gradient(new_mem)\n",
        "\n",
        "\n",
        "def rel_shift(x):\n",
        "    x_size = tf.shape(x)\n",
        "    x = tf.pad(x, [[0, 0], [1, 0], [0, 0], [0, 0]])\n",
        "    x = tf.reshape(x, [x_size[1] + 1, x_size[0], x_size[2], x_size[3]])\n",
        "    x = tf.slice(x, [1, 0, 0, 0], [-1, -1, -1, -1])\n",
        "    x = tf.reshape(x, x_size)\n",
        "    return x\n",
        "\n",
        "\n",
        "def rel_multihead_attn(w, r, r_w_bias, r_r_bias, attn_mask, mems, d_model,\n",
        "                       n_head, d_head, dropout, dropatt, is_training,\n",
        "                       kernel_initializer, scope='rel_attn'):\n",
        "    scale = 1 / (d_head ** 0.5)\n",
        "    with tf.compat.v1.variable_scope(scope):\n",
        "        qlen = tf.shape(w)[0]\n",
        "        rlen = tf.shape(r)[0]\n",
        "        bsz = tf.shape(w)[1]\n",
        "\n",
        "        cat = tf.concat([mems, w], 0) if mems is not None and mems.shape.ndims > 1 else w\n",
        "\n",
        "        w_heads = tf.keras.layers.Dense(3 * n_head * d_head, use_bias=False, \n",
        "                                        kernel_initializer=kernel_initializer, name='qkv')(cat)\n",
        "        r_head_k = tf.keras.layers.Dense(n_head * d_head, use_bias=False,\n",
        "                                         kernel_initializer=kernel_initializer, name='r')(r)\n",
        "        \n",
        "        w_head_q, w_head_k, w_head_v = tf.split(w_heads, 3, -1)\n",
        "        w_head_q = w_head_q[-qlen:]\n",
        "\n",
        "        klen = tf.shape(w_head_k)[0]\n",
        "\n",
        "        w_head_q = tf.reshape(w_head_q, [qlen, bsz, n_head, d_head])\n",
        "        w_head_k = tf.reshape(w_head_k, [klen, bsz, n_head, d_head])\n",
        "        w_head_v = tf.reshape(w_head_v, [klen, bsz, n_head, d_head])\n",
        "\n",
        "        r_head_k = tf.reshape(r_head_k, [rlen, n_head, d_head])\n",
        "\n",
        "        rw_head_q = w_head_q + r_w_bias\n",
        "        rr_head_q = w_head_q + r_r_bias\n",
        "\n",
        "        AC = tf.einsum('ibnd,jbnd->ijbn', rw_head_q, w_head_k)\n",
        "        BD = tf.einsum('ibnd,jnd->ijbn', rr_head_q, r_head_k)\n",
        "        BD = rel_shift(BD)\n",
        "\n",
        "        attn_score = (AC + BD) * scale\n",
        "        attn_mask_t = attn_mask[:, :, None, None]\n",
        "        attn_score = attn_score * (1 - attn_mask_t) - 1e30 * attn_mask_t\n",
        "\n",
        "        attn_prob = tf.nn.softmax(attn_score, 1)\n",
        "        attn_prob = tf.keras.layers.Dropout(dropatt)(attn_prob, training=is_training)\n",
        "\n",
        "        attn_vec = tf.einsum('ijbn,jbnd->ibnd', attn_prob, w_head_v)\n",
        "        size_t = tf.shape(attn_vec)\n",
        "        attn_vec = tf.reshape(attn_vec, [size_t[0], size_t[1], n_head * d_head])\n",
        "\n",
        "        attn_out = tf.keras.layers.Dense(d_model, use_bias=False, \n",
        "                                         kernel_initializer=kernel_initializer, name='o')(attn_vec)\n",
        "        attn_out = tf.keras.layers.Dropout(dropout)(attn_out, training=is_training)\n",
        "        output = tf.keras.layers.LayerNormalization(axis=-1)(attn_out + w)\n",
        "        return output\n",
        "\n",
        "\n",
        "\n",
        "def transformer(dec_inp, target, mems, n_token, n_layer, d_model, d_embed,\n",
        "                n_head, d_head, d_inner, dropout, dropatt,\n",
        "                initializer, is_training, proj_initializer=None,\n",
        "                mem_len=None, cutoffs=[], div_val=1, tie_projs=[],\n",
        "                same_length=False, clamp_len=-1,\n",
        "                input_perms=None, target_perms=None, head_target=None,\n",
        "                untie_r=False, proj_same_dim=True,\n",
        "                scope='transformer'):\n",
        "    \"\"\"\n",
        "    cutoffs: a list of python int. Cutoffs for adaptive softmax.\n",
        "    tie_projs: a list of python bools. Whether to tie the projections.\n",
        "    perms: a list of tensors. Each tensor should of size [len, bsz, bin_size].\n",
        "        Only used in the adaptive setting.\n",
        "    \"\"\"\n",
        "\n",
        "    new_mems = []\n",
        "    with tf.compat.v1.variable_scope(scope, reuse= tf.compat.v1.AUTO_REUSE):\n",
        "        if untie_r:\n",
        "            r_w_bias = tf.compat.v1.get_variable('r_w_bias', [n_layer, n_head, d_head], initializer=initializer)\n",
        "            r_r_bias = tf.compat.v1.get_variable('r_r_bias', [n_layer, n_head, d_head], initializer=initializer)\n",
        "        else:\n",
        "            r_w_bias = tf.compat.v1.get_variable('r_w_bias', [n_head, d_head], initializer=initializer)\n",
        "            r_r_bias = tf.compat.v1.get_variable('r_r_bias', [n_head, d_head], initializer=initializer)\n",
        "\n",
        "        qlen = tf.shape(dec_inp)[0]\n",
        "        mlen = tf.shape(mems[0])[0] if mems is not None else 0\n",
        "        klen = qlen + mlen\n",
        "\n",
        "        if proj_initializer is None:\n",
        "            proj_initializer = initializer\n",
        "\n",
        "        embeddings, shared_params = normal_embedding_lookup(\n",
        "            x=dec_inp,\n",
        "            n_token=n_token,\n",
        "            d_embed=d_embed,\n",
        "            d_proj=d_model,\n",
        "            initializer=initializer,\n",
        "            proj_initializer=proj_initializer)\n",
        "        \n",
        "        attn_mask = _create_mask(qlen, mlen, same_length)\n",
        "        \n",
        "        pos_seq = tf.range(klen - 1, -1, -1.0)\n",
        "        if clamp_len > 0:\n",
        "            pos_seq = tf.minimum(pos_seq, clamp_len)\n",
        "        inv_freq = 1 / (10000 ** (tf.range(0, d_model, 2.0) / d_model))\n",
        "        pos_emb = positional_embedding(pos_seq, inv_freq)\n",
        "\n",
        "        output = tf.keras.layers.Dropout(rate=dropout)(embeddings, training=is_training)\n",
        "        pos_emb = tf.keras.layers.Dropout(rate=dropout)(pos_emb, training=is_training)\n",
        "\n",
        "        if mems is None:\n",
        "            mems = [None] * n_layer\n",
        "\n",
        "        for i in range(n_layer):\n",
        "            # cache new mems\n",
        "            new_mems.append(_cache_mem(output, mems[i], mem_len))\n",
        "\n",
        "            with tf.compat.v1.variable_scope('layer_{}'.format(i)):\n",
        "                output = rel_multihead_attn(\n",
        "                    w=output,\n",
        "                    r=pos_emb,\n",
        "                    r_w_bias=r_w_bias if not untie_r else r_w_bias[i],\n",
        "                    r_r_bias=r_r_bias if not untie_r else r_r_bias[i],\n",
        "                    attn_mask=attn_mask,\n",
        "                    mems=mems[i],\n",
        "                    d_model=d_model,\n",
        "                    n_head=n_head,\n",
        "                    d_head=d_head,\n",
        "                    dropout=dropout,\n",
        "                    dropatt=dropatt,\n",
        "                    is_training=is_training,\n",
        "                    kernel_initializer=initializer)\n",
        "\n",
        "                output = positionwise_FF(\n",
        "                    inp=output,\n",
        "                    d_model=d_model,\n",
        "                    d_inner=d_inner,\n",
        "                    dropout=dropout,\n",
        "                    kernel_initializer=initializer,\n",
        "                    is_training=is_training)\n",
        "\n",
        "        # apply Dropout\n",
        "        output = tf.keras.layers.Dropout(dropout)(output, training=is_training)\n",
        "\n",
        "        loss, logits = normal_softmax(\n",
        "            hidden=output,\n",
        "            target=target,\n",
        "            n_token=n_token,\n",
        "            params=shared_params)\n",
        "\n",
        "        return loss, logits, new_mems"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D1d_eZNZ2QTW",
        "cellView": "form"
      },
      "source": [
        "#@title Functions 4\n",
        "class TransformerXL(object):\n",
        "    ########################################\n",
        "    # initialize\n",
        "    ########################################\n",
        "    def __init__(self, vocab_size, checkpoint=None, is_training=False, training_seqs=None):\n",
        "        # load dictionary\n",
        "        self.event2word = vocab_size\n",
        "        # model settings\n",
        "        self.x_len = 512      #input sequence length\n",
        "        self.mem_len = 512    #\n",
        "        self.n_layer = 6\n",
        "        self.d_embed = 768\n",
        "        self.d_model = 768\n",
        "        self.dropout = 0.1    ##\n",
        "        self.n_head = 12\n",
        "        self.d_head = self.d_model // self.n_head\n",
        "        self.d_ff = 3072\n",
        "        self.n_token = (self.event2word)\n",
        "        self.learning_rate = 1e-4      ##\n",
        "        self.group_size = 3\n",
        "        self.entry_len = self.group_size * self.x_len\n",
        "        # mode\n",
        "        self.is_training = is_training\n",
        "        self.training_seqs = training_seqs\n",
        "        self.checkpoint = checkpoint\n",
        "        if self.is_training: # train from scratch or finetune\n",
        "            self.batch_size = 8        \n",
        "        else: # inference\n",
        "            self.batch_size = 1\n",
        "        # load model\n",
        "        self.load_model()\n",
        "\n",
        "    ########################################\n",
        "    # load model\n",
        "    ########################################\n",
        "    \n",
        "    def load_model(self):\n",
        "        tf.compat.v1.disable_eager_execution()\n",
        "        # placeholders ---> train\n",
        "        self.x = tf.compat.v1.placeholder(tf.int32, shape=[self.batch_size, None])\n",
        "        self.y = tf.compat.v1.placeholder(tf.int32, shape=[self.batch_size, None])\n",
        "        self.mems_i = [tf.compat.v1.placeholder(tf.float32, [self.mem_len, self.batch_size, self.d_model]) for _ in range(self.n_layer)]\n",
        "        # placeholders ---> test\n",
        "        self.x_t = tf.compat.v1.placeholder(tf.int32, shape=[1, None])\n",
        "        self.y_t = tf.compat.v1.placeholder(tf.int32, shape=[1, None])\n",
        "        self.mems_it = [tf.compat.v1.placeholder(tf.float32, [self.mem_len, 1, self.d_model]) for _ in range(self.n_layer)]\n",
        "        # model\n",
        "        self.global_step = tf.compat.v1.train.get_or_create_global_step()\n",
        "\n",
        "        # initialize parameters\n",
        "        initializer = tf.compat.v1.initializers.random_normal(stddev=0.02, seed=None)\n",
        "        proj_initializer = tf.compat.v1.initializers.random_normal(stddev=0.01, seed=None)\n",
        "        \n",
        "        with tf.compat.v1.variable_scope(tf.compat.v1.get_variable_scope()):\n",
        "            xx = tf.transpose(self.x, [1, 0])\n",
        "            yy = tf.transpose(self.y, [1, 0])\n",
        "            loss, self.logits, self.new_mem = transformer(\n",
        "                dec_inp=xx,\n",
        "                target=yy,\n",
        "                mems=self.mems_i,\n",
        "                n_token=self.n_token,\n",
        "                n_layer=self.n_layer,\n",
        "                d_model=self.d_model,\n",
        "                d_embed=self.d_embed,\n",
        "                n_head=self.n_head,\n",
        "                d_head=self.d_head,\n",
        "                d_inner=self.d_ff,\n",
        "                dropout=self.dropout,\n",
        "                dropatt=self.dropout,\n",
        "                initializer=initializer,\n",
        "                proj_initializer=proj_initializer,\n",
        "                is_training=self.is_training,\n",
        "                mem_len=self.mem_len,\n",
        "                cutoffs=[],\n",
        "                div_val=-1,\n",
        "                tie_projs=[],\n",
        "                same_length=False,\n",
        "                clamp_len=-1,\n",
        "                input_perms=None,\n",
        "                target_perms=None,\n",
        "                head_target=None,\n",
        "                untie_r=False,\n",
        "                proj_same_dim=True)\n",
        "        self.avg_loss = tf.reduce_mean(loss)\n",
        "        # vars\n",
        "        all_vars = tf.compat.v1.trainable_variables()\n",
        "        print ('num parameters:', np.sum([np.prod(v.get_shape().as_list()) for v in all_vars]))\n",
        "        grads = tf.gradients(self.avg_loss, all_vars)\n",
        "        grads_and_vars = list(zip(grads, all_vars))\n",
        "        # gradient clipping\n",
        "        def ClipIfNotNone(grad):\n",
        "            if grad is None:\n",
        "                return grad\n",
        "            return tf.clip_by_norm(grad, 100.)\n",
        "        \n",
        "        grads_and_vars = [(ClipIfNotNone(grad), var) for grad, var in grads_and_vars]\n",
        "        all_trainable_vars = tf.reduce_sum([tf.reduce_prod(v.shape) for v in tf.compat.v1.trainable_variables()])\n",
        "        # optimizer\n",
        "        #warmup_steps = 0\n",
        "        # increase the learning rate linearly\n",
        "        #if warmup_steps > 0:\n",
        "        #    warmup_lr = tf.compat.v1.to_float(self.global_step) / tf.compat.v1.to_float(warmup_steps) \\\n",
        "        #          * self.learning_rate\n",
        "        #else:\n",
        "        #    warmup_lr = 0.0\n",
        "\n",
        "        decay_lr = tf.compat.v1.train.cosine_decay(\n",
        "            self.learning_rate,\n",
        "            global_step=self.global_step,\n",
        "            decay_steps=200000,\n",
        "            alpha=0.004)\n",
        "        \n",
        "        #lr_decay_warmup = tf.where(self.global_step < warmup_steps,\n",
        "        #                    warmup_lr, decay_lr)\n",
        "        #decay_lr = tf.compat.v1.train.cosine_decay_warmup(     ##\n",
        "        #     self.learning_rate,\n",
        "        #     global_step=self.global_step,\n",
        "        #     decay_steps=200000,\n",
        "        #     warmup_steps=16000,\n",
        "        #     alpha=0.004\n",
        "        #)\n",
        "        \n",
        "        #try:\n",
        "            #self.optimizer = tfa.optimizers.LAMB(learning_rate=decay_lr)\n",
        "            #print('LAMBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB')\n",
        "        #except:\n",
        "            #self.optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=decay_lr)\n",
        "            #print('ADAMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM')\n",
        "            #pass\n",
        "        self.optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=decay_lr)\n",
        "        self.train_op = self.optimizer.apply_gradients(grads_and_vars, self.global_step)\n",
        "        # saver\n",
        "        self.saver = tf.compat.v1.train.Saver(max_to_keep=100)\n",
        "        config = tf.compat.v1.ConfigProto(allow_soft_placement=True)\n",
        "        config.gpu_options.allow_growth = True\n",
        "        self.sess = tf.compat.v1.Session(config=config)\n",
        "        # load pre-trained checkpoint or note\n",
        "        if self.checkpoint:\n",
        "            self.saver.restore(self.sess, self.checkpoint)\n",
        "        else:\n",
        "            self.sess.run(tf.compat.v1.global_variables_initializer())\n",
        "            \n",
        "    \n",
        "            \n",
        "    ########################################\n",
        "    # train\n",
        "    ########################################\n",
        "    def train(self, training_data, output_checkpoint_folder):\n",
        "        # check output folder\n",
        "        if not os.path.exists(output_checkpoint_folder):\n",
        "            os.mkdir(output_checkpoint_folder)\n",
        "        # shuffle\n",
        "        index = np.arange(len(training_data))\n",
        "        np.random.shuffle(index)\n",
        "        training_data = training_data[index]\n",
        "        num_batches = len(training_data) // self.batch_size\n",
        "        st = time.time()\n",
        "        for e in range(1000):\n",
        "            total_loss = []\n",
        "            for i in range(num_batches):\n",
        "                segments = training_data[self.batch_size*i:self.batch_size*(i+1)]\n",
        "                batch_m = [np.zeros((self.mem_len, self.batch_size, self.d_model), dtype=np.float32) for _ in range(self.n_layer)]\n",
        "                for j in range(self.group_size):\n",
        "                    batch_x = segments[:, j, 0, :]\n",
        "                    batch_y = segments[:, j, 1, :]\n",
        "                    # prepare feed dict\n",
        "                    feed_dict = {self.x: batch_x, self.y: batch_y}\n",
        "                    for m, m_np in zip(self.mems_i, batch_m):\n",
        "                        feed_dict[m] = m_np\n",
        "                    # run\n",
        "                    _, gs_, loss_, new_mem_ = self.sess.run([self.train_op, self.global_step, self.avg_loss, self.new_mem], feed_dict=feed_dict)\n",
        "                    batch_m = new_mem_\n",
        "                    total_loss.append(loss_)\n",
        "                    # print ('Current lr: {}'.format(self.sess.run(self.optimizer._lr)))\n",
        "                    print('>>> Epoch: {}, Step: {}, Loss: {:.5f}, Time: {:.2f}'.format(e, gs_, loss_, time.time()-st))\n",
        "                    print('i : ',i,' j : ',j)\n",
        "                    if not i % 500:\n",
        "                        self.saver.save(self.sess, '{}/model-{:03d}-{:.3f}'.format(output_checkpoint_folder, e, np.mean(total_loss)))\n",
        "                    \n",
        "\n",
        "            print ('[epoch {} avg loss] {:.5f}'.format(e, np.mean(total_loss)))\n",
        "            if not e % 6:\n",
        "                self.saver.save(self.sess, '{}/model-{:03d}-{:.3f}'.format(output_checkpoint_folder, e, np.mean(total_loss)))\n",
        "            # stop\n",
        "            if np.mean(total_loss) <= 0.0001:\n",
        "                break\n",
        "\n",
        "    ########################################\n",
        "    # search strategy: temperature (re-shape)\n",
        "    ########################################\n",
        "    def temperature(self, logits, temperature):\n",
        "        probs = np.exp(logits / temperature) / np.sum(np.exp(logits / temperature))\n",
        "        return probs\n",
        "\n",
        "\n",
        "    ########################################\n",
        "    # search strategy: nucleus (truncate)\n",
        "    ########################################\n",
        "    def nucleus(self, probs, p):\n",
        "        probs /= sum(probs)\n",
        "        sorted_probs = np.sort(probs)[::-1]\n",
        "        sorted_index = np.argsort(probs)[::-1]\n",
        "        cusum_sorted_probs = np.cumsum(sorted_probs)\n",
        "        after_threshold = cusum_sorted_probs > p\n",
        "        if sum(after_threshold) > 0:\n",
        "            last_index = np.where(after_threshold)[0][-1]\n",
        "            candi_index = sorted_index[:last_index]\n",
        "        else:\n",
        "            candi_index = sorted_index[:3] # just assign a value\n",
        "        candi_probs = [probs[i] for i in candi_index]\n",
        "        candi_probs /= sum(candi_probs)\n",
        "        word = np.random.choice(candi_index, size=1, p=candi_probs)[0]\n",
        "        return word\n",
        "\n",
        "    ########################################\n",
        "    # evaluate (for batch size = 1)\n",
        "    ########################################\n",
        "    def evaluate(self, notes, num_notes, k, strategies, use_structure=False, init_mem = None):\n",
        "\n",
        "      batch_size = 1\n",
        "      # initialize mem\n",
        "      if init_mem is None:\n",
        "          batch_m = [np.zeros((self.mem_len, batch_size, self.d_model), dtype=np.float32) for _ in range(self.n_layer)]\n",
        "          print('new memmmmm')\n",
        "      else:\n",
        "          batch_m = init_mem \n",
        "\n",
        "      initial_flag = True\n",
        "      fail = 0\n",
        "      i = 0\n",
        "\n",
        "      while i < num_notes:\n",
        "            if fail>200:\n",
        "              print('Fail : ',fail)\n",
        "              #continue\n",
        "\n",
        "            # prepare input\n",
        "            if initial_flag:\n",
        "                temp_x = np.zeros((batch_size, len(notes[0])))\n",
        "                for b in range(batch_size):\n",
        "                    for z, t in enumerate(notes[b]):\n",
        "                        temp_x[b][z] = t\n",
        "                initial_flag = False\n",
        "            else:\n",
        "                temp_x = np.zeros((batch_size, 1))\n",
        "                for b in range(batch_size):\n",
        "                    temp_x[b][0] = notes[b][-1]\n",
        "\n",
        "            # prepare feed dict\n",
        "            # inside a feed dict\n",
        "            # placeholder : data\n",
        "            # put input into feed_dict\n",
        "            feed_dict = {self.x: temp_x}\n",
        "\n",
        "            # put memeory into feed_dict\n",
        "            for m, m_np in zip(self.mems_i, batch_m):\n",
        "                feed_dict[m] = m_np\n",
        "            \n",
        "            # model (prediction)\n",
        "            _logits, _new_mem = self.sess.run([self.logits, self.new_mem], feed_dict=feed_dict)\n",
        "            #print('mem : ',_new_mem,' shape : ',len(_new_mem))\n",
        "            #print('shape : ',_logits.shape)\n",
        "            logits = _logits[-1, 0]\n",
        "\n",
        "            # temperature or not\n",
        "            if k == 0:\n",
        "              ran = float((np.random.randint(14,16))/10)\n",
        "            else:\n",
        "              ran = float((np.random.randint(7,10))/10)\n",
        "            \n",
        "            probs = self.temperature(logits=logits, temperature=ran)\n",
        "\n",
        "            # sampling\n",
        "            # note : the generated tokenized event\n",
        "            #ran_n = float((np.random.randint(90,98))/100)\n",
        "            note = self.nucleus(probs=probs, p=0.90)\n",
        "            \n",
        "\n",
        "            if note not in tokenizer.index_word:\n",
        "              continue\n",
        "\n",
        "            if (tokenizer.index_word[int(notes[0][-1])])[0] == 'n' and (tokenizer.index_word[int(note)])[0] != 'd':\n",
        "              print((tokenizer.index_word[int(notes[0][-1])]),' : ', tokenizer.index_word[int(note)])\n",
        "              fail += 1\n",
        "              continue\n",
        "            if (tokenizer.index_word[int(notes[0][-1])])[0] == 'd' and ((tokenizer.index_word[int(note)])[0] != 'i' and (tokenizer.index_word[int(note)]) != 'xxni'):\n",
        "              fail += 1\n",
        "              print((tokenizer.index_word[int(notes[0][-1])]),' : ',tokenizer.index_word[int(note)])\n",
        "              continue\n",
        "            if ((tokenizer.index_word[int(notes[0][-1])])[0] == 'i' or tokenizer.index_word[int(notes[0][-1])] == 'xxni') and ((tokenizer.index_word[int(note)])[0] != 'n' and (tokenizer.index_word[int(note)]) != 'xxsep'):\n",
        "              fail += 1\n",
        "              print((tokenizer.index_word[int(notes[0][-1])]),' : ',tokenizer.index_word[int(note)])\n",
        "              continue\n",
        "            if (tokenizer.index_word[int(notes[0][-1])]) == 'xxsep' and ((tokenizer.index_word[int(note)])[0] != 'd' and (tokenizer.index_word[int(note)])[0] != 'n'):\n",
        "              fail += 1\n",
        "              print((tokenizer.index_word[int(notes[0][-1])]),' : ',tokenizer.index_word[int(note)])\n",
        "              continue\n",
        "            \n",
        "            \n",
        "\n",
        "            # add new event to record sequence\n",
        "            notes = np.append(notes[0], note)\n",
        "            notes = np.reshape(notes, (1, len(notes)))\n",
        "            #print('notes : ',notes.shape)\n",
        "            \n",
        "            # re-new mem\n",
        "            batch_m = _new_mem\n",
        "            fail = 0\n",
        "            i += 1\n",
        "\n",
        "      return notes[0]\n",
        "\n",
        "    ########################################\n",
        "        # predict (for batch size = 1)\n",
        "    ########################################\n",
        "    def predict(self, notes, num_notes, k, strategies, use_structure=False):\n",
        "      prediction = self.evaluate(notes, num_notes, k, strategies, use_structure)\n",
        "\n",
        "      predicted_sentence = []\n",
        "  \n",
        "      for i in prediction:\n",
        "          # print('helllllo',int(i))\n",
        "          i = int(i)\n",
        "          if i < len(tokenizer.word_index) and i>0:\n",
        "              predicted_sentence.append(tokenizer.index_word[i])\n",
        "      return predicted_sentence"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HU5daS2L2UUl",
        "cellView": "form"
      },
      "source": [
        "#@title Functions 5\n",
        "def get_all_midi_dir(root_dir):\n",
        "    all_midi = []\n",
        "    for dirName, _, fileList in os.walk(root_dir):\n",
        "        for fname in fileList:\n",
        "            if '.mid' in fname:\n",
        "                all_midi.append(dirName + '/' + fname)\n",
        "\n",
        "    return all_midi\n",
        "\n",
        "\n",
        "    \n",
        "def get_data(notes_chords, sequence_length):\n",
        "    \n",
        "    # sequence_length = 100\n",
        "    notes_input = []\n",
        "    notes_output = []\n",
        "    shift = 1\n",
        "    \n",
        "    for i in range(0, len(notes_chords) - sequence_length, 1):\n",
        "        temp_input = ''\n",
        "        temp_output = ''\n",
        "        for j in range(i,i + sequence_length):\n",
        "            temp_input += notes_chords[j] + ' '\n",
        "        notes_input.append(temp_input)\n",
        "        for j in range(i+shift,i + sequence_length+shift):\n",
        "            temp_output += notes_chords[j] + ' '\n",
        "        notes_output.append(temp_output)\n",
        "\n",
        "\n",
        "    n_patterns = len(notes_input)\n",
        "    # notes_normalized_input = np.reshape(notes_input, (n_patterns, sequence_length))\n",
        "    # notes_normalized_input =  notes_normalized_input / float(n_vocab)\n",
        "    #notes_output = np.array(notes_output)\n",
        "\n",
        "\n",
        "    return (notes_input, notes_output)\n",
        "\n",
        "\n",
        "########################################\n",
        "    # Prepare data\n",
        "########################################\n",
        "        \n",
        "def xl_data(input_, output, group_size):\n",
        "        training_data = []\n",
        "    \n",
        "        pairs = []\n",
        "        for i in range(0, len(input_)):\n",
        "            x, y = input_[i], output[i]\n",
        "            \n",
        "            pairs.append([x, y])\n",
        "\n",
        "        pairs = np.array(pairs)\n",
        "    \n",
        "        # put pairs into training data by groups\n",
        "        for i in range(0, len(pairs) - group_size + 1, group_size):\n",
        "            segment = pairs[i:i+group_size]\n",
        "            assert len(segment) == group_size\n",
        "            training_data.append(segment)\n",
        "            \n",
        "        training_data = np.array(training_data)\n",
        "        \n",
        "        return training_data        \n",
        "        \n",
        "        \n",
        "        \n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y4FM_r6YAPON",
        "cellView": "form"
      },
      "source": [
        "#@title Functions 6\n",
        "def check_valid_ins(ins):\n",
        "  count = 0\n",
        "  ls = list(set(val for val in ins.values()))\n",
        "  for i in ls:\n",
        "    if i == 'Piano':\n",
        "      count+= 1\n",
        "    elif i == 'Acoustic Bass' or i == 'Electric Bass':\n",
        "      count += 1\n",
        "    elif i == 'Acoustic Guitar' or i == 'Electric Guitar':\n",
        "      count += 1\n",
        "    elif i == 'Violin':\n",
        "      count += 1\n",
        "    elif i == 'Flute':\n",
        "      count += 1\n",
        "  if(count>=2):\n",
        "    return True\n",
        "  return False"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QclsGGEy2dqP",
        "cellView": "form"
      },
      "source": [
        "#@title Functions 7\n",
        "#required listsx\n",
        "chordarr_list = []\n",
        "npenc_list = []\n",
        "ins_list = []\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UK81q6cEIbYK",
        "cellView": "form"
      },
      "source": [
        "#@title Process MIDIs\n",
        "#read multiple files \n",
        "i = 0\n",
        "overall = 0\n",
        "for file_name in get_all_midi_dir('/content/midis'):\n",
        "        if overall>90:\n",
        "          break\n",
        "        print('Now loading, ',i,' : ',overall,': \\n',file_name)\n",
        "        try:\n",
        "           mf = file2mf(file_name)\n",
        "        except:\n",
        "           continue\n",
        "           pass\n",
        "        try:\n",
        "           stream =mf2stream(mf)\n",
        "        except:\n",
        "           continue\n",
        "           pass\n",
        "        i += 1\n",
        "        overall += 1\n",
        "        chordarr,ins = stream2chordarr(stream)\n",
        "\n",
        "        if (not(check_valid_ins(ins))):\n",
        "          print('Discarding File :\\n', file_name)\n",
        "          try:\n",
        "             print('1') \n",
        "          except:\n",
        "              print('2')\n",
        "              pass\n",
        "          i -= 1\n",
        "          continue\n",
        "\n",
        "        ins_list.append(ins)\n",
        "        chordarr_list.append(chordarr)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C9iuC8MA2OwW",
        "cellView": "form"
      },
      "source": [
        "#@title Save Chords and Instruments Lists\n",
        "with open('/content/chord_list', 'wb') as filepath:\n",
        "     pickle.dump(chordarr_list, filepath)\n",
        "with open('/content/ins_list', 'wb') as filepath:\n",
        "     pickle.dump(ins_list, filepath)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bMxsThkRISuT",
        "cellView": "form"
      },
      "source": [
        "#@title Load Chord and Instruments Lists\n",
        "with open('/content/chord_list_2018', 'rb') as filepath:\n",
        "         chordarr_list = pickle.load(filepath)\n",
        "with open('/content/ins_list_2018', 'rb') as filepath:\n",
        "         ins_list = pickle.load(filepath)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gTAjNIxMvIH3",
        "cellView": "form"
      },
      "source": [
        "#@title Process Chords and Instruments into Final List\n",
        "print('Processing Now')\n",
        "#making uniq list for transformation \n",
        "res = list(set(val for dic in ins_list for val in dic.values())) \n",
        "uniq_ins = dict()\n",
        "for i in range(len(res)):\n",
        "        uniq_ins[res[i]] = i   \n",
        "INS_TOKS = [f'i{i}' for i in range(len(uniq_ins))]\n",
        "rev_uniq_ins = {value : key for (key, value) in uniq_ins.items()} \n",
        "    \n",
        "    \n",
        "for c in range(len(chordarr_list)):\n",
        "        npenc = chordarr2npenc(chordarr_list[c])\n",
        "        for i in npenc:\n",
        "            if(i[2] == -2):\n",
        "                i[2] = -2\n",
        "            else:\n",
        "                i[2] = uniq_ins[ins_list[c][i[2]]]\n",
        "        npenc_list.append(npenc)\n",
        "  \n",
        "    \n",
        "       \n",
        "#the final list or sequence \n",
        "final_list = []\n",
        "    \n",
        "for npenc in npenc_list:\n",
        "        final_list.append(BOS)\n",
        "        final_list.append(PAD)\n",
        "        \n",
        "        for i in range(len(npenc)):\n",
        "            if(npenc[i][0] == -1):\n",
        "                 x = SEP\n",
        "            else:\n",
        "                x = 'n' + str(npenc[i][0])\n",
        "            if npenc[i][1] > 16:\n",
        "              npenc[i][1] = 8\n",
        "            y = 'd' + str(npenc[i][1])\n",
        "            if(npenc[i][2] == -2):\n",
        "                z = IN\n",
        "            else:\n",
        "                z = 'i' + str(npenc[i][2])\n",
        "            final_list.append(x)\n",
        "            final_list.append(y)\n",
        "            final_list.append(z)\n",
        "        \n",
        "        final_list.append(PAD)\n",
        "        final_list.append(EOS)    \n",
        "\n",
        "    \n",
        "unique_notes = list(set(final_list))\n",
        "n_vocab = len(set(unique_notes))\n",
        "\n",
        "tokenizer = tf.keras.preprocessing.text.Tokenizer(filters='')\n",
        "tokenizer.fit_on_texts(final_list)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8oj_ESalRncp",
        "cellView": "form"
      },
      "source": [
        "#@title Save Final List\n",
        "with open('/content/final_list_single', 'wb') as filepath:\n",
        "     pickle.dump(final_list, filepath)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ClB0cTTjdfS0",
        "cellView": "form"
      },
      "source": [
        "#@title Load Final List\n",
        "with open('/content/final_list_single', 'rb') as filepath:\n",
        "         final_list = pickle.load(filepath)\n",
        "\n",
        "final = []\n",
        "\n",
        "\n",
        "count = 129\n",
        "word_index = dict()\n",
        "index_word = dict()\n",
        "\n",
        "for item in final_list:\n",
        "  if item in word_index:\n",
        "     final.append(word_index[item])\n",
        "     continue\n",
        "  if item[0] == 'n':\n",
        "    final.append(int(item[1:]))\n",
        "    word_index[item] = final[-1]\n",
        "    index_word[final[-1]] = item\n",
        "  else : \n",
        "    final.append(int(count))\n",
        "    word_index[item] = final[-1]\n",
        "    index_word[final[-1]] = item\n",
        "    count += 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6XMNZsBU_R1J",
        "cellView": "form"
      },
      "source": [
        "#@title Define and Initialize Sequence Length\n",
        "sequence_length = 512\n",
        "network_input,network_output = get_data(final_list,sequence_length)\n",
        "print(network_input.shape)\n",
        "print(network_output.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K6YxpJN-_RxK",
        "cellView": "form"
      },
      "source": [
        "#@title Initialize Vocabulary\n",
        "MAX_LENGTH = n_vocab\n",
        "\n",
        "network_in = tokenizer.texts_to_sequences(network_input)\n",
        "network_in = tf.keras.preprocessing.sequence.pad_sequences(network_in,\n",
        "                                                           padding='post')\n",
        "  \n",
        "network_out = tokenizer.texts_to_sequences(network_output)\n",
        "network_out = tf.keras.preprocessing.sequence.pad_sequences(network_out,\n",
        "                                                       padding='post')\n",
        "VOCAB_SIZE =  len(tokenizer.word_index)+1\n",
        "print(VOCAB_SIZE)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cAyesuDK2gHa",
        "cellView": "form"
      },
      "source": [
        "#@title Initialize and Declare the Model\n",
        "# decoder inputs use the previous target as input\n",
        "# remove START_TOKEN from targets\n",
        "\n",
        "group_size = 3\n",
        "data = xl_data(network_in, network_out, group_size)\n",
        "\n",
        "network_in = []\n",
        "network_out = []\n",
        "\n",
        "train_len = int(len(data)*0.7)\n",
        "\n",
        "training_data = data[:train_len]\n",
        "val_data = data[train_len:]\n",
        "\n",
        "# declare model\n",
        "model = TransformerXL(\n",
        "       vocab_size=VOCAB_SIZE, \n",
        "       checkpoint=None,\n",
        "       is_training=True)\n",
        "model.summary()\n",
        "VOCAB_SIZE\n",
        "training_data.shape\n",
        "tokenizer.word_index"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qQt_qkEtUnoE"
      },
      "source": [
        "# Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "75kBf9gT2h6I",
        "cellView": "form"
      },
      "source": [
        "#@title Train the Model\n",
        "# train\n",
        "model.train(training_data, output_checkpoint_folder='/content')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UBdOFEXHQmML"
      },
      "source": [
        "# Generate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cTsVtAMsQ0EB",
        "cellView": "form"
      },
      "source": [
        "#@title Declare Prediction Model (Run only once)\n",
        "# Predict\n",
        "model_p = TransformerXL(\n",
        "       #vocab_size=107,\n",
        "       vocab_size=VOCAB_SIZE, \n",
        "       checkpoint='/content/model-000-0.279',\n",
        "       is_training=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "US_vZodwaFQX",
        "cellView": "form"
      },
      "source": [
        "#@title Generate Output\n",
        "notes = []\n",
        "num = np.random.randint(len(val_data)-(2*sequence_length))\n",
        "#num = 160\n",
        "num_seq = 1\n",
        "lenxx = 512\n",
        "for i in range(num_seq):\n",
        "    notes.append(val_data[num+int(i*sequence_length/3), i, 0, :])\n",
        "notes = np.array(notes)[0][-lenxx:]\n",
        "notes = notes.flatten()\n",
        "notes = np.reshape(notes, (1, len(notes)))\n",
        "\n",
        "notes.shape\n",
        "\n",
        "len(notes[0])\n",
        "\n",
        "network_input[train_len + (num*3)]\n",
        "\n",
        "flag = False\n",
        "final_output = []\n",
        "#oooh = []\n",
        "lens_in = len(notes[0])\n",
        "shift = 1\n",
        "num_times = 2\n",
        "num_notes = 1576\n",
        "#num_notes = sequence_length-lenxx\n",
        "k = 3\n",
        "print('num_notes : ',num_notes)\n",
        "\n",
        "\n",
        "final_output = []\n",
        "for i in range(1):\n",
        "    print(\"########################################################################## : \",i)\n",
        "    print('lens_in : ',lens_in)\n",
        "    output = model_p.predict(notes, num_notes, k,\n",
        "                             strategies=['temperature', 'nucleus'],\n",
        "                             use_structure=True)\n",
        "    lens = len(output)\n",
        "    notes_temp = []\n",
        "    count = 0\n",
        "    print(enumerate(output))\n",
        "    for index, j in enumerate(output):\n",
        "        \n",
        "        if index >= (lens_in)-shift:\n",
        "           print(' j : ',j,' index : ',index)\n",
        "           if i == 0 and flag == False:\n",
        "              final_output.append(j)\n",
        "              flag = True\n",
        "              count = 0\n",
        "              notes_temp.append(tokenizer.word_index[j])\n",
        "           elif (final_output[-1])[0] == 'n' and (j)[0] == 'd':\n",
        "              final_output.append(j)\n",
        "              count = 0\n",
        "              notes_temp.append(tokenizer.word_index[j])\n",
        "           elif (final_output[-1])[0] == 'd' and ((j)[0] == 'i' or (j) == 'xxni'):\n",
        "              final_output.append(j)\n",
        "              count = 0\n",
        "              notes_temp.append(tokenizer.word_index[j])\n",
        "           elif ((final_output[-1])[0] == 'i' or (final_output[-1]) == 'xxni') and ((j)[0] == 'n' or (j) == 'xxsep'):\n",
        "              final_output.append(j)\n",
        "              count = 0\n",
        "              notes_temp.append(tokenizer.word_index[j])\n",
        "           elif (final_output[-1]) == 'xxsep' and ((j)[0] == 'd' or (j)[0] == 'n') and count <= 1:\n",
        "              final_output.append(j)\n",
        "              count += 1\n",
        "              notes_temp.append(tokenizer.word_index[j])\n",
        "\n",
        "    notes = np.array(notes_temp)[-lenxx:]\n",
        "    lens_in = len(notes)\n",
        "    notes = np.reshape(notes, (1, len(notes)))\n",
        "    print('\\nlast : ', final_output[-1])\n",
        "    print('\\nlen : ', lens_in)\n",
        "    print(\"The iteration output : \",final_output,'\\n')\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "store_ = final_output\n",
        "temps = network_input[train_len + (num*3)] + network_input[train_len + (num*3) + 640] + network_input[train_len + (num*3) + 1280]\n",
        "#temps = network_input[train_len + (num*3)]\n",
        "notes_in = []\n",
        "temp = '' \n",
        "for sentence in temps:\n",
        "  for i in sentence:\n",
        "      if i != ' ':\n",
        "        temp += i \n",
        "      else:\n",
        "        notes_in.append(temp)\n",
        "        temp = ''  \n",
        "\n",
        "len(notes_in)\n",
        "\n",
        "#final_output = notes_in + store_\n",
        "#final_output = store_\n",
        "final_output = notes_in\n",
        "print(len(final_output))\n",
        "\n",
        "npenc_out = []    \n",
        "\n",
        "pred = []\n",
        "\n",
        "for i in final_output:\n",
        "  if i != 'xxeos' and i != 'xxpad' and i != 'xxbos':\n",
        "    pred.append(i)\n",
        "\n",
        "\n",
        "while(pred[0][0] == 'd' or pred[0][0] == 'i' or pred[0] == 'xxni'):\n",
        "    pred = pred[1:]\n",
        "\n",
        "print(pred)\n",
        "\n",
        "npenc_out = []    \n",
        "x = 0\n",
        "y = len(pred)\n",
        "\n",
        "for i in range(x,y,3):\n",
        "    x = i\n",
        "    if not( x+1<y and x+2<y):\n",
        "      continue\n",
        "\n",
        "    temp = []\n",
        "    if(pred[x] == 'xxsep'):\n",
        "        temp.append(-1)\n",
        "    elif(pred[x] == 'xxni'):\n",
        "        temp.append(-2)\n",
        "    else:\n",
        "        temp.append(int(pred[x][1:]))\n",
        "\n",
        "          \n",
        "    if(pred[x+1] == 'xxsep'):\n",
        "        temp.append(-1)\n",
        "    elif(pred[x+1] == 'xxni'):\n",
        "        temp.append(-2)\n",
        "    else:\n",
        "        temp.append(int(pred[x+1][1:]))\n",
        "           \n",
        "    if(pred[x+2] == 'xxsep'):\n",
        "        temp.append(-1)\n",
        "    elif(pred[x+2] == 'xxni'):\n",
        "        temp.append(-2)\n",
        "    else:\n",
        "        temp.append(int(pred[x+2][1:]))\n",
        "    npenc_out.append(temp)\n",
        "        \n",
        "print(npenc_out)\n",
        "\n",
        "npenc_out_2 = []\n",
        "for i in npenc_out:\n",
        "  if not(i[0] != -1 and i[2] == -2):\n",
        "    npenc_out_2.append(i)\n",
        "\n",
        "len(npenc_out_2)\n",
        "\n",
        "s = npenc2stream(npenc_out_2, rev_uniq_ins, 120)\n",
        "s.write('midi', fp='/content/output2.mid')\n",
        "\n",
        "rev_uniq_ins\n",
        "\n",
        "npenc_out_1 = []\n",
        "for i in npenc_out:\n",
        "  if i[0] == -1 and i[1]>=3:\n",
        "    print(i)\n",
        "    i[1] = 2\n",
        "  if not(i[0] != -1 and i[2] == -2):\n",
        "    npenc_out_1.append(i)\n",
        "\n",
        "s = npenc2stream(npenc_out_1,rev_uniq_ins,120)\n",
        "s.write('midi', fp='/content/output1.mid')\n",
        "\n",
        "#npenc_out[:]  "
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}