From b1f9236bb6ea0a89193477b3bec93f3efb095dec Mon Sep 17 00:00:00 2001 From: Ray Smets Date: Thu, 23 Nov 2023 01:21:43 -0800 Subject: [PATCH 1/8] [Setup] script. (#1) --- README.md | 7 +++++++ setup.sh | 16 ++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100755 setup.sh diff --git a/README.md b/README.md index c10bdcb..1c9b150 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,13 @@ Make a new voice in Eleven and get the voice id of that voice using their [get v export ELEVENLABS_VOICE_ID= ``` +### Setup Script + +Alternatively, one can use the `setup.sh` script to facilitate getting the shell envs ready to rock by updating the API key values in `setup.sh` and run. + +_Note: may have to manually run `source source venv/bin/activate` afterwards depending on shell env._ + + ## Run it! In on terminal, run the webcam capture: diff --git a/setup.sh b/setup.sh new file mode 100755 index 0000000..bab016d --- /dev/null +++ b/setup.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# create a virtual environment +python3 -m pip install virtualenv +python3 -m virtualenv venv + +# source the virtual environment +source venv/bin/activate + +# install the dependencies +pip install -r requirements.txt + +# set the environment variables +export ELEVENLABS_VOICE_ID= +export OPENAI_API_KEY= +export ELEVENLABS_API_KEY= \ No newline at end of file From 4ab05a4b1d13dab4e047e000e78d9c897d02467d Mon Sep 17 00:00:00 2001 From: Ray Smets Date: Thu, 23 Nov 2023 01:22:52 -0800 Subject: [PATCH 2/8] [Narrator] prompt to describe the image like David Attenborough for increased complex descriptors. (#2) --- narrator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/narrator.py b/narrator.py index cd086f7..845158f 100644 --- a/narrator.py +++ b/narrator.py @@ -43,7 +43,7 @@ def generate_new_line(base64_image): { "role": "user", "content": [ - {"type": "text", "text": "Describe this image"}, + {"type": "text", "text": "Describe this image as if you David Attenborough"}, { "type": "image_url", "image_url": f"data:image/jpeg;base64,{base64_image}", From 1bb728ada311c0892ac18f61718e6538279a3192 Mon Sep 17 00:00:00 2001 From: Ray Smets Date: Thu, 23 Nov 2023 01:45:28 -0800 Subject: [PATCH 3/8] [Narrator] fix --- narrator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/narrator.py b/narrator.py index 845158f..7eca4f5 100644 --- a/narrator.py +++ b/narrator.py @@ -43,7 +43,7 @@ def generate_new_line(base64_image): { "role": "user", "content": [ - {"type": "text", "text": "Describe this image as if you David Attenborough"}, + {"type": "text", "text": "Describe this image as if you are David Attenborough"}, { "type": "image_url", "image_url": f"data:image/jpeg;base64,{base64_image}", From a4847a83450bd6a3a8fc03f19679e8e15e52fce0 Mon Sep 17 00:00:00 2001 From: Ray Smets Date: Thu, 23 Nov 2023 15:06:17 -0800 Subject: [PATCH 4/8] [Narrator] streaming (#3) --- .gitignore | 3 ++- README.md | 23 ++++++++++++++++------- narrator.py | 32 ++++++++++++++++++++++++++------ setup.sh | 4 +++- 4 files changed, 47 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 4d9cf0b..825f964 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ /venv /narration /frames/* -!/frames/.gitkeep \ No newline at end of file +!/frames/.gitkeep +.env \ No newline at end of file diff --git a/README.md b/README.md index 1c9b150..b2c7f25 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,9 @@ -# David Attenborough narrates your life. +# David Attenborough narrates your life. https://twitter.com/charliebholtz/status/1724815159590293764 ## Want to make your own AI app? + Check out [Replicate](https://replicate.com). We make it easy to run machine learning models with an API. ## Setup @@ -20,33 +21,41 @@ Then, install the dependencies: Make a [Replicate](https://replicate.com), [OpenAI](https://beta.openai.com/), and [ElevenLabs](https://elevenlabs.io) account and set your tokens: -``` +```bash export OPENAI_API_KEY= export ELEVENLABS_API_KEY= ``` Make a new voice in Eleven and get the voice id of that voice using their [get voices](https://elevenlabs.io/docs/api-reference/voices) API, or by clicking the flask icon next to the voice in the VoiceLab tab. -``` +```bash export ELEVENLABS_VOICE_ID= ``` -### Setup Script +### Streaming -Alternatively, one can use the `setup.sh` script to facilitate getting the shell envs ready to rock by updating the API key values in `setup.sh` and run. +If you would like the speech to start quicker via a streaming manner set the environment variable to enable. The concession is that the audio snippet is not saved in the `/narration` directory. -_Note: may have to manually run `source source venv/bin/activate` afterwards depending on shell env._ +```bash +export ELEVENLABS_STREAMING=true +``` + +### Script +Alternative to running the commands above individually, one can use the `setup.sh` script to facilitate getting the two required shell envs ready to rock by updating the environment variable values in `setup.sh` and executing the script. + +_Note: may have to manually run `source source venv/bin/activate` afterwards depending on shell env._ ## Run it! In on terminal, run the webcam capture: + ```bash python capture.py ``` + In another terminal, run the narrator: ```bash python narrator.py ``` - diff --git a/narrator.py b/narrator.py index 7eca4f5..d33da74 100644 --- a/narrator.py +++ b/narrator.py @@ -1,16 +1,24 @@ -import os -from openai import OpenAI import base64 +import errno import json +import os import time + import simpleaudio as sa -import errno -from elevenlabs import generate, play, set_api_key, voices +from elevenlabs import generate, play, set_api_key, stream, voices +from openai import OpenAI client = OpenAI() set_api_key(os.environ.get("ELEVENLABS_API_KEY")) + +# This code initializes the variable 'isStreaming' based on the value of the environment variable 'ELEVENLABS_STREAMIMAGES'. +# If the value of 'ELEVENLABS_STREAMIMAGES' is "true", then 'isStreaming' is set to True. +# Otherwise, 'isStreaming' is set to False. +isStreaming = os.environ.get("ELEVENLABS_STREAMING", "false") == "true" + + def encode_image(image_path): while True: try: @@ -25,7 +33,16 @@ def encode_image(image_path): def play_audio(text): - audio = generate(text, voice=os.environ.get("ELEVENLABS_VOICE_ID")) + audio = generate( + text, + voice=os.environ.get("ELEVENLABS_VOICE_ID"), + model="eleven_turbo_v2", + stream=isStreaming, + ) + + if isStreaming: + stream(audio) + return unique_id = base64.urlsafe_b64encode(os.urandom(30)).decode("utf-8").rstrip("=") dir_path = os.path.join("narration", unique_id) @@ -43,7 +60,10 @@ def generate_new_line(base64_image): { "role": "user", "content": [ - {"type": "text", "text": "Describe this image as if you are David Attenborough"}, + { + "type": "text", + "text": "Describe this image as if you are David Attenborough", + }, { "type": "image_url", "image_url": f"data:image/jpeg;base64,{base64_image}", diff --git a/setup.sh b/setup.sh index bab016d..823a544 100755 --- a/setup.sh +++ b/setup.sh @@ -13,4 +13,6 @@ pip install -r requirements.txt # set the environment variables export ELEVENLABS_VOICE_ID= export OPENAI_API_KEY= -export ELEVENLABS_API_KEY= \ No newline at end of file +export ELEVENLABS_API_KEY= + +export ELEVENLABS_STREAMING=false From daa9bb6b17126502573a58283c254358466471ff Mon Sep 17 00:00:00 2001 From: Ray Smets Date: Thu, 23 Nov 2023 17:12:16 -0800 Subject: [PATCH 5/8] spacebar --- narrator.py | 71 +++++++++++++++++++++++++++++++++++++++--------- requirements.txt | 3 +- 2 files changed, 60 insertions(+), 14 deletions(-) diff --git a/narrator.py b/narrator.py index d33da74..40b12fa 100644 --- a/narrator.py +++ b/narrator.py @@ -7,11 +7,35 @@ import simpleaudio as sa from elevenlabs import generate, play, set_api_key, stream, voices from openai import OpenAI +from pynput import keyboard client = OpenAI() set_api_key(os.environ.get("ELEVENLABS_API_KEY")) +script = [] + + +def on_press(key): + print(f"Key {key} pressed.") + if key == keyboard.Key.space: + _main() + + +def on_release(key): + print(f"Key {key} released.") + + if key == keyboard.Key.esc: + # Stop listener + return False + + +# Create a listener +listener = keyboard.Listener(on_press=on_press, on_release=on_release) + +# Start the listener +listener.start() + # This code initializes the variable 'isStreaming' based on the value of the environment variable 'ELEVENLABS_STREAMIMAGES'. # If the value of 'ELEVENLABS_STREAMIMAGES' is "true", then 'isStreaming' is set to True. @@ -93,29 +117,50 @@ def analyze_image(base64_image, script): return response_text +def _main(): + global script + # path to your image + image_path = os.path.join(os.getcwd(), "./frames/frame.jpg") + + # getting the base64 encoding + base64_image = encode_image(image_path) + + # analyze posture + print("👀 David is watching...") + analysis = analyze_image(base64_image, script=script) + + print("🎙️ David says:") + print(analysis) + + play_audio(analysis) + + script = script + [{"role": "assistant", "content": analysis}] + + def main(): - script = [] + # script = [] while True: + pass # path to your image - image_path = os.path.join(os.getcwd(), "./frames/frame.jpg") + # image_path = os.path.join(os.getcwd(), "./frames/frame.jpg") - # getting the base64 encoding - base64_image = encode_image(image_path) + # # getting the base64 encoding + # base64_image = encode_image(image_path) - # analyze posture - print("👀 David is watching...") - analysis = analyze_image(base64_image, script=script) + # # analyze posture + # print("👀 David is watching...") + # analysis = analyze_image(base64_image, script=script) - print("🎙️ David says:") - print(analysis) + # print("🎙️ David says:") + # print(analysis) - play_audio(analysis) + # play_audio(analysis) - script = script + [{"role": "assistant", "content": analysis}] + # script = script + [{"role": "assistant", "content": analysis}] - # wait for 5 seconds - time.sleep(5) + # # wait for 5 seconds + # time.sleep(5) if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt index 12cae1c..0f145b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,6 +28,7 @@ pure-eval==0.2.2 pydantic==2.4.2 pydantic_core==2.10.1 Pygments==2.16.1 +pynput==1.7.6 requests==2.31.0 simpleaudio==1.0.4 six==1.16.0 @@ -38,4 +39,4 @@ traitlets==5.13.0 typing_extensions==4.8.0 urllib3==2.0.7 wcwidth==0.2.10 -websockets==12.0 +websockets==12.0 \ No newline at end of file From eb654f1f7702c6378cdaf65b957011135011313e Mon Sep 17 00:00:00 2001 From: Ray Smets Date: Thu, 23 Nov 2023 20:39:26 -0800 Subject: [PATCH 6/8] trunk --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 825f964..6667455 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ /narration /frames/* !/frames/.gitkeep -.env \ No newline at end of file +.trunk \ No newline at end of file From 1a70805f9ddb2f7631d172853ddc5d730669d11a Mon Sep 17 00:00:00 2001 From: Ray Smets Date: Fri, 24 Nov 2023 09:05:32 -0800 Subject: [PATCH 7/8] cleanup --- capture.py | 9 ++++---- narrator.py | 62 +++++++++++++++++++++-------------------------------- 2 files changed, 30 insertions(+), 41 deletions(-) diff --git a/capture.py b/capture.py index bc9845c..71349bd 100644 --- a/capture.py +++ b/capture.py @@ -1,8 +1,9 @@ -import cv2 +import os import time -from PIL import Image + +import cv2 import numpy as np -import os +from PIL import Image # Folder folder = "frames" @@ -30,7 +31,7 @@ # Resize the image max_size = 250 ratio = max_size / max(pil_img.size) - new_size = tuple([int(x*ratio) for x in pil_img.size]) + new_size = tuple([int(x * ratio) for x in pil_img.size]) resized_img = pil_img.resize(new_size, Image.LANCZOS) # Convert the PIL image back to an OpenCV image diff --git a/narrator.py b/narrator.py index 40b12fa..3a652c3 100644 --- a/narrator.py +++ b/narrator.py @@ -1,30 +1,33 @@ import base64 import errno -import json import os import time -import simpleaudio as sa -from elevenlabs import generate, play, set_api_key, stream, voices +from elevenlabs import generate, play, set_api_key, stream from openai import OpenAI -from pynput import keyboard +from pynput import ( # Using pynput to listen for a keypress instead of native keyboard module which was requiring admin privileges + keyboard, +) client = OpenAI() set_api_key(os.environ.get("ELEVENLABS_API_KEY")) +# Initializes the variables based their respective environment variable values, defaulting to false +isStreaming = os.environ.get("ELEVENLABS_STREAMING", "false") == "true" +isPhotoBooth = os.environ.get("PHOTOBOOTH_MODE", "false") == "true" + script = [] +narrator = "Sir David Attenborough" def on_press(key): - print(f"Key {key} pressed.") if key == keyboard.Key.space: + # When space bar is pressed, run the main function which analyzes the image and generates the audio _main() def on_release(key): - print(f"Key {key} released.") - if key == keyboard.Key.esc: # Stop listener return False @@ -37,12 +40,6 @@ def on_release(key): listener.start() -# This code initializes the variable 'isStreaming' based on the value of the environment variable 'ELEVENLABS_STREAMIMAGES'. -# If the value of 'ELEVENLABS_STREAMIMAGES' is "true", then 'isStreaming' is set to True. -# Otherwise, 'isStreaming' is set to False. -isStreaming = os.environ.get("ELEVENLABS_STREAMING", "false") == "true" - - def encode_image(image_path): while True: try: @@ -65,9 +62,11 @@ def play_audio(text): ) if isStreaming: + # Stream the audio for more real-time responsiveness stream(audio) return + # Save the audio to a file and play it unique_id = base64.urlsafe_b64encode(os.urandom(30)).decode("utf-8").rstrip("=") dir_path = os.path.join("narration", unique_id) os.makedirs(dir_path, exist_ok=True) @@ -86,7 +85,7 @@ def generate_new_line(base64_image): "content": [ { "type": "text", - "text": "Describe this image as if you are David Attenborough", + "text": f"Describe this image as if you are {narrator}", }, { "type": "image_url", @@ -103,8 +102,8 @@ def analyze_image(base64_image, script): messages=[ { "role": "system", - "content": """ - You are Sir David Attenborough. Narrate the picture of the human as if it is a nature documentary. + "content": f""" + You are {narrator}. Narrate the picture of the human as if it is a nature documentary. Make it snarky and funny. Don't repeat yourself. Make it short. If I do anything remotely interesting, make a big deal about it! """, }, @@ -119,6 +118,7 @@ def analyze_image(base64_image, script): def _main(): global script + # path to your image image_path = os.path.join(os.getcwd(), "./frames/frame.jpg") @@ -126,7 +126,7 @@ def _main(): base64_image = encode_image(image_path) # analyze posture - print("👀 David is watching...") + print(f"👀 {narrator} is watching...") analysis = analyze_image(base64_image, script=script) print("🎙️ David says:") @@ -138,30 +138,18 @@ def _main(): def main(): - # script = [] - while True: - pass - # path to your image - # image_path = os.path.join(os.getcwd(), "./frames/frame.jpg") - - # # getting the base64 encoding - # base64_image = encode_image(image_path) - - # # analyze posture - # print("👀 David is watching...") - # analysis = analyze_image(base64_image, script=script) - - # print("🎙️ David says:") - # print(analysis) - - # play_audio(analysis) + if isPhotoBooth: + pass + else: + _main() - # script = script + [{"role": "assistant", "content": analysis}] + # wait for 5 seconds + time.sleep(5) - # # wait for 5 seconds - # time.sleep(5) +if isPhotoBooth: + print(f"Press the spacebar to trigger {narrator}") if __name__ == "__main__": main() From b922ac0ffc74a613130d996b85cff56db6fd8e82 Mon Sep 17 00:00:00 2001 From: Ray Smets Date: Fri, 24 Nov 2023 09:13:54 -0800 Subject: [PATCH 8/8] README --- README.md | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b2c7f25..ee102f5 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,6 @@ Make a new voice in Eleven and get the voice id of that voice using their [get v export ELEVENLABS_VOICE_ID= ``` -### Streaming - -If you would like the speech to start quicker via a streaming manner set the environment variable to enable. The concession is that the audio snippet is not saved in the `/narration` directory. - -```bash -export ELEVENLABS_STREAMING=true -``` - ### Script Alternative to running the commands above individually, one can use the `setup.sh` script to facilitate getting the two required shell envs ready to rock by updating the environment variable values in `setup.sh` and executing the script. @@ -59,3 +51,21 @@ In another terminal, run the narrator: ```bash python narrator.py ``` + +## Options + +### Streaming + +If you would like the speech to start quicker via a streaming manner set the environment variable to enable. The concession is that the audio snippet is not saved in the `/narration` directory. + +```bash +export ELEVENLABS_STREAMING=true +``` + +### PhotoBooth + +The default behavior of this app will continually analyze images. If you would like to use in a mode more similar to a photo booth, set the environment variable. In this mode, the image will only be analyzed when the spacebar key is pressed. + +```bash +export PHOTOBOOTH_MODE=true +```