diff --git a/samples/ServalApp/README.md b/samples/ServalApp/README.md new file mode 100644 index 00000000..ae37df99 --- /dev/null +++ b/samples/ServalApp/README.md @@ -0,0 +1,13 @@ +### Running the Serval APP +Before running the app, verify that both `SERVAL_APP_EMAIL_PASSWORD` and `SERVAL_APP_PASSCODE` are appropriately populated. +Then, run: +``` +streamlit run serval_app.py +``` + +### Regenerating the Python Client +When the Serval API is updated, use the tool [swagger-to](https://pypi.org/project/swagger-to/) to generate a new `serval_client_module.py` using the following command: +``` +swagger_to_py_client.py --swagger_path path/to/swagger.json --outpath serval_client_module.py +``` +Note: You may need to delete the authorization-related elements of the "swagger.json" before generating. \ No newline at end of file diff --git a/samples/ServalApp/REAME.md b/samples/ServalApp/REAME.md deleted file mode 100644 index 37b195be..00000000 --- a/samples/ServalApp/REAME.md +++ /dev/null @@ -1,6 +0,0 @@ -### Running the Serval APP -Before running the app, verify that both `SERVAL_APP_EMAIL_PASSWORD` and `SERVAL_APP_PASSCODE` are appropriately populated. -Then, run: -``` -./start_app.sh -``` \ No newline at end of file diff --git a/samples/ServalApp/builds.db b/samples/ServalApp/builds.db index 101600b0..d0e91a7f 100644 Binary files a/samples/ServalApp/builds.db and b/samples/ServalApp/builds.db differ diff --git a/samples/ServalApp/db.py b/samples/ServalApp/db.py index 97f97ab7..ab59bbce 100644 --- a/samples/ServalApp/db.py +++ b/samples/ServalApp/db.py @@ -24,10 +24,7 @@ def __str__(self): def __repr__(self): return self.__str__() - -def create_tables(): +def clear_and_regenerate_tables(): engine = create_engine("sqlite:///builds.db") metadata.drop_all(bind=engine) metadata.create_all(bind=engine) - - diff --git a/samples/ServalApp/send_updates.py b/samples/ServalApp/send_updates.py deleted file mode 100644 index 631b33fe..00000000 --- a/samples/ServalApp/send_updates.py +++ /dev/null @@ -1,62 +0,0 @@ -from serval_client_module import * -from serval_auth_module import * -import os -from time import sleep -from db import Build, State -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from serval_email_module import ServalAppEmailServer - -def main(): - def started(build:Build, email_server:ServalAppEmailServer): - print(f"\tStarted {build}") - session.delete(build) - email_server.send_build_started_email(build.email) - session.add(Build(build_id=build.build_id, engine_id=build.engine_id, email=build.email, state=State.Active, corpus_id=build.corpus_id)) - - def faulted(build:Build, email_server:ServalAppEmailServer): - print(f"\tFaulted {build}") - session.delete(build) - email_server.send_build_faulted_email(build.email) - - def completed(build:Build, email_server:ServalAppEmailServer): - print(f"\tCompleted {build}") - session.delete(build) - pretranslations = client.translation_engines_get_all_pretranslations(build.engine_id, build.corpus_id) - email_server.send_build_completed_email(build.email, '\n'.join([f"{'|'.join(pretranslation.refs)}\t{pretranslation.translation}" for pretranslation in pretranslations])) - - def update(build:Build, email_server:ServalAppEmailServer): - print(f"\tUpdated {build}") - - serval_auth = ServalBearerAuth() - client = RemoteCaller(url_prefix="http://localhost",auth=serval_auth) - responses:"dict[str,function]" = {"Completed":completed, "Faulted":faulted, "Canceled":faulted} - - engine = create_engine("sqlite:///builds.db") - Session = sessionmaker(bind=engine) - session = Session() - - def get_update(build:Build, email_server:ServalAppEmailServer): - build_update = client.translation_engines_get_build(id=build.engine_id, build_id=build.build_id) - if build.state == State.Pending and build_update.state == "Active": - started(build, email_server) - else: - responses.get(build_update.state, update)(build, email_server) - session.commit() - - def send_updates(email_server:ServalAppEmailServer): - print(f"Checking for updates:") - builds = session.query(Build).all() - for build in builds: - try: - get_update(build, email_server) - except Exception as e: - print(f"\tFailed to update {build} because of exception {e}") - sleep(60) - - with ServalAppEmailServer(os.environ.get('SERVAL_APP_EMAIL_PASSWORD')) as email_server: - while(True): - send_updates(email_server) - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/samples/ServalApp/serval_app.py b/samples/ServalApp/serval_app.py index 0acb2c50..7e6e1e8e 100644 --- a/samples/ServalApp/serval_app.py +++ b/samples/ServalApp/serval_app.py @@ -1,10 +1,78 @@ import streamlit as st +from streamlit.runtime.scriptrunner import add_script_run_ctx from serval_client_module import * from serval_auth_module import * from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from db import Build from time import sleep +from threading import Thread +import os +from db import Build, State +from serval_email_module import ServalAppEmailServer +import re + +def send_emails(): + engine = create_engine("sqlite:///builds.db") + Session = sessionmaker(bind=engine) + session = Session() + try: + def started(build:Build, email_server:ServalAppEmailServer, data=None): + print(f"\tStarted {build}") + session.delete(build) + email_server.send_build_started_email(build.email) + session.add(Build(build_id=build.build_id, engine_id=build.engine_id, email=build.email, state=State.Active, corpus_id=build.corpus_id)) + + def faulted(build:Build, email_server:ServalAppEmailServer, data=None): + print(f"\tFaulted {build}") + session.delete(build) + email_server.send_build_faulted_email(build.email, error=data) + + def completed(build:Build, email_server:ServalAppEmailServer, data=None): + print(f"\tCompleted {build}") + session.delete(build) + pretranslations = client.translation_engines_get_all_pretranslations(build.engine_id, build.corpus_id) + email_server.send_build_completed_email(build.email, '\n'.join([f"{'|'.join(pretranslation.refs)}\t{pretranslation.translation}" for pretranslation in pretranslations])) + + def update(build:Build, email_server:ServalAppEmailServer, data=None): + print(f"\tUpdated {build}") + + serval_auth = ServalBearerAuth() + client = RemoteCaller(url_prefix="http://localhost",auth=serval_auth) + responses:"dict[str,function]" = {"Completed":completed, "Faulted":faulted, "Canceled":faulted} + + def get_update(build:Build, email_server:ServalAppEmailServer): + build_update = client.translation_engines_get_build(id=build.engine_id, build_id=build.build_id) + if build.state == State.Pending and build_update.state == "Active": + started(build, email_server) + else: + responses.get(build_update.state, update)(build, email_server, build_update.message) + session.commit() + + def send_updates(email_server:ServalAppEmailServer): + print(f"Checking for updates...") + with session.no_autoflush: + builds = session.query(Build).all() + for build in builds: + try: + get_update(build, email_server) + except Exception as e: + print(f"\tFailed to update {build} because of exception {e}") + raise e + + with ServalAppEmailServer(os.environ.get('SERVAL_APP_EMAIL_PASSWORD')) as email_server: + while(True): + send_updates(email_server) + sleep(300) #Once every five minutes... + except Exception as e: + print(e) + st.session_state['background_process_has_started'] = False + +if not st.session_state.get('background_process_has_started',False): + cron_thread = Thread(target=send_emails) + add_script_run_ctx(cron_thread) + cron_thread.start() + st.session_state['background_process_has_started'] = True serval_auth = None if not st.session_state.get('authorized',False): @@ -30,19 +98,19 @@ def submit(): engine = json.loads(client.translation_engines_create(TranslationEngineConfig(source_language=st.session_state['source_language'],target_language=st.session_state['target_language'],type='Nmt',name=f'serval_app_engine:{st.session_state["email"]}'))) - source_file = json.loads(client.data_files_create(st.session_state['source_file'], format="Text")) - target_file = json.loads(client.data_files_create(st.session_state['target_file'], format="Text")) + source_files = [json.loads(client.data_files_create(st.session_state['source_files'][i], format="Paratext" if st.session_state['source_files'][i].name[-4:] == '.zip' else "Text")) for i in range(len(st.session_state['source_files']))] + target_files = [json.loads(client.data_files_create(st.session_state['target_files'][i], format="Paratext" if st.session_state['target_files'][i].name[-4:] == '.zip' else "Text")) for i in range(len(st.session_state['target_files']))] corpus = json.loads(client.translation_engines_add_corpus( engine['id'], TranslationCorpusConfig( - source_files=[TranslationCorpusFileConfig(file_id=source_file['id'], text_id=st.session_state['source_file'].name)], - target_files=[TranslationCorpusFileConfig(file_id=target_file['id'], text_id=st.session_state['source_file'].name)], + source_files=[TranslationCorpusFileConfig(file_id=file['id'], text_id=name) for file, name in zip(source_files, list(map(lambda f: f.name, st.session_state['source_files'])))], + target_files=[TranslationCorpusFileConfig(file_id=file['id'], text_id=name) for file, name in zip(target_files, list(map(lambda f: f.name, st.session_state['target_files'])))], source_language=st.session_state['source_language'], target_language=st.session_state['target_language'] ) ) ) - build = json.loads(client.translation_engines_start_build(engine['id'], TranslationBuildConfig(pretranslate=[PretranslateCorpusConfig(corpus_id=corpus["id"], text_ids=[st.session_state['source_file'].name])]))) + build = json.loads(client.translation_engines_start_build(engine['id'], TranslationBuildConfig(pretranslate=[PretranslateCorpusConfig(corpus_id=corpus["id"], text_ids= [] if st.session_state['source_files'][0].name[-4:] == '.zip' else list(map(lambda f: f.name, st.session_state['source_files'])))], options="{\"max_steps\":10}"))) session.add(Build(build_id=build['id'],engine_id=engine['id'],email=st.session_state['email'],state=build['state'],corpus_id=corpus['id'])) session.commit() @@ -55,23 +123,28 @@ def already_active_build_for(email:str): with st.form(key="NmtTranslationForm"): st.session_state['source_language'] = st.text_input(label="Source language tag*", placeholder="en") if st.session_state.get('source_language','') == '' and tried_to_submit: - st.warning("Please enter a source language tag before submitting", icon='⬆️') + st.error("Please enter a source language tag before submitting", icon='⬆️') - st.session_state['source_file'] = st.file_uploader(label="Source File") - if st.session_state.get('source_file',None) is None and tried_to_submit: - st.warning("Please upload a source file before submitting", icon='⬆️') + st.session_state['source_files'] = st.file_uploader(label="Source File(s)", accept_multiple_files=True) + if len(st.session_state.get('source_files',[])) == 0 and tried_to_submit: + st.error("Please upload a source file before submitting", icon='⬆️') + if len(st.session_state.get('source_files',[])) > 1: + st.warning('Please note that source and target text files will be paired together by file name', icon='💡') st.session_state['target_language'] = st.text_input(label="Target language tag*", placeholder="es") if st.session_state.get('target_language','') == '' and tried_to_submit: - st.warning("Please enter a target language tag before submitting", icon='⬆️') + st.error("Please enter a target language tag before submitting", icon='⬆️') - st.session_state['target_file'] = st.file_uploader(label="Target File") - if st.session_state.get('target_file',None) is None and tried_to_submit: - st.warning("Please upload a target file before submitting", icon='⬆️') + st.session_state['target_files'] = st.file_uploader(label="Target File(s)", accept_multiple_files=True) + if len(st.session_state.get('target_files',[])) > 1: + st.warning('Please note that source and target text files will be paired together by file name', icon='💡') st.session_state['email'] = st.text_input(label="Email", placeholder="johndoe@example.com") if st.session_state.get('email','') == '' and tried_to_submit: - st.warning("Please enter an email address", icon='⬆️') + st.error("Please enter an email address", icon='⬆️') + elif not re.match(r"^\S+@\S+\.\S+$", st.session_state['email']) and tried_to_submit: + st.error("Please enter a valid email address", icon='⬆️') + st.session_state['email'] = '' if tried_to_submit: st.error(st.session_state.get('error',"Something went wrong. Please try again in a moment.")) if st.form_submit_button("Generate translations"): @@ -79,8 +152,9 @@ def already_active_build_for(email:str): st.session_state['tried_to_submit'] = True st.session_state['error'] = "There is already an a pending or active build associated with this email address. Please wait for the previous build to finish." st.rerun() - elif st.session_state['source_language'] != '' and st.session_state['target_language'] != '' and st.session_state['source_file'] is not None and st.session_state['target_file'] is not None and st.session_state['email'] != '': - submit() + elif st.session_state['source_language'] != '' and st.session_state['target_language'] != '' and len(st.session_state['source_files']) > 0 and st.session_state['email'] != '': + with st.spinner(): + submit() st.session_state['tried_to_submit'] = False st.toast("Translations are on their way! You'll receive an email when your translation job has begun.") sleep(4) diff --git a/samples/ServalApp/serval_email_module.py b/samples/ServalApp/serval_email_module.py index d61a20a9..5876e622 100644 --- a/samples/ServalApp/serval_email_module.py +++ b/samples/ServalApp/serval_email_module.py @@ -8,11 +8,11 @@ def __init__(self, password, sender_address = 'serval-app@languagetechnology.org self.host = host self.port = port self.server = None - + @property def password(self): return len(self.__password)*"*" - + def __enter__(self): context = ssl.create_default_context() self.server = smtplib.SMTP_SSL(host=self.host, port=self.port, context=context) @@ -21,38 +21,36 @@ def __enter__(self): def __exit__(self, *args): self.server.close() - + def send_build_completed_email(self, recipient_address:str, pretranslations_file_data:str): msg = EmailMessage() msg.set_content( - ''' - Hi! +'''Hi! - Your NMT engine has completed building. Attached are the translations of untranslated source text in the files you included. +Your NMT engine has completed building. Attached are the translations of untranslated source text in the files you included. - If you are experiencing difficulties using this application, please contact eli_lowry@sil.org. - - Thank you! - ''' +If you are experiencing difficulties using this application, please contact eli_lowry@sil.org. + +Thank you! +''' ) msg['From'] = self.sender_address msg['To'] = recipient_address msg['Subject'] = 'Your NMT build job is complete!' msg.add_attachment(pretranslations_file_data, filename='translations.txt') self.server.send_message(msg) - - def send_build_faulted_email(self, recipient_address:str): + + def send_build_faulted_email(self, recipient_address:str, error=""): msg = EmailMessage() - msg.add_attachment( - ''' - Hi! - - Your NMT engine has failed to build. Please make sure the information you specified is correct and try again after a while. - - If you continue to experience difficulties using this application, please contact eli_lowry@sil.org. - - Thank you! - ''' + msg.set_content( +f'''Hi! + +Your NMT engine has failed to build{" with the following error message: " + error if error != "" else ""}. Please make sure the information you specified is correct and try again after a while. + +If you continue to experience difficulties using this application, please contact eli_lowry@sil.org. + +Thank you! +''' ) msg['From'] = self.sender_address msg['To'] = recipient_address @@ -62,15 +60,14 @@ def send_build_faulted_email(self, recipient_address:str): def send_build_started_email(self, recipient_address:str): msg = EmailMessage() msg.set_content( - ''' - Hi! - - Your NMT engine has started building. We will contact you when it is complete. - - If you are experiencing difficulties using this application, please contact eli_lowry@sil.org. - - Thank you! - ''' +'''Hi! + +Your NMT engine has started building. We will contact you when it is complete. + +If you are experiencing difficulties using this application, please contact eli_lowry@sil.org. + +Thank you! +''' ) msg['From'] = self.sender_address msg['To'] = recipient_address diff --git a/samples/ServalApp/start_app.sh b/samples/ServalApp/start_app.sh deleted file mode 100755 index f65c7d7f..00000000 --- a/samples/ServalApp/start_app.sh +++ /dev/null @@ -1,4 +0,0 @@ -python3 send_updates.py & -SEND_UPDATES_PID=$! -streamlit run serval_app.py -kill $SEND_UPDATES_PID \ No newline at end of file