Serval App working MVP compatible with streamlit deployment method

Also now capable of handling paratext projects and multiple files as well as no target file(s).

samples/ServalApp/README.md

@@ -0,0 +1,13 @@
+### Running the Serval APP
+Before running the app, verify that both `SERVAL_APP_EMAIL_PASSWORD` and `SERVAL_APP_PASSCODE` are appropriately populated.
+Then, run:
+```
+streamlit run serval_app.py
+```
+
+### Regenerating the Python Client
+When the Serval API is updated, use the tool [swagger-to](https://pypi.org/project/swagger-to/) to generate a new `serval_client_module.py` using the following command:
+```
+swagger_to_py_client.py --swagger_path path/to/swagger.json --outpath serval_client_module.py
+```
+Note: You may need to delete the authorization-related elements of the "swagger.json" before generating.

samples/ServalApp/db.py

@@ -24,10 +24,7 @@ def __str__(self):
 
     def __repr__(self):
         return self.__str__()
-
-def create_tables():
+def clear_and_regenerate_tables():
     engine = create_engine("sqlite:///builds.db")
     metadata.drop_all(bind=engine)
     metadata.create_all(bind=engine)
-
-

samples/ServalApp/serval_app.py

@@ -1,10 +1,78 @@
 import streamlit as st
+from streamlit.runtime.scriptrunner import add_script_run_ctx
 from serval_client_module import *
 from serval_auth_module import *
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 from db import Build
 from time import sleep
+from threading import Thread
+import os
+from db import Build, State
+from serval_email_module import ServalAppEmailServer
+import re
+
+def send_emails():
+    engine = create_engine("sqlite:///builds.db")
+    Session = sessionmaker(bind=engine)
+    session = Session()
+    try:
+        def started(build:Build, email_server:ServalAppEmailServer, data=None):
+            print(f"\tStarted {build}")
+            session.delete(build)
+            email_server.send_build_started_email(build.email)
+            session.add(Build(build_id=build.build_id, engine_id=build.engine_id, email=build.email, state=State.Active, corpus_id=build.corpus_id))
+
+        def faulted(build:Build, email_server:ServalAppEmailServer, data=None):
+            print(f"\tFaulted {build}")
+            session.delete(build)
+            email_server.send_build_faulted_email(build.email, error=data)
+
+        def completed(build:Build, email_server:ServalAppEmailServer, data=None):
+            print(f"\tCompleted {build}")
+            session.delete(build)
+            pretranslations = client.translation_engines_get_all_pretranslations(build.engine_id, build.corpus_id)
+            email_server.send_build_completed_email(build.email, '\n'.join([f"{'|'.join(pretranslation.refs)}\t{pretranslation.translation}" for pretranslation in pretranslations]))
+
+        def update(build:Build, email_server:ServalAppEmailServer, data=None):
+            print(f"\tUpdated {build}")
+
+        serval_auth = ServalBearerAuth()
+        client = RemoteCaller(url_prefix="http://localhost",auth=serval_auth)
+        responses:"dict[str,function]" = {"Completed":completed, "Faulted":faulted, "Canceled":faulted}
+
+        def get_update(build:Build, email_server:ServalAppEmailServer):
+            build_update = client.translation_engines_get_build(id=build.engine_id, build_id=build.build_id)
+            if build.state == State.Pending and build_update.state == "Active":
+                started(build, email_server)
+            else:
+                responses.get(build_update.state, update)(build, email_server, build_update.message)
+            session.commit()
+
+        def send_updates(email_server:ServalAppEmailServer):
+            print(f"Checking for updates...")
+            with session.no_autoflush:
+                builds = session.query(Build).all()
+                for build in builds:
+                    try:
+                        get_update(build, email_server)
+                    except Exception as e:
+                        print(f"\tFailed to update {build} because of exception {e}")
+                        raise e
+
+        with ServalAppEmailServer(os.environ.get('SERVAL_APP_EMAIL_PASSWORD')) as email_server:
+            while(True):
+                send_updates(email_server)
+                sleep(300) #Once every five minutes...
+    except Exception as e:
+        print(e)
+        st.session_state['background_process_has_started'] = False
+
+if not st.session_state.get('background_process_has_started',False):
+    cron_thread = Thread(target=send_emails)
+    add_script_run_ctx(cron_thread)
+    cron_thread.start()
+    st.session_state['background_process_has_started'] = True
 
 serval_auth = None
 if not st.session_state.get('authorized',False):
@@ -30,19 +98,19 @@
 
     def submit():
         engine = json.loads(client.translation_engines_create(TranslationEngineConfig(source_language=st.session_state['source_language'],target_language=st.session_state['target_language'],type='Nmt',name=f'serval_app_engine:{st.session_state["email"]}')))
-        source_file = json.loads(client.data_files_create(st.session_state['source_file'], format="Text"))
-        target_file = json.loads(client.data_files_create(st.session_state['target_file'], format="Text"))
+        source_files = [json.loads(client.data_files_create(st.session_state['source_files'][i], format="Paratext" if st.session_state['source_files'][i].name[-4:] == '.zip' else "Text")) for i in range(len(st.session_state['source_files']))]
+        target_files = [json.loads(client.data_files_create(st.session_state['target_files'][i], format="Paratext" if st.session_state['target_files'][i].name[-4:] == '.zip' else "Text")) for i in range(len(st.session_state['target_files']))]
         corpus = json.loads(client.translation_engines_add_corpus(
             engine['id'],
             TranslationCorpusConfig(
-                source_files=[TranslationCorpusFileConfig(file_id=source_file['id'], text_id=st.session_state['source_file'].name)],
-                target_files=[TranslationCorpusFileConfig(file_id=target_file['id'], text_id=st.session_state['source_file'].name)],
+                source_files=[TranslationCorpusFileConfig(file_id=file['id'], text_id=name) for file, name in zip(source_files, list(map(lambda f: f.name, st.session_state['source_files'])))],
+                target_files=[TranslationCorpusFileConfig(file_id=file['id'], text_id=name) for file, name in zip(target_files, list(map(lambda f: f.name, st.session_state['target_files'])))],
                 source_language=st.session_state['source_language'],
                 target_language=st.session_state['target_language']
                 )
             )
         )
-        build = json.loads(client.translation_engines_start_build(engine['id'], TranslationBuildConfig(pretranslate=[PretranslateCorpusConfig(corpus_id=corpus["id"], text_ids=[st.session_state['source_file'].name])])))
+        build = json.loads(client.translation_engines_start_build(engine['id'], TranslationBuildConfig(pretranslate=[PretranslateCorpusConfig(corpus_id=corpus["id"], text_ids= [] if st.session_state['source_files'][0].name[-4:] == '.zip' else list(map(lambda f: f.name, st.session_state['source_files'])))], options="{\"max_steps\":10}")))
         session.add(Build(build_id=build['id'],engine_id=engine['id'],email=st.session_state['email'],state=build['state'],corpus_id=corpus['id']))
         session.commit()
 
@@ -55,32 +123,38 @@ def already_active_build_for(email:str):
     with st.form(key="NmtTranslationForm"):
         st.session_state['source_language'] = st.text_input(label="Source language tag*", placeholder="en")
         if st.session_state.get('source_language','') == '' and tried_to_submit:
-            st.warning("Please enter a source language tag before submitting", icon='⬆️')
+            st.error("Please enter a source language tag before submitting", icon='⬆️')
 
-        st.session_state['source_file'] = st.file_uploader(label="Source File")
-        if st.session_state.get('source_file',None) is None and tried_to_submit:
-            st.warning("Please upload a source file before submitting", icon='⬆️')
+        st.session_state['source_files'] = st.file_uploader(label="Source File(s)", accept_multiple_files=True)
+        if len(st.session_state.get('source_files',[])) == 0 and tried_to_submit:
+            st.error("Please upload a source file before submitting", icon='⬆️')
+        if len(st.session_state.get('source_files',[])) > 1:
+            st.warning('Please note that source and target text files will be paired together by file name', icon='💡')
 
         st.session_state['target_language'] = st.text_input(label="Target language tag*", placeholder="es")
         if st.session_state.get('target_language','') == '' and tried_to_submit:
-            st.warning("Please enter a target language tag before submitting", icon='⬆️')
+            st.error("Please enter a target language tag before submitting", icon='⬆️')
 
-        st.session_state['target_file'] = st.file_uploader(label="Target File")
-        if st.session_state.get('target_file',None) is None and tried_to_submit:
-            st.warning("Please upload a target file before submitting", icon='⬆️')
+        st.session_state['target_files'] = st.file_uploader(label="Target File(s)", accept_multiple_files=True)
+        if len(st.session_state.get('target_files',[])) > 1:
+            st.warning('Please note that source and target text files will be paired together by file name', icon='💡')
 
         st.session_state['email'] = st.text_input(label="Email", placeholder="[email protected]")
         if st.session_state.get('email','') == '' and tried_to_submit:
-            st.warning("Please enter an email address", icon='⬆️')
+            st.error("Please enter an email address", icon='⬆️')
+        elif not re.match(r"^\S+@\S+\.\S+$", st.session_state['email']) and tried_to_submit:
+            st.error("Please enter a valid email address", icon='⬆️')
+            st.session_state['email'] = ''
         if tried_to_submit:
             st.error(st.session_state.get('error',"Something went wrong. Please try again in a moment."))
         if st.form_submit_button("Generate translations"):
             if already_active_build_for(st.session_state['email']):
                 st.session_state['tried_to_submit'] = True
                 st.session_state['error'] = "There is already an a pending or active build associated with this email address. Please wait for the previous build to finish."
                 st.rerun()
-            elif st.session_state['source_language'] != '' and st.session_state['target_language'] != '' and st.session_state['source_file'] is not None and st.session_state['target_file'] is not None and st.session_state['email'] != '':
-                submit()
+            elif st.session_state['source_language'] != '' and st.session_state['target_language'] != '' and len(st.session_state['source_files']) > 0  and st.session_state['email'] != '':
+                with st.spinner():
+                    submit()
                 st.session_state['tried_to_submit'] = False
                 st.toast("Translations are on their way! You'll receive an email when your translation job has begun.")
                 sleep(4)

samples/ServalApp/serval_email_module.py

@@ -8,11 +8,11 @@ def __init__(self, password, sender_address = '[email protected]
         self.host = host
         self.port = port
         self.server = None
-    
+
     @property
     def password(self):
         return len(self.__password)*"*"
-    
+
     def __enter__(self):
         context = ssl.create_default_context()
         self.server = smtplib.SMTP_SSL(host=self.host, port=self.port, context=context)
@@ -21,38 +21,36 @@ def __enter__(self):
 
     def __exit__(self, *args):
         self.server.close()
-    
+
     def send_build_completed_email(self, recipient_address:str, pretranslations_file_data:str):
         msg = EmailMessage()
         msg.set_content(
-            '''
-            Hi!
+'''Hi!
 
-            Your NMT engine has completed building. Attached are the translations of untranslated source text in the files you included.
+Your NMT engine has completed building. Attached are the translations of untranslated source text in the files you included.
 
-            If you are experiencing difficulties using this application, please contact [email protected].
-                        
-            Thank you!
-            '''
+If you are experiencing difficulties using this application, please contact [email protected].
+
+Thank you!
+'''
         )
         msg['From'] = self.sender_address
         msg['To'] = recipient_address
         msg['Subject'] = 'Your NMT build job is complete!'
         msg.add_attachment(pretranslations_file_data, filename='translations.txt')
         self.server.send_message(msg)
-        
-    def send_build_faulted_email(self, recipient_address:str):
+
+    def send_build_faulted_email(self, recipient_address:str, error=""):
         msg = EmailMessage()
-        msg.add_attachment(
-            '''
-            Hi!
-
-            Your NMT engine has failed to build. Please make sure the information you specified is correct and try again after a while.
-                
-            If you continue to experience difficulties using this application, please contact [email protected].
-                
-            Thank you!
-            '''
+        msg.set_content(
+f'''Hi!
+
+Your NMT engine has failed to build{" with the following error message: " + error if error != "" else ""}. Please make sure the information you specified is correct and try again after a while.
+
+If you continue to experience difficulties using this application, please contact [email protected].
+
+Thank you!
+'''
         )
         msg['From'] = self.sender_address
         msg['To'] = recipient_address
@@ -62,15 +60,14 @@ def send_build_faulted_email(self, recipient_address:str):
     def send_build_started_email(self, recipient_address:str):
         msg = EmailMessage()
         msg.set_content(
-            '''
-            Hi!
-
-            Your NMT engine has started building. We will contact you when it is complete.
-            
-            If you are experiencing difficulties using this application, please contact [email protected].
-            
-            Thank you!
-            '''
+'''Hi!
+
+Your NMT engine has started building. We will contact you when it is complete.
+
+If you are experiencing difficulties using this application, please contact [email protected].
+
+Thank you!
+'''
         )
         msg['From'] = self.sender_address
         msg['To'] = recipient_address