From ef3d43526955c99d131e4893f861e4882829a08d Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Tue, 10 Oct 2023 10:56:51 -0400 Subject: [PATCH 1/5] Commit the load testing script to the repo --- scripts/load_testing.py | 318 ++++++++++++++++++++++++ scripts/load_testing_data/testsrc.txt | 31 +++ scripts/load_testing_data/testsrc2.txt | 32 +++ scripts/load_testing_data/testtarg.txt | 43 ++++ scripts/load_testing_data/testtarg2.txt | 25 ++ 5 files changed, 449 insertions(+) create mode 100644 scripts/load_testing.py create mode 100644 scripts/load_testing_data/testsrc.txt create mode 100644 scripts/load_testing_data/testsrc2.txt create mode 100644 scripts/load_testing_data/testtarg.txt create mode 100644 scripts/load_testing_data/testtarg2.txt diff --git a/scripts/load_testing.py b/scripts/load_testing.py new file mode 100644 index 00000000..d879bd08 --- /dev/null +++ b/scripts/load_testing.py @@ -0,0 +1,318 @@ +#! /usr/bin/python3 +import json +import os, stat, time +import urllib3 +import shutil +import random +import string +from tqdm import tqdm + +def main(): + start = time.time() + SERVAL_AUTH_URL = os.environ.get("SERVAL_AUTH_URL") + SERVAL_CLIENT_ID = os.environ.get("SERVAL_CLIENT_ID") + SERVAL_CLIENT_SECRET = os.environ.get("SERVAL_CLIENT_SECRET") + REQUESTS_PER_SECOND = 5 + NUM_CONCURRENT_CONNECTIONS = 20 + NUM_NMT_ENGINES_TO_ADD = 10_000 + NUM_SMT_ENGINES_TO_ADD = 500 + + base_url = "localhost" #"https://qa-int.serval-api.org" + + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + print('Fetching authorization token...') + data = { + "client_id": f"{SERVAL_CLIENT_ID}", + "client_secret":f"{SERVAL_CLIENT_SECRET}", + "audience":"https://machine.sil.org", + "grant_type":"client_credentials" + } + + encoded_data = json.dumps(data).encode('utf-8') + + http = urllib3.PoolManager() #Use the following parameters if base_url is not localhost: cert_reqs='CERT_NONE', assert_hostname=False + r:urllib3.response.HTTPResponse =http.request( + 'POST', + f'{SERVAL_AUTH_URL}/oauth/token', + body=encoded_data, + headers={"content-type": "application/json"} + ) + access_token = json.loads(r.data.decode('utf-8'))['access_token'] + + src_id = "" + trg_id = "" + + print('Setting up bombardier...') + bombardier_path_to_exe = 'load_testing_data/bombardier-linux-amd64' + if not os.path.exists(bombardier_path_to_exe): + with http.request('GET', 'https://github.com/codesenberg/bombardier/releases/download/v1.2.6/bombardier-linux-amd64', preload_content=False) as r, open(bombardier_path_to_exe, 'wb') as out_file: + shutil.copyfileobj(r, out_file) + + os.chmod(bombardier_path_to_exe, mode=stat.S_IXUSR) + try: + print('Bombarding get all translation engines endpoint...') + os.system( + "./" + + bombardier_path_to_exe + + f' --print r -k -l -d 60s -r {REQUESTS_PER_SECOND} -c {NUM_CONCURRENT_CONNECTIONS} -H "authorization: Bearer {access_token}" -H "accept: application/json" -m "GET" "{base_url}/api/v1/translation/engines"') + + print("Posting engines to DB...") + nmt_engine_ids:set[str] = set() + smt_engine_ids:set[str] = set() + def post_nmt_engine(): + r:urllib3.response.HTTPResponse =http.request( + 'POST', + f'{base_url}/api/v1/translation/engines', + body = json.dumps({"name": "load_testing_engine", "targetLanguage": "en_Latn", "sourceLanguage" : "ell_Grek", "type":"Nmt"}).encode('utf-8'), + headers={"content-type": "application/json", "accept":"application/json", "authorization" : f"Bearer {access_token}"} + ) + nmt_engine_ids.add(json.loads(r.data.decode('utf-8'))['id']) + + print("Posting NMT") + for _ in tqdm(range(NUM_NMT_ENGINES_TO_ADD)): + post_nmt_engine() + + def post_smt_engine(): + r:urllib3.response.HTTPResponse =http.request( + 'POST', + f'{base_url}/api/v1/translation/engines', + body = json.dumps({"name": "load_testing_engine", "targetLanguage": "en_Latn", "sourceLanguage" : "ell_Grek", "type":"SmtTransfer"}).encode('utf-8'), + headers={"content-type": "application/json", "accept":"application/json", "authorization" : f"Bearer {access_token}"} + ) + smt_engine_ids.add(json.loads(r.data.decode('utf-8'))['id']) + + print("Posting SMT") + for _ in tqdm(range(NUM_SMT_ENGINES_TO_ADD)): + post_smt_engine() + + #use bombadier get + print('Bombarding get all translation engines endpoint after adding docs...') + os.system( + "./" + + bombardier_path_to_exe + + f' --print r -k -l -d 60s -r {REQUESTS_PER_SECOND} -c {NUM_CONCURRENT_CONNECTIONS} -H "authorization: Bearer {access_token}" -H "accept: application/json" -m "GET" "{base_url}/api/v1/translation/engines"') + #add necessary files + print('Adding corpus to smt engine...') + with open('load_testing_data/testsrc.txt', 'r') as src_file: + src_data = src_file.read() + r:urllib3.response.HTTPResponse =http.request_encode_body( + 'POST', + f'{base_url}/api/v1/files', + fields={ + 'file': ('testsrc.txt', src_data, 'text/plain'), + 'format':'Text' + }, + headers={'accept' : 'application/json', "authorization" : f"Bearer {access_token}"}, + encode_multipart=True + ) + src_id = json.loads(r.data.decode('utf-8'))['id'] + + with open('load_testing_data/testtarg.txt', 'r') as targ_file: + targ_data = targ_file.read() + r:urllib3.response.HTTPResponse =http.request( + 'POST', + f'{base_url}/api/v1/files', + fields={'file':('testtarg.txt', targ_data, 'text/plain'), 'format':'Text'}, + headers={'accept': 'application/json', "authorization" : f"Bearer {access_token}"} + ) + trg_id = json.loads(r.data.decode('utf-8'))['id'] + + print("Building an SMT engine for bombardment...") + #add corpora and build an smt + smt_id = list(smt_engine_ids)[0] + http.request( + 'POST', + f'{base_url}/api/v1/translation/engines/{smt_id}/corpora', + body=json.dumps( + { + "sourceLanguage":"ell_Grek", + "targetLanguage":"en_Latn", + "sourceFiles": + [ + { + "fileId" : src_id, + "textId":"all" + } + ], + "targetFiles": + [ + { + "fileId" : trg_id, + "textId":"all" + } + ] + }).encode('utf-8'), + headers={'content-type':'application/json', 'accept': 'application/json', "authorization" : f"Bearer {access_token}"} + ) + # corpus_id = json.loads(r.data.decode('utf-8'))['id'] + r:urllib3.response.HTTPResponse = http.request( + 'POST', + f'{base_url}/api/v1/translation/engines/{smt_id}/builds', + body=json.dumps({}).encode('utf-8'), + headers={"authorization" : f"Bearer {access_token}", 'content-type':'application/json'} + ) + + #waiting for build + is_built = False + retry_index = 0 + while not is_built: + r:urllib3.response.HTTPResponse = http.request( + 'GET', + f'{base_url}/api/v1/translation/engines/{smt_id}/current-build', + headers={"authorization" : f"Bearer {access_token}"} + ) + is_built = r.status == 204 + if r.status//100 != 2: + raise Exception(f"Received response of {r.status} while trying to build engine; cannot continue testing!") + if retry_index > 15: + r:urllib3.response.HTTPResponse = http.request( + 'POST', + f'{base_url}/api/v1/translation/engines/{smt_id}/current-build/cancel', + headers={"authorization" : f"Bearer {access_token}"} + ) + + print("Engine is taking too long to build to continue testing. Cancelling build...") + raise Exception("Engine is taking too long to build to continue testing. Cancelling build...") + time.sleep(60 if retry_index == 0 else 20*retry_index) + retry_index += 1 + + segment_file_name = "".join(random.choices(string.ascii_letters, k=24)) + '.json' + f = open(segment_file_name, 'w', encoding='utf-8') + f.write(json.dumps("Βίβλος γενέσεως Ἰησοῦ Χριστοῦ")) + f.flush() + f.close() + + #bombard get word graph + print("Bombarding word graph endpoint...") + os.system( + "./" + + bombardier_path_to_exe + + f' --print r -k -l -d 60s -r {REQUESTS_PER_SECOND} -c {NUM_CONCURRENT_CONNECTIONS} -H "authorization: Bearer {access_token}" -H "accept: application/json" -H "content-type: application/json" -m "POST" "{base_url}/api/v1/translation/engines/{smt_id}/get-word-graph" ' + + f'-f "{segment_file_name}"' + ) + + os.remove(segment_file_name) + + nmt_id = list(nmt_engine_ids)[0] + + print("Building NMT engine...") + r:urllib3.response.HTTPResponse = http.request( + 'POST', + f'{base_url}/api/v1/translation/engines/{nmt_id}/corpora', + body=json.dumps( + { + "sourceLanguage":"ell_Grek", + "targetLanguage":"en_Latn", + "sourceFiles": + [ + { + "fileId" : src_id, + "textId":"all" + } + ], + "targetFiles": + [ + { + "fileId" : trg_id, + "textId":"all" + } + ] + }).encode('utf-8'), + headers={'content-type':'application/json', 'accept': 'application/json', "authorization" : f"Bearer {access_token}"} + ) + + corpus_id = json.loads(r.data.decode('utf-8'))['id'] + + r:urllib3.response.HTTPResponse = http.request( + 'POST', + f'{base_url}/api/v1/translation/engines/{nmt_id}/builds', + body=json.dumps( + { + "pretranslate" : [ + { + "corpusId": corpus_id, + "textIds": [ + "all" + ] + } + ], + "options":"{\"max_steps\":10}" + } + ), + headers={"authorization" : f"Bearer {access_token}", 'content-type':'application/json'} + ) + + #waiting for build + is_built = False + retry_index = 0 + while not is_built: + r:urllib3.response.HTTPResponse = http.request( + 'GET', + f'{base_url}/api/v1/translation/engines/{nmt_id}/current-build', + headers={"authorization" : f"Bearer {access_token}"} + ) + is_built = r.status == 204 + if r.status//100 != 2: + raise Exception(f"Received response of {r.status} while trying to build engine; cannot continue testing!") + if retry_index > 15: + print("Engine is taking too long to build to continue testing. Cancelling build...") + r:urllib3.response.HTTPResponse = http.request( + 'POST', + f'{base_url}/api/v1/translation/engines/{nmt_id}/current-build/cancel', + headers={"authorization" : f"Bearer {access_token}"} + ) + raise Exception("Engine is taking too long to build to continue testing. Cancelling build...") + time.sleep(240 if retry_index == 0 else 60*retry_index) + retry_index += 1 + + print("Bombarding pretranslation endpoint...") + #bombard get pretrans + os.system( + "./" + + bombardier_path_to_exe + + f' --print r -k -l -d 60s -r {REQUESTS_PER_SECOND} -c {NUM_CONCURRENT_CONNECTIONS} -H "authorization: Bearer {access_token}" -H "accept: application/json" -m "GET" "{base_url}/api/v1/translation/engines/{nmt_id}/corpora/{corpus_id}/pretranslations" ' + ) + except Exception as e: + print("Something went wrong:", str(e) if str(e) != "" else "[No information]") + finally: + #cleanup files, smt, nmt, bombardier + print('Cleaning up...') + print('Deleting added translation engines...') + def delete_engine(engine_id): + r:urllib3.response.HTTPResponse =http.request( + 'DELETE', + f'{base_url}/api/v1/translation/engines/{engine_id}', + body = json.dumps({"id":engine_id}).encode('utf-8'), + headers={"content-type": "application/json", "authorization" : f"Bearer {access_token}"} + ) + if(r.status != 200): + print(f"Failed to delete engine {engine_id}") + + for engine_id in tqdm(nmt_engine_ids): + delete_engine(engine_id) + for engine_id in tqdm(smt_engine_ids): + delete_engine(engine_id) + + r:urllib3.response.HTTPResponse =http.request( + 'DELETE', + f'{base_url}/api/v1/files/{src_id}', + headers={"content-type": "application/json", "authorization" : f"Bearer {access_token}"} + ) + if(r.status != 200): + print(f"Failed to delete file {src_id}") + + r:urllib3.response.HTTPResponse =http.request( + 'DELETE', + f'{base_url}/api/v1/files/{trg_id}', + headers={"content-type": "application/json", "authorization" : f"Bearer {access_token}"} + ) + if(r.status != 200): + print(f"Failed to delete file {trg_id}") + + + os.remove(bombardier_path_to_exe) + print("Finished testing in", round((time.time()-start)/60, 2), "minutes.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/load_testing_data/testsrc.txt b/scripts/load_testing_data/testsrc.txt new file mode 100644 index 00000000..0615bc6d --- /dev/null +++ b/scripts/load_testing_data/testsrc.txt @@ -0,0 +1,31 @@ +h_001 ΙΟΥΔΑ ss +mt1_001 ΙΟΥΔΑ +mt1_002 ΕΠΙΣΤΟΛΗ ΚΑΘΟΛΙΚΗ +toc1_001 ΙΟΥΔΑ ΕΠΙΣΤΟΛΗ ΚΑΘΟΛΙΚΗ +toc2_001 ΙΟΥΔΑ +toc3_001 ΙΟΥΔΑ +verse_001_001 Ἰούδας Ἰησοῦ Χριστοῦ δοῦλος , ἀδελφὸς δὲ Ἰακώβου , τοῖς ἐν Θεῷ πατρὶ ἡγιασμένοις , καὶ Ἰησοῦ Χριστῷ τετηρημένοις , κλητοῖς · +verse_001_002 ἔλεος ὑμῖν καὶ εἰρήνη καὶ ἀγάπη πληθυνθείη . +verse_001_003 Ἀγαπητοί , πᾶσαν σπουδὴν ποιούμενος γράφειν ὑμῖν περὶ τῆς κοινῆς σωτηρίας , ἀνάγκην ἔσχον γράψαι ὑμῖν , παρακαλῶν ἐπαγωνίζεσθαι τῇ ἅπαξ παραδοθείσῃ τοῖς ἁγίοις πίστει . +verse_001_004 Παρεισέδυσαν γάρ τινες ἄνθρωποι , οἱ πάλαι προγεγραμμένοι εἰς τοῦτο τὸ κρίμα , ἀσεβεῖς , τὴν τοῦ Θεοῦ ἡμῶν χάριν μετατιθέντες εἰς ἀσέλγειαν , καὶ τὸν μόνον δεσπότην Θεὸν καὶ Κύριον ἡμῶν Ἰησοῦν Χριστὸν ἀρνούμενοι . ss +verse_001_005 Ὑπομνῆσαι δὲ ὑμᾶς βούλομαι , εἰδότας ὑμᾶς ἅπαξ τοῦτο , ὅτι ὁ Κύριος , λαὸν ἐκ γῆς Αἰγύπτου σώσας , τὸ δεύτερον τοὺς μὴ πιστεύσαντας ἀπώλεσεν . +verse_001_006 Ἀγγέλους τε τοὺς μὴ τηρήσαντας τὴν ἑαυτῶν ἀρχήν , ἀλλὰ ἀπολιπόντας τὸ ἴδιον οἰκητήριον , εἰς κρίσιν μεγάλης ἡμέρας δεσμοῖς ἀϊδίοις ὑπὸ ζόφον τετήρηκεν . ss +verse_001_007 Ὡς Σόδομα καὶ Γόμορρα , καὶ αἱ περὶ αὐτὰς πόλεις , τὸν ὅμοιον τούτοις τρόπον ἐκπορνεύσασαι , καὶ ἀπελθοῦσαι ὀπίσω σαρκὸς ἑτέρας , πρόκεινται δεῖγμα , πυρὸς αἰωνίου δίκην ὑπέχουσαι . ss +verse_001_008 Ὁμοίως μέντοι καὶ οὗτοι ἐνυπνιαζόμενοι σάρκα μὲν μιαίνουσι , κυριότητα δὲ ἀθετοῦσι , δόξας δὲ βλασφημοῦσιν . ss +verse_001_009 Ὁ δὲ Μιχαὴλ ὁ ἀρχάγγελος , ὅτε τῷ διαβόλῳ διακρινόμενος διελέγετο περὶ τοῦ Μωσέως σώματος , οὐκ ἐτόλμησε κρίσιν ἐπενεγκεῖν βλασφημίας , ἀλλ ᾿ εἶπεν , Ἐπιτιμήσαι σοι Κύριος . ss +verse_001_010 Οὗτοι δὲ ὅσα μὲν οὐκ οἴδασι βλασφημοῦσιν · ὅσα δὲ φυσικῶς , ὡς τὰ ἄλογα ζῷα , ἐπίστανται , ἐν τούτοις φθείρονται . ss +verse_001_011 Οὐαὶ αὐτοῖς · ὅτι τῇ ὁδῷ τοῦ Κάϊν ἐπορεύθησαν , καὶ τῇ πλάνῃ τοῦ Βαλαὰμ μισθοῦ ἐξεχύθησαν , καὶ τῇ ἀντιλογίᾳ τοῦ Κόρε ἀπώλοντο . ss +verse_001_012 Οὗτοί εἰσιν ἐν ταῖς ἀγάπαις ὑμῶν σπιλάδες , συνευωχούμενοι , ἀφόβως ἑαυτοὺς ποιμαίνοντες · νεφέλαι ἄνυδροι , ὑπὸ ἀνέμων περιφερόμεναι · δένδρα φθινοπωρινά , ἄκαρπα , δὶς ἀποθανόντα , ἐκριζωθέντα · ss +verse_001_013 κύματα ἄγρια θαλάσσης , ἐπαφρίζοντα τὰς ἑαυτῶν αἰσχύνας · ἀστέρες πλανῆται , οἷς ὁ ζόφος τοῦ σκότους εἰς τὸν αἰῶνα τετήρηται . +verse_001_014 Προεφήτευσε δὲ καὶ τούτοις ἕβδομος ἀπὸ Ἀδὰμ Ἐνώχ , λέγων , Ἰδού , ἦλθε Κύριος ἐν μυριάσιν ἁγίαις αὐτοῦ , ss +verse_001_015 ποιῆσαι κρίσιν κατὰ πάντων , καὶ ἐξελέγξαι πάντας τοὺς ἀσεβεῖς αὐτῶν περὶ πάντων τῶν ἔργων ἀσεβείας αὐτῶν ὧν ἠσέβησαν , καὶ περὶ πάντων τῶν σκληρῶν ὧν ἐλάλησαν κατ ᾿ αὐτοῦ ἁμαρτωλοὶ ἀσεβεῖς . +verse_001_016 Οὗτοί εἰσι γογγυσταί , μεμψίμοιροι , κατὰ τὰς ἐπιθυμίας αὐτῶν πορευόμενοι , καὶ τὸ στόμα αὐτῶν λαλεῖ ὑπέρογκα , θαυμάζοντες πρόσωπα ὠφελείας χάριν . ss +verse_001_017 Ὑμεῖς δέ , ἀγαπητοί , μνήσθητε τῶν ῥημάτων τῶν προειρημένων ὑπὸ τῶν ἀποστόλων τοῦ Κυρίου ἡμῶν Ἰησοῦ Χριστοῦ · +verse_001_018 ὅτι ἔλεγον ὑμῖν ὅτι ἐν ἐσχάτῳ χρόνῳ ἔσονται ἐμπαῖκται , κατὰ τὰς ἑαυτῶν ἐπιθυμίας πορευόμενοι τῶν ἀσεβειῶν . +verse_001_019 Οὗτοί εἰσιν οἱ ἀποδιορίζοντες , ψυχικοί , πνεῦμα μὴ ἔχοντες . ss +verse_001_020 Ὑμεῖς δέ , ἀγαπητοί , τῇ ἁγιωτάτῃ ὑμῶν πίστει ἐποικοδομοῦντες ἑαυτούς , ἐν πνεύματι ἁγίῳ προσευχόμενοι , ss +verse_001_021 ἑαυτοὺς ἐν ἀγάπῃ Θεοῦ τηρήσατε , προσδεχόμενοι τὸ ἔλεος τοῦ Κυρίου ἡμῶν Ἰησοῦ Χριστοῦ εἰς ζωὴν αἰώνιον . +verse_001_022 Καὶ οὓς μὲν ἐλεεῖτε διακρινόμενοι · ss +verse_001_023 οὓς δὲ ἐν φόβῳ σώζετε , ἐκ τοῦ πυρὸς ἁρπάζοντες , μισοῦντες καὶ τὸν ἀπὸ τῆς σαρκὸς ἐσπιλωμένον χιτῶνα . +verse_001_024 Τῷ δὲ δυναμένῳ φυλάξαι αὐτοὺς ἀπταίστους , καὶ στῆσαι κατενώπιον τῆς δόξης αὐτοῦ ἀμώμους ἐν ἀγαλλιάσει , +verse_001_025 μόνῳ σοφῷ Θεῷ σωτῆρι ἡμῶν , δόξα καὶ μεγαλωσύνη , κράτος καὶ ἐξουσία , καὶ νῦν καὶ εἰς πάντας τοὺς αἰῶνας . Ἀμήν . \ No newline at end of file diff --git a/scripts/load_testing_data/testsrc2.txt b/scripts/load_testing_data/testsrc2.txt new file mode 100644 index 00000000..7c19abe9 --- /dev/null +++ b/scripts/load_testing_data/testsrc2.txt @@ -0,0 +1,32 @@ +h_001 ΚΑΤΑ ΜΑΤΘΑΙΟΝ ss +mt1_001 ΕΥΑΓΓΕΛΙΟΝ +mt1_002 ΤΟ ΚΑΤΑ ΜΑΤΘΑΙΟΝ +toc1_001 ΕΥΑΓΓΕΛΙΟΝ ΤΟ ΚΑΤΑ ΜΑΤΘΑΙΟΝ +toc2_001 ΚΑΤΑ ΜΑΤΘΑΙΟΝ +toc3_001 ΚΑΤΑ ΜΑΤΘΑΙΟΝ +verse_001_001 Βίβλος γενέσεως Ἰησοῦ Χριστοῦ , υἱοῦ Δαβίδ , υἱοῦ Ἀβραάμ . +verse_001_002 Ἀβραὰμ ἐγέννησε τὸν Ἰσαάκ · Ἰσαὰκ δὲ ἐγέννησε τὸν Ἰακώβ · Ἰακὼβ δὲ ἐγέννησε τὸν Ἰούδαν καὶ τοὺς ἀδελφοὺς αὐτοῦ · +verse_001_003 Ἰούδας δὲ ἐγέννησε τὸν Φαρὲς καὶ τὸν Ζαρὰ ἐκ τῆς Θάμαρ · Φαρὲς δὲ ἐγέννησε τὸν Ἑσρώμ · Ἑσρὼμ δὲ ἐγέννησε τὸν Ἀράμ · +verse_001_004 Ἀρὰμ δὲ ἐγέννησε τὸν Ἀμιναδάβ · Ἀμιναδὰβ δὲ ἐγέννησε τὸν Ναασσών · Ναασσὼν δὲ ἐγέννησε τὸν Σαλμών · +verse_001_005 Σαλμὼν δὲ ἐγέννησε τὸν Βοὸζ ἐκ τῆς Ῥαχάβ · Βοὸζ δὲ ἐγέννησε τὸν Ὠβὴδ ἐκ τῆς Ῥούθ · Ὠβὴδ δὲ ἐγέννησε τὸν Ἰεσσαί · +verse_001_006 Ἰεσσαὶ δὲ ἐγέννησε τὸν Δαβὶδ τὸν βασιλέα . +verse_001_006_001 Δαβὶδ δὲ ὁ βασιλεὺς ἐγέννησε τὸν Σολομῶντα ἐκ τῆς τοῦ Οὐρίου · ss +verse_001_007 Σολομὼν δὲ ἐγέννησε τὸν Ῥοβοάμ · Ῥοβοὰμ δὲ ἐγέννησε τὸν Ἀβιά · Ἀβιὰ δὲ ἐγέννησε τὸν Ἀσά · +verse_001_008 Ἀσὰ δὲ ἐγέννησε τὸν Ἰωσαφάτ · Ἰωσαφὰτ δὲ ἐγέννησε τὸν Ἰωράμ · Ἰωρὰμ δὲ ἐγέννησε τὸν Ὀζίαν · +verse_001_009 Ὀζίας δὲ ἐγέννησε τὸν Ἰωάθαμ · Ἰωάθαμ δὲ ἐγέννησε τὸν Ἄχαζ · Ἄχαζ δὲ ἐγέννησε τὸν Ἑζεκίαν · +verse_001_010 Ἑζεκίας δὲ ἐγέννησε τὸν Μανασσῆ · Μανασσῆς δὲ ἐγέννησε τὸν Ἀμών · Ἀμὼν δὲ ἐγέννησε τὸν Ἰωσίαν · +verse_001_011 Ἰωσίας δὲ ἐγέννησε τὸν Ἰεχονίαν καὶ τοὺς ἀδελφοὺς αὐτοῦ , ἐπὶ τῆς μετοικεσίας Βαβυλῶνος . +verse_001_012 Μετὰ δὲ τὴν μετοικεσίαν Βαβυλῶνος , Ἰεχονίας ἐγέννησε τὸν Σαλαθιήλ · Σαλαθιὴλ δὲ ἐγέννησε τὸν Ζοροβάβελ · +verse_001_013 Ζοροβάβελ δὲ ἐγέννησε τὸν Ἀβιούδ · Ἀβιοὺδ δὲ ἐγέννησε τὸν Ἐλιακείμ · Ἐλιακεὶμ δὲ ἐγέννησε τὸν Ἀζώρ · +verse_001_014 Ἀζὼρ δὲ ἐγέννησε τὸν Σαδώκ · Σαδὼκ δὲ ἐγέννησε τὸν Ἀχείμ · Ἀχεὶμ δὲ ἐγέννησε τὸν Ἐλιούδ · +verse_001_015 Ἐλιοὺδ δὲ ἐγέννησε τὸν Ἐλεάζαρ · Ἐλεάζαρ δὲ ἐγέννησε τὸν Ματθάν · Ματθὰν δὲ ἐγέννησε τὸν Ἰακώβ · +verse_001_016 Ἰακὼβ δὲ ἐγέννησε τὸν Ἰωσὴφ τὸν ἄνδρα Μαρίας , ἐξ ἧς ἐγεννήθη Ἰησοῦς , ὁ λεγόμενος Χριστός . +verse_001_017 Πᾶσαι οὖν αἱ γενεαὶ ἀπὸ Ἀβραὰμ ἕως Δαβὶδ γενεαὶ δεκατέσσαρες · καὶ ἀπὸ Δαβὶδ ἕως τῆς μετοικεσίας Βαβυλῶνος , γενεαὶ δεκατέσσαρες · καὶ ἀπὸ τῆς μετοικεσίας Βαβυλῶνος ἕως τοῦ Χριστοῦ , γενεαὶ δεκατέσσαρες . +verse_001_018 Τοῦ δὲ Ἰησοῦ Χριστοῦ ἡ γέννησις οὕτως ἦν . Μνηστευθείσης γὰρ τῆς μητρὸς αὐτοῦ Μαρίας τῷ Ἰωσήφ , πρὶν ἢ συνελθεῖν αὐτούς , εὑρέθη ἐν γαστρὶ ἔχουσα ἐκ πνεύματος ἁγίου . +verse_001_019 Ἰωσὴφ δὲ ὁ ἀνὴρ αὐτῆς , δίκαιος ὤν , καὶ μὴ θέλων αὐτὴν παραδειγματίσαι , ἐβουλήθη λάθρᾳ ἀπολῦσαι αὐτήν . ss +verse_001_020 Ταῦτα δὲ αὐτοῦ ἐνθυμηθέντος , ἰδού , ἄγγελος Κυρίου κατ ᾿ ὄναρ ἐφάνη αὐτῷ , λέγων , Ἰωσήφ , υἱὸς Δαβίδ , μὴ φοβηθῇς παραλαβεῖν Μαριὰμ τὴν γυναῖκά σου · τὸ γὰρ ἐν αὐτῇ γεννηθὲν ἐκ πνεύματός ἐστιν ἁγίου . ss +verse_001_021 Τέξεται δὲ υἱόν , καὶ καλέσεις τὸ ὄνομα αὐτοῦ Ἰησοῦν · αὐτὸς γὰρ σώσει τὸν λαὸν αὐτοῦ ἀπὸ τῶν ἁμαρτιῶν αὐτῶν . ss +verse_001_022 Τοῦτο δὲ ὅλον γέγονεν , ἵνα πληρωθῇ τὸ ῥηθὲν ὑπὸ τοῦ Κυρίου διὰ τοῦ προφήτου , λέγοντος , ss +verse_001_023 Ἰδού , ἡ παρθένος ἐν γαστρὶ ἕξει καὶ τέξεται υἱόν , καὶ καλέσουσι τὸ ὄνομα αὐτοῦ Ἐμμανουήλ , ὅ ἐστι μεθερμηνευόμενον , Μεθ ᾿ ἡμῶν ὁ Θεός . +verse_001_024 Διεγερθεὶς δὲ ὁ Ἰωσὴφ ἀπὸ τοῦ ὕπνου , ἐποίησεν ὡς προσέταξεν αὐτῷ ὁ ἄγγελος Κυρίου · καὶ παρέλαβε τὴν γυναῖκα αὐτοῦ , ss +verse_001_025 καὶ οὐκ ἐγίνωσκεν αὐτὴν ἕως οὗ ἔτεκε τὸν υἱὸν αὐτῆς τὸν πρωτότοκον · καὶ ἐκάλεσε τὸ ὄνομα αὐτοῦ Ἰησοῦν . \ No newline at end of file diff --git a/scripts/load_testing_data/testtarg.txt b/scripts/load_testing_data/testtarg.txt new file mode 100644 index 00000000..59e55a43 --- /dev/null +++ b/scripts/load_testing_data/testtarg.txt @@ -0,0 +1,43 @@ +mt1_001 Jude +mt2_001 The General Epistle of +p_001 +p_002 +p_003 +p_004 +p_005 +p_006 +rem_001 Jude Draft Translation ss +s_001 Greetings +s_002 The false teachers ss +s_003 Exhortations to remain steadfast and rebuke those in error ss +s_004 Benediction ss +verse_001_001 Jude , a slave of Jesus Christ , and brother of James . +verse_001_001_001 To those who are called , sanctified by God the Father , and preserved in Jesus Christ : +verse_001_001_002 +verse_001_002 +verse_001_003 Beloved , while I was making every effort to write to you concerning the common salvation , I found it necessary to write to you , exhorting you to struggle for the faith that was once for all delivered to the saints . +verse_001_004 For certain men have sneaked in , who long ago were written down for this judgment , ungodly men , who turn the grace of our God into sensuality and deny the only Lord God , and our Lord Jesus Christ . ss +verse_001_005 But I want to remind you , though you once knew this , that the Lord , having saved the people out of the land of Egypt , afterwards destroyed those who did not believe . +verse_001_006 And the angels who did not keep their domain , but abandoned their own dwelling , He has kept in eternal chains under darkness for the judgment of the great day . +verse_001_007 As Sodom and Gomorrah , and the cities around them , who in a similar way lived in sexual immorality and went after strange flesh , are set forth as an example , suffering the vengance of eternal fire . +verse_001_007_001 ss +verse_001_008 Nevertheless , in the same way these dreamers defile the flesh , reject authority , and blaspheme the glorious . +verse_001_009 But Michael the archangel , when disputing with the devil , arguing about the body of Moses , dared not give him a blasphemous judgment , but said , " The Lord rebuke you ! " ss +verse_001_010 But these blaspheme whatever they do not know , and whatever they know of the world , like irrational animals , in these things are corrupted . ss +verse_001_011 Woe to them ! For they have gone in the way of Cain , and run after the error of Balaam for reward , and perished in the rebellion of Korah . ss +verse_001_012 These are hidden reefs in your love feasts , feasting with you fearlessly , caring only for themselves . Waterless clouds , carried about by wind . Autumn trees , unfruitful , twice dead , uprooted ; +verse_001_013 wild sea waves , foaming up their own shame ; wandering stars for who the gloom of darkness is reserved forever . +verse_001_013_001 ss +verse_001_014 And also Enoch , the seventh from Adam , prophesised about these , saying , " Behold the Lord comes with ten thousands of His saints , +verse_001_015 to execute judgment on all , to convict all who are ungodly among them of all their ungodly works , their ungodly lives , and all of the cruelties these ungodly sinners have spoken against Him . " +verse_001_016 These are grumblers , complainers , going after their own lusts , and their mouth speaks boastfully , flattering people for the sake of advantage . +verse_001_017 But you , beloved , remember the words which were spoken before by the apostles of our Lord Jesus Christ , ss +verse_001_018 that they told you that : " In the last time there will be mockers , following after their own ungodly lusts . " +verse_001_019 +verse_001_020 But you , beloved , building yourselves up on your most holy faith , praying in the Holy Spirit , +verse_001_021 keep yourselves in the love of God , waiting for the mercy of our Lord Jesus Christ to eternal life . +verse_001_021_001 ss +verse_001_022 And for those who are doubting , have mercy ; +verse_001_023 +verse_001_024 Now unto him who is able to keep you from falling , and present you faultless before the presence of His glory with exceeding joy . ss +verse_001_025 To the only wise God our Saviour , be glory and majesty , dominion and power , both now and forever . Amen . \ No newline at end of file diff --git a/scripts/load_testing_data/testtarg2.txt b/scripts/load_testing_data/testtarg2.txt new file mode 100644 index 00000000..e46adf0d --- /dev/null +++ b/scripts/load_testing_data/testtarg2.txt @@ -0,0 +1,25 @@ +verse_001_001 ss +verse_001_002 +verse_001_003 +verse_001_004 +verse_001_005 +verse_001_006 +verse_001_007 +verse_001_008 +verse_001_009 +verse_001_010 +verse_001_011 +verse_001_012 +verse_001_013 +verse_001_014 +verse_001_015 +verse_001_016 +verse_001_017 +verse_001_018 +verse_001_019 +verse_001_020 +verse_001_021 +verse_001_022 +verse_001_023 +verse_001_024 +verse_001_025 \ No newline at end of file From c1eb84adb481b8bc5b7f00a0ef753188e7c85b9b Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Mon, 9 Oct 2023 14:10:57 -0400 Subject: [PATCH 2/5] Moved keys, queue etc. to 'ClearML' no longer 'ClearMLNmtEngine' --- docker-compose.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 95c89de7..9d13514e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -90,12 +90,12 @@ services: - ASPNETCORE_Logging__LogLevel__Microsoft.AspNetCore=Warning - ASPNETCORE_TranslationEngines__0=SmtTransfer - ASPNETCORE_TranslationEngines__1=Nmt - - ClearMLNmtEngine__ApiServer=https://api.sil.hosted.allegro.ai - - ClearMLNmtEngine__Queue=production - - ClearMLNmtEngine__DockerImage=ghcr.io/sillsdev/machine.py:0.9.5 - - ClearMLNmtEngine__MaxSteps=10 - - "ClearMLNmtEngine__AccessKey=${ClearML_AccessKey:?access key needed}" - - "ClearMLNmtEngine__SecretKey=${ClearML_SecretKey:?secret key needed}" + - ClearML__ApiServer=https://api.sil.hosted.allegro.ai + - ClearML__Queue=production + - ClearML__DockerImage=ghcr.io/sillsdev/machine.py:0.9.5 + - ClearML__MaxSteps=10 + - "ClearML__AccessKey=${ClearML_AccessKey:?access key needed}" + - "ClearML__SecretKey=${ClearML_SecretKey:?secret key needed}" - SharedFile__Uri=s3://aqua-ml-data/docker-compose/ - "SharedFile__S3AccessKeyId=${AWS_ACCESS_KEY_ID:?access key needed}" - "SharedFile__S3SecretAccessKey=${AWS_SECRET_ACCESS_KEY:?secret key needed}" @@ -136,12 +136,12 @@ services: - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2 - ASPNETCORE_TranslationEngines__0=SmtTransfer - ASPNETCORE_TranslationEngines__1=Nmt - - ClearMLNmtEngine__ApiServer=https://api.sil.hosted.allegro.ai - - ClearMLNmtEngine__Queue=lambert_24gb - - ClearMLNmtEngine__DockerImage=ghcr.io/sillsdev/machine.py:0.9.5 - - ClearMLNmtEngine__MaxSteps=10 - - "ClearMLNmtEngine__AccessKey=${ClearML_AccessKey:?access key needed}" - - "ClearMLNmtEngine__SecretKey=${ClearML_SecretKey:?secret key needed}" + - ClearML__ApiServer=https://api.sil.hosted.allegro.ai + - ClearML__Queue=lambert_24gb + - ClearML__DockerImage=ghcr.io/sillsdev/machine.py:0.9.5 + - ClearML__MaxSteps=10 + - "ClearML__AccessKey=${ClearML_AccessKey:?access key needed}" + - "ClearML__SecretKey=${ClearML_SecretKey:?secret key needed}" - SharedFile__Uri=s3://aqua-ml-data/docker-compose/ - "SharedFile__S3AccessKeyId=${AWS_ACCESS_KEY_ID:?access key needed}" - "SharedFile__S3SecretAccessKey=${AWS_SECRET_ACCESS_KEY:?secret key needed}" From 0f7d71f3cfd61cf53ab7fd784d1b0af6ac064dd6 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Tue, 10 Oct 2023 09:09:57 -0400 Subject: [PATCH 3/5] Renamed ClearML-related configuration elements Removed maxSteps from yml and manually passed maxSteps=10 in the E2E tests Updated the swagger documentation to provide an explanation about the options parameter Updated the README to point to production-serval swagger documentation --- README.md | 3 ++ docker-compose.yml | 6 ++-- src/Serval.Client/Client.g.cs | 28 ++++++++++++------- .../TranslationEnginesController.cs | 4 +++ tests/Serval.E2ETests/ServalClientHelper.cs | 6 +++- 5 files changed, 32 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index f467eedb..5b37120c 100644 --- a/README.md +++ b/README.md @@ -117,3 +117,6 @@ All C# code should be formatted using [CSharpier](https://csharpier.com/). The b [Here](https://learn.microsoft.com/en-us/dotnet/csharp/fundamentals/coding-style/identifier-names) is a good overview of naming conventions. [Here](https://learn.microsoft.com/en-us/dotnet/csharp/fundamentals/coding-style/coding-conventions) is a good overview of coding conventions. If you want to get in to even more detail, check out the [Framework design guidelines](https://learn.microsoft.com/en-us/dotnet/standard/design-guidelines/). +## Documentation + +See the Swagger documentation for Serval [here](https://prod.serval-api.org/swagger/index.html). \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 9d13514e..fd9a50fa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -92,8 +92,7 @@ services: - ASPNETCORE_TranslationEngines__1=Nmt - ClearML__ApiServer=https://api.sil.hosted.allegro.ai - ClearML__Queue=production - - ClearML__DockerImage=ghcr.io/sillsdev/machine.py:0.9.5 - - ClearML__MaxSteps=10 + - ClearML__DockerImage=ghcr.io/sillsdev/machine.py:0.9.5.1 - "ClearML__AccessKey=${ClearML_AccessKey:?access key needed}" - "ClearML__SecretKey=${ClearML_SecretKey:?secret key needed}" - SharedFile__Uri=s3://aqua-ml-data/docker-compose/ @@ -138,8 +137,7 @@ services: - ASPNETCORE_TranslationEngines__1=Nmt - ClearML__ApiServer=https://api.sil.hosted.allegro.ai - ClearML__Queue=lambert_24gb - - ClearML__DockerImage=ghcr.io/sillsdev/machine.py:0.9.5 - - ClearML__MaxSteps=10 + - ClearML__DockerImage=ghcr.io/sillsdev/machine.py:0.9.5.1 - "ClearML__AccessKey=${ClearML_AccessKey:?access key needed}" - "ClearML__SecretKey=${ClearML_SecretKey:?secret key needed}" - SharedFile__Uri=s3://aqua-ml-data/docker-compose/ diff --git a/src/Serval.Client/Client.g.cs b/src/Serval.Client/Client.g.cs index e02ca400..0dd43d87 100644 --- a/src/Serval.Client/Client.g.cs +++ b/src/Serval.Client/Client.g.cs @@ -999,6 +999,10 @@ public partial interface ITranslationEnginesClient ///
untranslated text but no translated text. If a corpus is a Paratext project, ///
you may flag a subset of books for pretranslation by including their [abbreviations](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs) ///
in the textIds parameter. If the engine does not support pretranslation, these fields have no effect. + ///
+ ///
The `"options"` parameter of the build config provides the ability to pass build configuration parameters as a JSON string. + ///
A typical use case would be to set `"options"` to `"{\"max_steps\":10}"` in order to configure the maximum + ///
number of training iterations in order to reduce turnaround time for testing purposes. /// /// The translation engine id /// The build config (see remarks) @@ -1011,13 +1015,13 @@ public partial interface ITranslationEnginesClient /// Get a build job /// /// - /// If the `minRevision` is not defined, the current build at whatever state it is + /// If the `minRevision` is not defined, the current build, at whatever state it is, ///
will be immediately returned. If `minRevision` is defined, Serval will wait for ///
up to 40 seconds for the engine to build to the `minRevision` specified, else ///
will timeout. ///
A use case is to actively query the state of the current build, where the subsequent - ///
request sets the `minRevision` to the returned `revision` + 1. Note: this method - ///
should use request throttling. + ///
request sets the `minRevision` to the returned `revision` + 1 and timeouts are handled gracefully. + ///
Note: this method should use request throttling. /// /// The translation engine id /// The build job id @@ -1031,7 +1035,7 @@ public partial interface ITranslationEnginesClient /// Get the currently running build job for a translation engine /// /// - /// See "Get a Build Job" for details on minimum revision. + /// See documentation on endpoint /translation/engines/{id}/builds/{id} - "Get a Build Job" for details on using `minRevision`. /// /// The translation engine id /// The minimum revision @@ -2801,6 +2805,10 @@ public string BaseUrl ///
untranslated text but no translated text. If a corpus is a Paratext project, ///
you may flag a subset of books for pretranslation by including their [abbreviations](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs) ///
in the textIds parameter. If the engine does not support pretranslation, these fields have no effect. + ///
+ ///
The `"options"` parameter of the build config provides the ability to pass build configuration parameters as a JSON string. + ///
A typical use case would be to set `"options"` to `"{\"max_steps\":10}"` in order to configure the maximum + ///
number of training iterations in order to reduce turnaround time for testing purposes. /// /// The translation engine id /// The build config (see remarks) @@ -2922,13 +2930,13 @@ public string BaseUrl /// Get a build job /// /// - /// If the `minRevision` is not defined, the current build at whatever state it is + /// If the `minRevision` is not defined, the current build, at whatever state it is, ///
will be immediately returned. If `minRevision` is defined, Serval will wait for ///
up to 40 seconds for the engine to build to the `minRevision` specified, else ///
will timeout. ///
A use case is to actively query the state of the current build, where the subsequent - ///
request sets the `minRevision` to the returned `revision` + 1. Note: this method - ///
should use request throttling. + ///
request sets the `minRevision` to the returned `revision` + 1 and timeouts are handled gracefully. + ///
Note: this method should use request throttling. /// /// The translation engine id /// The build job id @@ -3020,7 +3028,7 @@ public string BaseUrl if (status_ == 408) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The long polling request timed out", status_, responseText_, headers_, null); + throw new ServalApiException("The long polling request timed out. This is expected behavior if you\'re using long-polling with the minRevision strategy specified in the docs", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -3053,7 +3061,7 @@ public string BaseUrl /// Get the currently running build job for a translation engine /// /// - /// See "Get a Build Job" for details on minimum revision. + /// See documentation on endpoint /translation/engines/{id}/builds/{id} - "Get a Build Job" for details on using `minRevision`. /// /// The translation engine id /// The minimum revision @@ -3146,7 +3154,7 @@ public string BaseUrl if (status_ == 408) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The long polling request timed out. Did you start the build?", status_, responseText_, headers_, null); + throw new ServalApiException("The long polling request timed out. This is expected behavior if you\'re using long-polling with the minRevision strategy specified in the docs", status_, responseText_, headers_, null); } else if (status_ == 503) diff --git a/src/Serval.Translation/Controllers/TranslationEnginesController.cs b/src/Serval.Translation/Controllers/TranslationEnginesController.cs index eaa55d75..953063bd 100644 --- a/src/Serval.Translation/Controllers/TranslationEnginesController.cs +++ b/src/Serval.Translation/Controllers/TranslationEnginesController.cs @@ -740,6 +740,10 @@ CancellationToken cancellationToken /// untranslated text but no translated text. If a corpus is a Paratext project, /// you may flag a subset of books for pretranslation by including their [abbreviations](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs) /// in the textIds parameter. If the engine does not support pretranslation, these fields have no effect. + /// + /// The `"options"` parameter of the build config provides the ability to pass build configuration parameters as a JSON string. + /// A typical use case would be to set `"options"` to `"{\"max_steps\":10}"` in order to configure the maximum + /// number of training iterations in order to reduce turnaround time for testing purposes. /// /// The translation engine id /// The build config (see remarks) diff --git a/tests/Serval.E2ETests/ServalClientHelper.cs b/tests/Serval.E2ETests/ServalClientHelper.cs index 6a944910..df1b209f 100644 --- a/tests/Serval.E2ETests/ServalClientHelper.cs +++ b/tests/Serval.E2ETests/ServalClientHelper.cs @@ -29,7 +29,11 @@ public ServalClientHelper(string audience, string prefix = "SCE_", bool ignoreSS $"Bearer {GetAuth0Authentication(env["authUrl"], audience, env["clientId"], env["clientSecret"]).Result}" ); _prefix = prefix; - TranslationBuildConfig = new TranslationBuildConfig { Pretranslate = new List() }; + TranslationBuildConfig = new TranslationBuildConfig + { + Pretranslate = new List(), + Options = "{\"max_steps\":10}" + }; } public TranslationBuildConfig TranslationBuildConfig { get; set; } From 0f6a78e9e7a61dc57c4227432d28a606016d8dd5 Mon Sep 17 00:00:00 2001 From: "Eli C. Lowry" <83078660+Enkidu93@users.noreply.github.com> Date: Tue, 10 Oct 2023 16:30:49 -0400 Subject: [PATCH 4/5] Expand documentation on valid text files (#176) * Expand documentation on valid text files Note: This was part of earlier work done when addressing the issue of strict parsing before it was decided that Serval should not validate files. It would still be beneficial to incorporate. * Fix typo * Remove extra tab * Fix merge error * Build to update Client.g.cs --- src/Serval.Client/Client.g.cs | 14 ++++++++++---- .../Controllers/DataFilesController.cs | 7 +++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/Serval.Client/Client.g.cs b/src/Serval.Client/Client.g.cs index 0dd43d87..f48a41cc 100644 --- a/src/Serval.Client/Client.g.cs +++ b/src/Serval.Client/Client.g.cs @@ -47,8 +47,11 @@ public partial interface IDataFilesClient /// The file to upload. Max size: 100MB /// File format options: ///
* **Text**: One translation unit (a.k.a., verse) per line - ///
* If there is a tab, the content before the tab is the unique identifier for the line - ///
* Otherwise, no tabs should be used in the file. + ///
* If a line contains a tab, characters before the tab are used as a unique identifier for the line, characters after the tab are understood as the content of the verse, and if there is another tab following the verse content, characters after this second tab are assumed to be column codes like "ss" etc. for sectioning and other formatting. See this example of a tab-delimited text file: + ///
> verse_001_005 (tab) Ὑπομνῆσαι δὲ ὑμᾶς βούλομαι , εἰδότας ὑμᾶς ἅπαξ τοῦτο + ///
> verse_001_006 (tab) Ἀγγέλους τε τοὺς μὴ τηρήσαντας τὴν ἑαυτῶν ἀρχήν , ἀλλὰ (tab) ss + ///
> verse_001_007 (tab) Ὡς Σόδομα καὶ Γόμορρα , καὶ αἱ περὶ αὐτὰς πόλεις (tab) ss + ///
* Otherwise, *no tabs* should be used in the file and a unique identifier will generated for each translation unit based on the line number. ///
* **Paratext**: A complete, zipped Paratext project backup: that is, a .zip archive of files including the USFM files and "Settings.xml" file. To generate a zipped backup for a project in Paratext, navigate to "Paratext/Advanced/Backup project to file..." and follow the dialogue. /// A name to help identify and distinguish the file. ///
Recommendation: Create a multi-part name to distinguish between projects, uses, languages, etc. @@ -231,8 +234,11 @@ public string BaseUrl /// The file to upload. Max size: 100MB /// File format options: ///
* **Text**: One translation unit (a.k.a., verse) per line - ///
* If there is a tab, the content before the tab is the unique identifier for the line - ///
* Otherwise, no tabs should be used in the file. + ///
* If a line contains a tab, characters before the tab are used as a unique identifier for the line, characters after the tab are understood as the content of the verse, and if there is another tab following the verse content, characters after this second tab are assumed to be column codes like "ss" etc. for sectioning and other formatting. See this example of a tab-delimited text file: + ///
> verse_001_005 (tab) Ὑπομνῆσαι δὲ ὑμᾶς βούλομαι , εἰδότας ὑμᾶς ἅπαξ τοῦτο + ///
> verse_001_006 (tab) Ἀγγέλους τε τοὺς μὴ τηρήσαντας τὴν ἑαυτῶν ἀρχήν , ἀλλὰ (tab) ss + ///
> verse_001_007 (tab) Ὡς Σόδομα καὶ Γόμορρα , καὶ αἱ περὶ αὐτὰς πόλεις (tab) ss + ///
* Otherwise, *no tabs* should be used in the file and a unique identifier will generated for each translation unit based on the line number. ///
* **Paratext**: A complete, zipped Paratext project backup: that is, a .zip archive of files including the USFM files and "Settings.xml" file. To generate a zipped backup for a project in Paratext, navigate to "Paratext/Advanced/Backup project to file..." and follow the dialogue. /// A name to help identify and distinguish the file. ///
Recommendation: Create a multi-part name to distinguish between projects, uses, languages, etc. diff --git a/src/Serval.DataFiles/Controllers/DataFilesController.cs b/src/Serval.DataFiles/Controllers/DataFilesController.cs index e333e186..b1e938ae 100644 --- a/src/Serval.DataFiles/Controllers/DataFilesController.cs +++ b/src/Serval.DataFiles/Controllers/DataFilesController.cs @@ -90,8 +90,11 @@ public async Task> GetAsync([NotNull] string id, Cance /// /// File format options: /// * **Text**: One translation unit (a.k.a., verse) per line - /// * If there is a tab, the content before the tab is the unique identifier for the line - /// * Otherwise, no tabs should be used in the file. + /// * If a line contains a tab, characters before the tab are used as a unique identifier for the line, characters after the tab are understood as the content of the verse, and if there is another tab following the verse content, characters after this second tab are assumed to be column codes like "ss" etc. for sectioning and other formatting. See this example of a tab-delimited text file: + /// > verse_001_005 (tab) Ὑπομνῆσαι δὲ ὑμᾶς βούλομαι , εἰδότας ὑμᾶς ἅπαξ τοῦτο + /// > verse_001_006 (tab) Ἀγγέλους τε τοὺς μὴ τηρήσαντας τὴν ἑαυτῶν ἀρχήν , ἀλλὰ (tab) ss + /// > verse_001_007 (tab) Ὡς Σόδομα καὶ Γόμορρα , καὶ αἱ περὶ αὐτὰς πόλεις (tab) ss + /// * Otherwise, *no tabs* should be used in the file and a unique identifier will generated for each translation unit based on the line number. /// * **Paratext**: A complete, zipped Paratext project backup: that is, a .zip archive of files including the USFM files and "Settings.xml" file. To generate a zipped backup for a project in Paratext, navigate to "Paratext/Advanced/Backup project to file..." and follow the dialogue. /// /// From 5e0f893fd496de56b2f4b5c8856003c7ce8fc7a0 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 11 Oct 2023 13:45:11 -0400 Subject: [PATCH 5/5] GRPC 0.9.0 --- src/Serval.Grpc/Serval.Grpc.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Serval.Grpc/Serval.Grpc.csproj b/src/Serval.Grpc/Serval.Grpc.csproj index 00b3d7a9..ed97581e 100644 --- a/src/Serval.Grpc/Serval.Grpc.csproj +++ b/src/Serval.Grpc/Serval.Grpc.csproj @@ -4,7 +4,7 @@ net6.0 enable enable - 0.8.0 + 0.9.0 The Serval gRPC APIs.