-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
111 running long tasks on mmoda (#121)
* saving callback URL to notebook working dir * callback test updated * fixing test_find_notebooks * parameter callback renamed to callback_url, function _pass_callback renamed to _pass_callback_url * _pass_callback_url parameter comments updated * _pass_callback_url parameter comments updated * bugfixing after refactoring * verifying progress report from callback.ipynb * trying to fix issue problem with outdated version of owlready2 * extra asserts added to tests/test_callback.py * merging changes from master
- Loading branch information
Showing
5 changed files
with
216 additions
and
21 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| import pytest | ||
| import nb2workflow | ||
| import logging | ||
| import threading | ||
| import time | ||
| from os import path | ||
| import json | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
| status_callback_file = "status.json" | ||
|
|
||
| @pytest.fixture | ||
| def app(): | ||
| app = nb2workflow.service.app | ||
| app.notebook_adapters = nb2workflow.nbadapter.find_notebooks('tests/testfiles') | ||
| nb2workflow.service.setup_routes(app) | ||
| print("creating app") | ||
| return app | ||
|
|
||
|
|
||
| def test_progress_callback(client): | ||
| callback_url = 'file://' + status_callback_file | ||
| query_string = dict( | ||
| a=20, | ||
| _async_request='yes', | ||
| _async_request_callback=callback_url) | ||
|
|
||
| r = client.get('/api/v1.0/get/callback', | ||
| query_string=query_string) | ||
|
|
||
| assert r.status_code == 201 | ||
|
|
||
| logger.info(r.json) | ||
|
|
||
| from nb2workflow.service import AsyncWorker | ||
|
|
||
| def test_worker_run(): | ||
| AsyncWorker('test-worker').run_one() | ||
|
|
||
| test_worker_thread = threading.Thread(target=test_worker_run) | ||
| test_worker_thread.start() | ||
|
|
||
| while True: | ||
| options = client.get('/api/v1.0/options') | ||
| assert options.status_code == 200 | ||
|
|
||
| r = client.get('/api/v1.0/get/callback', | ||
| query_string=query_string) | ||
|
|
||
| logger.info('service returns %s %s', r, r.json) | ||
|
|
||
| if r.json['workflow_status'] == 'done': | ||
| logger.info('workflow done!') | ||
| break | ||
|
|
||
| time.sleep(0.1) | ||
|
|
||
| test_worker_thread.join() | ||
| assert 'data' in r.json | ||
| assert 'output' in r.json['data'] | ||
| assert 'callback' in r.json['data']['output'] | ||
| assert r.json['data']['output']['callback'] == callback_url | ||
|
|
||
| workdir = r.json['data']['jobdir'] | ||
| with open(path.join(workdir, status_callback_file)) as json_file: | ||
| progress_params = json.load(json_file) | ||
|
|
||
| test_data = dict(action='progress', stage='simulation', progress=50, substage='spectra', subprogress=30, message='some message') | ||
| assert progress_params == test_data | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| { | ||
| "cells": [ | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "initial_id", | ||
| "metadata": { | ||
| "editable": true, | ||
| "slideshow": { | ||
| "slide_type": "" | ||
| }, | ||
| "tags": [ | ||
| "parameters" | ||
| ] | ||
| }, | ||
| "outputs": [], | ||
| "source": [ | ||
| "a = 10" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "outputs": [], | ||
| "source": [ | ||
| "!pip install git+https://github.com/oda-hub/oda_api.git@217-add-functionality-to-send-callback-messages" | ||
| ], | ||
| "metadata": { | ||
| "collapsed": false | ||
| }, | ||
| "id": "e5b37dc052a7bc" | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "ae02114d712f3ebc", | ||
| "metadata": { | ||
| "collapsed": false, | ||
| "editable": true, | ||
| "slideshow": { | ||
| "slide_type": "" | ||
| }, | ||
| "tags": [] | ||
| }, | ||
| "outputs": [], | ||
| "source": [ | ||
| "# from os.path import isfile\n", | ||
| "# callback_file = \".oda_api_callback\"\n", | ||
| "# if isfile(callback_file):\n", | ||
| "# with open(callback_file, 'r') as file:\n", | ||
| "# callback = file.read().strip()\n", | ||
| "# else:\n", | ||
| "# callback = ''" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "outputs": [], | ||
| "source": [ | ||
| "from oda_api.api import ProgressReporter\n", | ||
| "pr = ProgressReporter()\n", | ||
| "assert pr.enabled\n", | ||
| "pr.report_progress(stage='simulation', progress=50, substage='spectra', subprogress=30, message='some message')\n", | ||
| "callback = pr._callback" | ||
| ], | ||
| "metadata": { | ||
| "collapsed": false | ||
| }, | ||
| "id": "3c6a5d05bbbd359f" | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "256b6a6696ecc58d", | ||
| "metadata": { | ||
| "collapsed": false, | ||
| "editable": true, | ||
| "slideshow": { | ||
| "slide_type": "" | ||
| }, | ||
| "tags": [ | ||
| "outputs" | ||
| ] | ||
| }, | ||
| "outputs": [], | ||
| "source": [ | ||
| "callback" | ||
| ] | ||
| } | ||
| ], | ||
| "metadata": { | ||
| "kernelspec": { | ||
| "display_name": "Python 3 (ipykernel)", | ||
| "language": "python", | ||
| "name": "python3" | ||
| }, | ||
| "language_info": { | ||
| "codemirror_mode": { | ||
| "name": "ipython", | ||
| "version": 3 | ||
| }, | ||
| "file_extension": ".py", | ||
| "mimetype": "text/x-python", | ||
| "name": "python", | ||
| "nbconvert_exporter": "python", | ||
| "pygments_lexer": "ipython3", | ||
| "version": "3.9.18" | ||
| } | ||
| }, | ||
| "nbformat": 4, | ||
| "nbformat_minor": 5 | ||
| } |