Add publish timeout and retry the response check

Ref: https://github.com/niteoweb/easyblognetworks/issues/1217

README.md

@@ -16,22 +16,36 @@ A Github Action that tests the deployment status of a Heroku Review App.
             - master
 
     jobs:
-    review-app-test:
+      review-app-test:
 
         runs-on: ubuntu-latest
 
         steps:
         - name: Run review-app test
-          uses: niteoweb/reviewapps-deploy-status@v1.1.0
+          uses: niteoweb/reviewapps-deploy-status@v1.2.0
           env:
               GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           with:
-              checks: build, response  # check the build status and if the app is responding properly
-              build_time_delay: 5  # delay the checks till the app is built, default is 5 seconds
-              load_time_delay: 5  # delay the checks till the app is loaded after build, default is 5 seconds
-              interval: 10  # interval to repeat checks, default is 10 seconds
-              accepted_responses: 200  # comma separated status codes, optional, default is 200
-              deployments_timeout: 120  # in seconds, optional, default is 120
+            # Checks to be performed, default is all the checks
+            checks: build, response
+
+            # Delay for the application to be built in Heroku, default is 5 seconds
+            build_time_delay: 5
+
+            # Delay for the application to load and start serving, default is 5 seconds
+            load_time_delay: 5
+
+            # Interval for the repeating checks, default is 10 seconds
+            interval: 10
+
+            # Acceptable responses for the response check, default is 200
+            accepted_responses: 200
+
+            # Max time to be spent retrying for the build check, default is 120
+            deployments_timeout: 120
+
+            # Max time to be spent retrying for the response check, default is 120
+            publish_timeout: 120
     ```
 
 > Note: Work flow should include `pull_request` event.
@@ -42,12 +56,13 @@ A Github Action that tests the deployment status of a Heroku Review App.
 
     | Name | Description | Default | 
     |---|---|---|
-    | checks | Comma separated list of checks to be performed  | All checks: build, response  |
-    | build_time_delay | Delay for the build stage of the review app | 5  |
-    | load_time_delay | Delay for the app to load and start serving after it is built | 5  |
-    | interval | Wait for this amount of seconds before retrying the build check  | 10  |
-    | accepted_responses | Allow/Accept the specified status codes (comma separated) | 200  |
-    | deployments_timeout | Maximum waiting time (in seconds) to fetch the deployments | 120 |
+    | checks | Comma separated list of checks to be performed  | build, response |
+    | build_time_delay | Delay for the application to be built in Heroku | 5 |
+    | load_time_delay | Delay for the application to load and start serving | 5 |
+    | interval | Interval for the repeating checks (in seconds) | 10 |
+    | accepted_responses | Acceptable responses for the response check (comma separated) | 200  |
+    | deployments_timeout | Max time to be spent retrying for the build check (in seconds) | 120 |
+    | publish_timeout | Max time to be spent retrying for the response check (in seconds) | 120 |
 
 
 ## Workflow
@@ -74,10 +89,12 @@ Initialize
 │   ├── Yes
 │   │   ├── Do an HTTP request to the app URL.
 │   │   └── Is the HTTP response in the `accepted_responses`?
-│   │       ├── Yes
-│   │       │   └── Continue
 │   │       └── No
-│   │           └── Fail
+│   │           └── Are we past the `publish_timeout`?
+│   │               ├── Yes
+│   │               │   └── Fail
+│   │               └── No
+│   │                   └── Repeat from `Do an HTTP request to the app URL`
 │   └── No
 │       └── Continue
 └── Done (success)

action.yml

@@ -37,3 +37,7 @@ inputs:
     description: Maximum waiting time to fetch the deployments.
     required: false
     default: 120
+  publish_timeout:
+    description: Maximum time to spend retrying the HTTP response check until it succeeds.
+    required: false
+    default: 120

review_app_status.py

@@ -43,14 +43,17 @@ class Args:
     # Delay for the application to load and start serving
     load_time_delay: int
 
-    # Interval between the repeating checks
+    # Interval for the repeating checks
     interval: int
 
     # Acceptable responses for the response check
     accepted_responses: t.List[int]
 
-    # Max time spend retrying for the build check.
-    deployments_timeout: t.List[int]
+    # Max time to be spent retrying for the build check
+    deployments_timeout: int
+
+    # Max time to be spent retrying for the response check
+    publish_timeout: int
 
 
 def _make_github_api_request(url: str) -> dict:
@@ -94,7 +97,7 @@ def _get_github_deployment_status_url(
             if deployment["sha"] == commit_sha:
                 return deployment["statuses_url"]
         time.sleep(interval)
-        timeout = timeout - interval
+        timeout -= interval
         logger.info(f"Waiting for deployments. Will check after {interval} seconds.")
 
     raise ValueError("No deployment found for the lastest commit.")
@@ -133,20 +136,31 @@ def _get_build_data(url: str, interval: int) -> dict:
 
 
 def _check_review_app_deployment_status(
-    review_app_url: str, accepted_responses: t.List[int]
+    review_app_url: str, accepted_responses: t.List[int], timeout: int, interval: int
 ):
     """Check Review App deployment status code against accepted_responses.
     
     Inputs:
         review_app_url: URL of the Review App to be checked.
-        accepted_responses: status codes to be accepted.
+        accepted_responses: Status codes to be accepted.
+        timeout: Maximum time to spend retrying the HTTP response check until it succeeds.
+        interval: Interval for each HTTP response check.
     """
-    time.sleep(5)  # Let the deployment breathe.
-    r = requests.get(review_app_url)
-    review_app_status = r.status_code
-    logger.info(f"Review app status: {review_app_status}")
-    if review_app_status not in accepted_responses:
-        r.raise_for_status()
+    if interval > timeout:
+        raise ValueError("Interval can't be greater than publish_timeout.")
+
+    while timeout > 0:
+        r = requests.get(review_app_url)
+        review_app_status = r.status_code
+        logger.info(f"Review app status: {review_app_status}")
+        if review_app_status in accepted_responses:
+            return
+        time.sleep(interval)
+        timeout -= interval
+
+    raise TimeoutError(
+        f"Did not get any of the accepted status {accepted_responses} in the given time."
+    )
 
 
 def main() -> None:
@@ -161,6 +175,7 @@ def main() -> None:
         load_time_delay=int(os.environ["INPUT_LOAD_TIME_DELAY"]),
         interval=int(os.environ["INPUT_INTERVAL"]),
         deployments_timeout=int(os.environ["INPUT_DEPLOYMENTS_TIMEOUT"]),
+        publish_timeout=int(os.environ["INPUT_PUBLISH_TIMEOUT"]),
         accepted_responses=[
             int(x.strip()) for x in os.environ["INPUT_ACCEPTED_RESPONSES"].split(",")
         ],
@@ -195,16 +210,19 @@ def main() -> None:
         build_state = reviewapp_build_data["state"]
         if build_state != BuildStates.success.value:
             raise ValueError(f"Review App Build state: {build_state}")
-        
+
     if Checks.response in args.checks:
         # Delay the checks till the app is loads
         logger.info(f"Load time delay: {args.load_time_delay} seconds")
         time.sleep(args.load_time_delay)
-        
+
         # Check the HTTP response from app URL
         review_app_url = f"https://{reviewapp_build_data['environment']}.herokuapp.com"
         _check_review_app_deployment_status(
-            review_app_url=review_app_url, accepted_responses=args.accepted_responses
+            review_app_url=review_app_url,
+            accepted_responses=args.accepted_responses,
+            timeout=args.publish_timeout,
+            interval=args.interval,
         )
 
     print("Successful")

tests.py

@@ -201,7 +201,7 @@ def test_reviewapp_deployment_success(caplog):
 
     responses.add(responses.GET, "https://foo-pr-bar.com", status=200)
 
-    _check_review_app_deployment_status("https://foo-pr-bar.com", [200, 302])
+    _check_review_app_deployment_status("https://foo-pr-bar.com", [200, 302], 5, 5)
     assert len(responses.calls) == 1
     assert len(caplog.records) == 1
     assert caplog.records[0].message == "Review app status: 200"
@@ -213,24 +213,40 @@ def test_check_review_app_status_fail(caplog):
 
     responses.add(responses.GET, "https://foo-pr-bar.com", status=503)
 
-    with pytest.raises(exceptions.HTTPError) as excinfo:
-        _check_review_app_deployment_status("https://foo-pr-bar.com", [200, 302])
+    with pytest.raises(TimeoutError) as excinfo:
+        _check_review_app_deployment_status("https://foo-pr-bar.com", [200, 302], 5, 5)
 
     assert len(responses.calls) == 1
     assert (
-        "503 Server Error: Service Unavailable for url: https://foo-pr-bar.com/"
+        "Did not get any of the accepted status [200, 302] in the given time."
         in str(excinfo.value)
     )
     assert caplog.records[0].message == "Review app status: 503"
 
 
+@responses.activate
+def test_check_review_app_status_interval_greater_failure():
+
+    from review_app_status import _check_review_app_deployment_status
+
+    with pytest.raises(ValueError) as excinfo:
+        url = _check_review_app_deployment_status(
+            review_app_url="https://foo.bar",
+            accepted_responses=[200],
+            timeout=3,
+            interval=4,
+        )
+
+    assert "Interval can't be greater than publish_timeout." in str(excinfo.value)
+
+
 @responses.activate
 def test_check_review_app_custom_status_success(caplog):
     from review_app_status import _check_review_app_deployment_status
 
     responses.add(responses.GET, "https://foo-pr-bar.com", status=302)
 
-    _check_review_app_deployment_status("https://foo-pr-bar.com", [200, 302])
+    _check_review_app_deployment_status("https://foo-pr-bar.com", [200, 302], 5, 5)
     assert len(responses.calls) == 1
     assert len(caplog.records) == 1
     assert caplog.records[0].message == "Review app status: 302"
@@ -243,6 +259,7 @@ def test_check_review_app_custom_status_success(caplog):
         "INPUT_BUILD_TIME_DELAY": "5",
         "INPUT_LOAD_TIME_DELAY": "5",
         "INPUT_DEPLOYMENTS_TIMEOUT": "20",
+        "INPUT_PUBLISH_TIMEOUT": "20",
         "INPUT_INTERVAL": "10",
         "INPUT_ACCEPTED_RESPONSES": "200, 302",
         "GITHUB_EVENT_PATH": "./test_path",
@@ -283,7 +300,10 @@ def test_main_success(
         url="http://foo.bar/deployment_status", interval=10
     )
     mock_review_app_deployment.assert_called_once_with(
-        review_app_url="https://foo-pr-bar.herokuapp.com", accepted_responses=[200, 302]
+        review_app_url="https://foo-pr-bar.herokuapp.com",
+        accepted_responses=[200, 302],
+        timeout=20,
+        interval=10,
     )
 
     out, err = capsys.readouterr()
@@ -297,6 +317,7 @@ def test_main_success(
         "INPUT_BUILD_TIME_DELAY": "5",
         "INPUT_LOAD_TIME_DELAY": "5",
         "INPUT_DEPLOYMENTS_TIMEOUT": "20",
+        "INPUT_PUBLISH_TIMEOUT": "20",
         "INPUT_INTERVAL": "10",
         "INPUT_ACCEPTED_RESPONSES": "200, 302",
         "GITHUB_EVENT_PATH": "./test_path",