From a967c006993adb277c92c5dcbcf60cc037e456c0 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Wed, 19 Jun 2024 11:05:31 +0100
Subject: [PATCH 01/53] Master Points to 8.4.0 (#6159)

---
 cylc/flow/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cylc/flow/__init__.py b/cylc/flow/__init__.py
index 40718ebdf7e..34897137506 100644
--- a/cylc/flow/__init__.py
+++ b/cylc/flow/__init__.py
@@ -53,7 +53,7 @@ def environ_init():
 
 environ_init()
 
-__version__ = '8.3.1.dev'
+__version__ = '8.4.0.dev'
 
 
 def iter_entry_points(entry_point_name):

From c3d5f1d158e06609d39c5a05b6cfa7952f4c34da Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Wed, 19 Jun 2024 16:34:27 +0100
Subject: [PATCH 02/53] lint: address RUF015

---
 cylc/flow/broadcast_report.py | 2 +-
 cylc/flow/id_cli.py           | 4 ++--
 cylc/flow/scripts/show.py     | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cylc/flow/broadcast_report.py b/cylc/flow/broadcast_report.py
index 72fedb4cbef..cb44f2212f3 100644
--- a/cylc/flow/broadcast_report.py
+++ b/cylc/flow/broadcast_report.py
@@ -72,7 +72,7 @@ def get_broadcast_change_iter(modified_settings, is_cancel=False):
         value = setting
         keys_str = ""
         while isinstance(value, dict):
-            key, value = list(value.items())[0]
+            key, value = next(iter(value.items()))
             if isinstance(value, dict):
                 keys_str += "[" + key + "]"
             else:
diff --git a/cylc/flow/id_cli.py b/cylc/flow/id_cli.py
index 9c7493fd612..cad615885e4 100644
--- a/cylc/flow/id_cli.py
+++ b/cylc/flow/id_cli.py
@@ -347,7 +347,7 @@ async def parse_ids_async(
     if src:
         if not flow_file_path:
             # get the workflow file path from the run dir
-            flow_file_path = get_flow_file(list(workflows)[0])
+            flow_file_path = get_flow_file(next(iter(workflows)))
         return workflows, flow_file_path
     return workflows, multi_mode
 
@@ -375,7 +375,7 @@ async def parse_id_async(
             'max_tasks': 1,
         },
     )
-    workflow_id = list(workflows)[0]
+    workflow_id = next(iter(workflows))
     tokens_list = workflows[workflow_id]
     tokens: Optional[Tokens]
     if tokens_list:
diff --git a/cylc/flow/scripts/show.py b/cylc/flow/scripts/show.py
index b146f339e10..2fdfaf7fde4 100755
--- a/cylc/flow/scripts/show.py
+++ b/cylc/flow/scripts/show.py
@@ -490,7 +490,7 @@ def main(_, options: 'Values', *ids) -> None:
         constraint='mixed',
         max_workflows=1,
     )
-    workflow_id = list(workflow_args)[0]
+    workflow_id = next(iter(workflow_args))
     tokens_list = workflow_args[workflow_id]
 
     if tokens_list and options.task_defs:

From fcabe74e83de14d2b9fa09c6279696216a1892a0 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Wed, 19 Jun 2024 17:11:42 +0100
Subject: [PATCH 03/53] lint: address B904

---
 cylc/flow/clean.py                        |  9 ++--
 cylc/flow/command_validation.py           |  2 +-
 cylc/flow/commands.py                     |  4 +-
 cylc/flow/config.py                       | 55 +++++++++++++----------
 cylc/flow/cycling/integer.py              |  2 +-
 cylc/flow/cycling/iso8601.py              |  6 +--
 cylc/flow/dbstatecheck.py                 |  6 +--
 cylc/flow/graph_parser.py                 |  8 ++--
 cylc/flow/host_select.py                  |  2 +-
 cylc/flow/id.py                           |  2 +-
 cylc/flow/id_cli.py                       |  4 +-
 cylc/flow/install.py                      |  4 +-
 cylc/flow/install_plugins/log_vc_info.py  |  2 +-
 cylc/flow/loggingutil.py                  |  2 +-
 cylc/flow/main_loop/__init__.py           |  4 +-
 cylc/flow/main_loop/health_check.py       |  2 +-
 cylc/flow/network/__init__.py             | 15 ++++---
 cylc/flow/network/client.py               |  2 +-
 cylc/flow/network/resolvers.py            |  2 +-
 cylc/flow/network/schema.py               |  2 +-
 cylc/flow/network/ssh_client.py           |  2 +-
 cylc/flow/param_expand.py                 | 14 +++---
 cylc/flow/parsec/config.py                |  6 ++-
 cylc/flow/parsec/empysupport.py           |  2 +-
 cylc/flow/parsec/fileparse.py             | 12 +++--
 cylc/flow/parsec/jinja2support.py         |  6 +--
 cylc/flow/parsec/upgrade.py               |  2 +-
 cylc/flow/parsec/validate.py              | 42 +++++++++++------
 cylc/flow/pathutil.py                     |  6 +--
 cylc/flow/prerequisite.py                 |  3 +-
 cylc/flow/scripts/broadcast.py            |  2 +-
 cylc/flow/scripts/cat_log.py              |  2 +-
 cylc/flow/scripts/clean.py                |  2 +-
 cylc/flow/scripts/cylc.py                 |  2 +-
 cylc/flow/scripts/dump.py                 |  3 +-
 cylc/flow/scripts/ext_trigger.py          |  2 +-
 cylc/flow/scripts/get_workflow_contact.py |  4 +-
 cylc/flow/scripts/lint.py                 |  2 +-
 cylc/flow/scripts/report_timings.py       |  4 +-
 cylc/flow/scripts/validate.py             | 21 +++++----
 cylc/flow/scripts/workflow_state.py       |  5 ++-
 cylc/flow/task_id.py                      |  3 +-
 cylc/flow/time_parser.py                  |  2 +-
 cylc/flow/tui/data.py                     |  4 +-
 cylc/flow/util.py                         |  4 +-
 cylc/flow/workflow_db_mgr.py              |  4 +-
 cylc/flow/workflow_files.py               |  8 ++--
 cylc/flow/xtrigger_mgr.py                 | 12 ++---
 cylc/flow/xtriggers/wall_clock.py         |  4 +-
 49 files changed, 186 insertions(+), 134 deletions(-)

diff --git a/cylc/flow/clean.py b/cylc/flow/clean.py
index aec2c4efeb4..3378cfd1591 100644
--- a/cylc/flow/clean.py
+++ b/cylc/flow/clean.py
@@ -129,7 +129,7 @@ def _clean_check(opts: 'Values', id_: str, run_dir: Path) -> None:
     except ContactFileExists as exc:
         raise ServiceFileError(
             f"Cannot clean running workflow {id_}.\n\n{exc}"
-        )
+        ) from None
 
 
 def init_clean(id_: str, opts: 'Values') -> None:
@@ -173,7 +173,7 @@ def init_clean(id_: str, opts: 'Values') -> None:
             try:
                 platform_names = get_platforms_from_db(local_run_dir)
             except ServiceFileError as exc:
-                raise ServiceFileError(f"Cannot clean {id_} - {exc}")
+                raise ServiceFileError(f"Cannot clean {id_} - {exc}") from None
             except sqlite3.OperationalError as exc:
                 # something went wrong with the query
                 # e.g. the table/field we need isn't there
@@ -184,7 +184,7 @@ def init_clean(id_: str, opts: 'Values') -> None:
                     ' with to remove it.'
                     '\nOtherwise please delete the database file.'
                 )
-                raise ServiceFileError(f"Cannot clean {id_} - {exc}")
+                raise ServiceFileError(f"Cannot clean {id_} - {exc}") from exc
 
         if platform_names and platform_names != {'localhost'}:
             remote_clean(
@@ -359,7 +359,8 @@ def remote_clean(
     except PlatformLookupError as exc:
         raise PlatformLookupError(
             f"Cannot clean {id_} on remote platforms as the workflow database "
-            f"is out of date/inconsistent with the global config - {exc}")
+            f"is out of date/inconsistent with the global config - {exc}"
+        ) from None
 
     queue: Deque[RemoteCleanQueueTuple] = deque()
     remote_clean_cmd = partial(
diff --git a/cylc/flow/command_validation.py b/cylc/flow/command_validation.py
index 1c57d452c43..ea67f00206e 100644
--- a/cylc/flow/command_validation.py
+++ b/cylc/flow/command_validation.py
@@ -70,7 +70,7 @@ def flow_opts(flows: List[str], flow_wait: bool) -> None:
             try:
                 int(val)
             except ValueError:
-                raise InputError(ERR_OPT_FLOW_VAL.format(val))
+                raise InputError(ERR_OPT_FLOW_VAL.format(val)) from None
 
     if flow_wait and flows[0] in [FLOW_NEW, FLOW_NONE]:
         raise InputError(ERR_OPT_FLOW_WAIT)
diff --git a/cylc/flow/commands.py b/cylc/flow/commands.py
index a4ea43df5cf..2abf7f1c773 100644
--- a/cylc/flow/commands.py
+++ b/cylc/flow/commands.py
@@ -208,7 +208,7 @@ async def stop(
         try:
             mode = StopMode(mode)
         except ValueError:
-            raise CommandFailedError(f"Invalid stop mode: '{mode}'")
+            raise CommandFailedError(f"Invalid stop mode: '{mode}'") from None
         schd._set_stop(mode)
         if mode is StopMode.REQUEST_KILL:
             schd.time_next_kill = time()
@@ -304,7 +304,7 @@ async def set_verbosity(schd: 'Scheduler', level: Union[int, str]):
         lvl = int(level)
         LOG.setLevel(lvl)
     except (TypeError, ValueError) as exc:
-        raise CommandFailedError(exc)
+        raise CommandFailedError(exc) from None
     cylc.flow.flags.verbosity = log_level_to_verbosity(lvl)
 
 
diff --git a/cylc/flow/config.py b/cylc/flow/config.py
index 5b7738f3e6c..fdcc10adb05 100644
--- a/cylc/flow/config.py
+++ b/cylc/flow/config.py
@@ -199,11 +199,11 @@ def interpolate_template(tmpl, params_dict):
     try:
         return tmpl % params_dict
     except KeyError:
-        raise ParamExpandError('bad parameter')
+        raise ParamExpandError('bad parameter') from None
     except TypeError:
-        raise ParamExpandError('wrong data type for parameter')
+        raise ParamExpandError('wrong data type for parameter') from None
     except ValueError:
-        raise ParamExpandError('bad template syntax')
+        raise ParamExpandError('bad template syntax') from None
 
 
 class WorkflowConfig:
@@ -480,8 +480,8 @@ def __init__(
                             get_interval(offset_string).standardise())
                     except IntervalParsingError:
                         raise WorkflowConfigError(
-                            "Illegal %s spec: %s" % (
-                                s_type, offset_string))
+                            "Illegal %s spec: %s" % (s_type, offset_string)
+                        ) from None
                     extn = "(" + offset_string + ")"
 
                 # Replace family names with members.
@@ -709,7 +709,7 @@ def process_initial_cycle_point(self) -> None:
                 try:
                     icp = ingest_time(orig_icp, get_current_time_string())
                 except IsodatetimeError as exc:
-                    raise WorkflowConfigError(str(exc))
+                    raise WorkflowConfigError(str(exc)) from None
         if orig_icp != icp:
             # now/next()/previous() was used, need to store
             # evaluated point in DB
@@ -761,7 +761,7 @@ def process_start_cycle_point(self) -> None:
                     for taskid in self.options.starttask
                 ]
             except ValueError as exc:
-                raise InputError(str(exc))
+                raise InputError(str(exc)) from None
             self.start_point = min(
                 get_point(cycle).standardise()
                 for cycle in cycle_points if cycle
@@ -1114,7 +1114,7 @@ def _check_completion_expression(self, task_name: str, expr: str) -> None:
                     f'\n  {expr}'
                     '\nThe "finished" output cannot be used in completion'
                     ' expressions, use "succeeded or failed".'
-                )
+                ) from None
 
             for alt_qualifier, qualifier in ALT_QUALIFIERS.items():
                 _alt_compvar = trigger_to_completion_variable(alt_qualifier)
@@ -1125,21 +1125,21 @@ def _check_completion_expression(self, task_name: str, expr: str) -> None:
                         f'\n  {expr}'
                         f'\nUse "{_compvar}" not "{_alt_compvar}" '
                         'in completion expressions.'
-                    )
+                    ) from None
 
             raise WorkflowConfigError(
                 # NOTE: str(exc) == "name 'x' is not defined" tested in
                 # tests/integration/test_optional_outputs.py
                 f'Error in [runtime][{task_name}]completion:'
                 f'\n{error}'
-            )
+            ) from None
         except Exception as exc:  # includes InvalidCompletionExpression
             # expression contains non-whitelisted syntax or any other error in
             # the expression e.g. SyntaxError
             raise WorkflowConfigError(
                 f'Error in [runtime][{task_name}]completion:'
                 f'\n{str(exc)}'
-            )
+            ) from None
 
         # ensure consistency between the graph and the completion expression
         for compvar in (
@@ -1415,11 +1415,12 @@ def compute_family_tree(self):
                     c3_single.mro(name))
             except RecursionError:
                 raise WorkflowConfigError(
-                    "circular [runtime] inheritance?")
+                    "circular [runtime] inheritance?"
+                ) from None
             except Exception as exc:
                 # catch inheritance errors
                 # TODO - specialise MRO exceptions
-                raise WorkflowConfigError(str(exc))
+                raise WorkflowConfigError(str(exc)) from None
 
         for name in self.cfg['runtime']:
             ancestors = self.runtime['linearized ancestors'][name]
@@ -1758,7 +1759,7 @@ def _check_task_event_handlers(self):
                                     f' {taskdef.name}:'
                                     f' {handler_template}:'
                                     f' {repr(exc)}'
-                                )
+                                ) from None
 
     def _check_special_tasks(self):
         """Check declared special tasks are valid, and detect special
@@ -1865,7 +1866,9 @@ def generate_triggers(self, lexpression, left_nodes, right, seq,
         try:
             expr_list = listify(lexpression)
         except SyntaxError:
-            raise WorkflowConfigError('Error in expression "%s"' % lexpression)
+            raise WorkflowConfigError(
+                'Error in expression "%s"' % lexpression
+            ) from None
 
         triggers = {}
         xtrig_labels = set()
@@ -1942,7 +1945,9 @@ def generate_triggers(self, lexpression, left_nodes, right, seq,
                 xtrig = xtrigs[label]
             except KeyError:
                 if label != 'wall_clock':
-                    raise WorkflowConfigError(f"xtrigger not defined: {label}")
+                    raise WorkflowConfigError(
+                        f"xtrigger not defined: {label}"
+                    ) from None
                 else:
                     # Allow "@wall_clock" in graph as implicit zero-offset.
                     xtrig = SubFuncContext('wall_clock', 'wall_clock', [], {})
@@ -2276,7 +2281,7 @@ def load_graph(self):
                 msg += ' (final cycle point=%s)' % fcp
                 if isinstance(exc, CylcError):
                     msg += ' %s' % exc.args[0]
-                raise WorkflowConfigError(msg)
+                raise WorkflowConfigError(msg) from None
             self.sequences.append(seq)
             parser = GraphParser(
                 family_map,
@@ -2431,7 +2436,7 @@ def get_taskdef(
             except TaskDefError as exc:
                 if orig_expr:
                     LOG.error(orig_expr)
-                raise WorkflowConfigError(str(exc))
+                raise WorkflowConfigError(str(exc)) from None
             else:
                 # Record custom message outputs from [runtime].
                 for output, message in (
@@ -2443,14 +2448,14 @@ def get_taskdef(
                             f'Invalid task output "'
                             f'[runtime][{name}][outputs]'
                             f'{output} = {message}" - {msg}'
-                        )
+                        ) from None
                     valid, msg = TaskMessageValidator.validate(message)
                     if not valid:
                         raise WorkflowConfigError(
                             f'Invalid task message "'
                             f'[runtime][{name}][outputs]'
                             f'{output} = {message}" - {msg}'
-                        )
+                        ) from None
                     self.taskdefs[name].add_output(output, message)
 
         return self.taskdefs[name]
@@ -2462,7 +2467,7 @@ def _get_taskdef(self, name: str) -> TaskDef:
         try:
             rtcfg = self.cfg['runtime'][name]
         except KeyError:
-            raise WorkflowConfigError("Task not defined: %s" % name)
+            raise WorkflowConfigError("Task not defined: %s" % name) from None
         # We may want to put in some handling for cases of changing the
         # initial cycle via restart (accidentally or otherwise).
 
@@ -2554,7 +2559,9 @@ def process_metadata_urls(self):
                     'workflow': self.workflow,
                 }
             except (KeyError, ValueError):
-                raise InputError(f'Invalid template [meta]URL: {url}')
+                raise InputError(
+                    f'Invalid template [meta]URL: {url}'
+                ) from None
             else:
                 LOG.warning(
                     'Detected deprecated template variables in [meta]URL.'
@@ -2590,7 +2597,9 @@ def process_metadata_urls(self):
                         'task': name,
                     }
                 except (KeyError, ValueError):
-                    raise InputError(f'Invalid template [meta]URL: {url}')
+                    raise InputError(
+                        f'Invalid template [meta]URL: {url}'
+                    ) from None
                 else:
                     LOG.warning(
                         'Detected deprecated template variables in'
diff --git a/cylc/flow/cycling/integer.py b/cylc/flow/cycling/integer.py
index 749c651fc08..20cc3765670 100644
--- a/cylc/flow/cycling/integer.py
+++ b/cylc/flow/cycling/integer.py
@@ -150,7 +150,7 @@ def standardise(self):
         try:
             self.value = str(int(self))
         except (TypeError, ValueError) as exc:
-            raise PointParsingError(type(self), self.value, exc)
+            raise PointParsingError(type(self), self.value, exc) from None
         return self
 
     def __int__(self):
diff --git a/cylc/flow/cycling/iso8601.py b/cylc/flow/cycling/iso8601.py
index a66ce3f5ba0..2ab311df425 100644
--- a/cylc/flow/cycling/iso8601.py
+++ b/cylc/flow/cycling/iso8601.py
@@ -102,7 +102,7 @@ def standardise(self):
                     WorkflowSpecifics.NUM_EXPANDED_YEAR_DIGITS)
             else:
                 message = str(exc)
-            raise PointParsingError(type(self), self.value, message)
+            raise PointParsingError(type(self), self.value, message) from None
         return self
 
     def sub(self, other):
@@ -176,7 +176,7 @@ def standardise(self):
         try:
             self.value = str(interval_parse(self.value))
         except IsodatetimeError:
-            raise IntervalParsingError(type(self), self.value)
+            raise IntervalParsingError(type(self), self.value) from None
         return self
 
     def add(self, other):
@@ -782,7 +782,7 @@ def prev_next(
             raise WorkflowConfigError(
                 f'Invalid offset: {my_time}:'
                 f' Offset lists are semicolon separated, try {suggest}'
-            )
+            ) from None
 
         timepoints.append(parsed_point + now)
 
diff --git a/cylc/flow/dbstatecheck.py b/cylc/flow/dbstatecheck.py
index b38c394ccdb..1890c697c71 100644
--- a/cylc/flow/dbstatecheck.py
+++ b/cylc/flow/dbstatecheck.py
@@ -78,7 +78,7 @@ def __init__(self, rund, workflow, db_path=None):
         try:
             self.db_point_fmt = self._get_db_point_format()
             self.c7_back_compat_mode = False
-        except sqlite3.OperationalError as exc:
+        except sqlite3.OperationalError:
             # BACK COMPAT: Cylc 7 DB (see method below).
             try:
                 self.db_point_fmt = self._get_db_point_format_compat()
@@ -86,7 +86,7 @@ def __init__(self, rund, workflow, db_path=None):
             except sqlite3.OperationalError:
                 with suppress(Exception):
                     self.conn.close()
-                raise exc  # original error
+                raise
 
     def __enter__(self):
         return self
@@ -137,7 +137,7 @@ def adjust_point_to_db(self, cycle, offset):
             raise InputError(
                 f'Cycle point "{cycle}" is not compatible'
                 f' with DB point format "{self.db_point_fmt}"'
-            )
+            ) from None
         return cycle
 
     @staticmethod
diff --git a/cylc/flow/graph_parser.py b/cylc/flow/graph_parser.py
index efbce16fb36..3fbfd8c1754 100644
--- a/cylc/flow/graph_parser.py
+++ b/cylc/flow/graph_parser.py
@@ -346,7 +346,7 @@ def parse_graph(self, graph_string: str) -> None:
                         raise GraphParseError(
                             f"Dangling {seq}:"
                             f"{this_line}"
-                        )
+                        ) from None
             part_lines.append(this_line)
 
             # Check that a continuation sequence doesn't end this line and
@@ -638,7 +638,8 @@ def _proc_dep_pair(
                     except KeyError:
                         # "FAM:bad => foo" in LHS (includes "FAM => bar" too).
                         raise GraphParseError(
-                            f"Illegal family trigger in {expr}")
+                            f"Illegal family trigger in {expr}"
+                        ) from None
                 else:
                     # Not a family.
                     if trig in self.__class__.fam_to_mem_trigger_map:
@@ -911,7 +912,8 @@ def _compute_triggers(
                 except KeyError:
                     # Illegal family trigger on RHS of a pair.
                     raise GraphParseError(
-                        f"Illegal family trigger: {name}:{output}")
+                        f"Illegal family trigger: {name}:{output}"
+                    ) from None
             else:
                 fam = False
                 if not output:
diff --git a/cylc/flow/host_select.py b/cylc/flow/host_select.py
index 0eb34d088ca..69e32c68a71 100644
--- a/cylc/flow/host_select.py
+++ b/cylc/flow/host_select.py
@@ -373,7 +373,7 @@ def _filter_by_ranking(hosts, rankings, results, data=None):
                     f'\n    Expression: {item}'
                     f'\n    Configuration: {GLBL_CFG_STR}'
                     f'\n    Error: {exc}'
-                )
+                ) from None
             if isinstance(result, bool):
                 host_rankings[item] = result
                 data[host][item] = result
diff --git a/cylc/flow/id.py b/cylc/flow/id.py
index 58fff7fa7bc..f2c8b05b4a1 100644
--- a/cylc/flow/id.py
+++ b/cylc/flow/id.py
@@ -128,7 +128,7 @@ def __getitem__(self, key):
             return dict.__getitem__(self, key)
         except KeyError:
             if key not in self._KEYS:
-                raise ValueError(f'Invalid token: {key}')
+                raise ValueError(f'Invalid token: {key}') from None
             return None
 
     def __str__(self):
diff --git a/cylc/flow/id_cli.py b/cylc/flow/id_cli.py
index cad615885e4..35afbf80d5a 100644
--- a/cylc/flow/id_cli.py
+++ b/cylc/flow/id_cli.py
@@ -167,9 +167,9 @@ def _parse_cli(*ids: str) -> List[Tokens]:
                     # this ID is invalid with or without the trailing slash
                     tokens = cli_tokenise(id_[:-1])
                 except ValueError:
-                    raise InputError(f'Invalid ID: {id_}')
+                    raise InputError(f'Invalid ID: {id_}') from None
             else:
-                raise InputError(f'Invalid ID: {id_}')
+                raise InputError(f'Invalid ID: {id_}') from None
         is_partial = tokens.get('workflow') and not tokens.get('cycle')
         is_relative = not tokens.get('workflow')
 
diff --git a/cylc/flow/install.py b/cylc/flow/install.py
index 27810f72e97..2e94943d9d6 100644
--- a/cylc/flow/install.py
+++ b/cylc/flow/install.py
@@ -328,7 +328,7 @@ def install_workflow(
         # This occurs when the file exists but is _not_ a directory.
         raise WorkflowFilesError(
             f"Cannot install as there is an existing file at {rundir}."
-        )
+        ) from None
     if relink:
         link_runN(rundir)
     rsync_cmd = get_rsync_rund_cmd(source, rundir)
@@ -529,7 +529,7 @@ def parse_cli_sym_dirs(symlink_dirs: str) -> Dict[str, Dict[str, Any]]:
                 'There is an error in --symlink-dirs option:'
                 f' {pair}. Try entering option in the form '
                 '--symlink-dirs=\'log=$DIR, share=$DIR2, ...\''
-            )
+            ) from None
         if key not in possible_symlink_dirs:
             dirs = ', '.join(possible_symlink_dirs)
             raise InputError(
diff --git a/cylc/flow/install_plugins/log_vc_info.py b/cylc/flow/install_plugins/log_vc_info.py
index 29d861f7654..41479d6f1ed 100644
--- a/cylc/flow/install_plugins/log_vc_info.py
+++ b/cylc/flow/install_plugins/log_vc_info.py
@@ -253,7 +253,7 @@ def _run_cmd(
     except FileNotFoundError as exc:
         # This will only be raised if the VCS command is not installed,
         # otherwise Popen() will succeed with a non-zero return code
-        raise VCSNotInstalledError(vcs, exc)
+        raise VCSNotInstalledError(vcs, exc) from None
     if stdout == PIPE:
         out, err = pipe_poller(proc, proc.stdout, proc.stderr)
     else:
diff --git a/cylc/flow/loggingutil.py b/cylc/flow/loggingutil.py
index 3d77bdcb037..a6f1fbee558 100644
--- a/cylc/flow/loggingutil.py
+++ b/cylc/flow/loggingutil.py
@@ -206,7 +206,7 @@ def should_rollover(self, record: logging.LogRecord) -> bool:
             self.stream.seek(0, 2)
         except ValueError as exc:
             # intended to catch - ValueError: I/O operation on closed file
-            raise SystemExit(exc)
+            raise SystemExit(exc) from None
         return self.stream.tell() + len(msg.encode('utf8')) >= max_bytes
 
     @property
diff --git a/cylc/flow/main_loop/__init__.py b/cylc/flow/main_loop/__init__.py
index 2350153842c..e9f9f35f5da 100644
--- a/cylc/flow/main_loop/__init__.py
+++ b/cylc/flow/main_loop/__init__.py
@@ -329,14 +329,14 @@ def load(config, additional_plugins=None):
                 f'No main-loop plugin: "{plugin_name}"\n'
                 + '    Available plugins:\n'
                 + indent('\n'.join(sorted(entry_points)), '        ')
-            )
+            ) from None
         # load plugin
         try:
             module = entry_point.load()
         except Exception as exc:
             raise PluginError(
                 'cylc.main_loop', entry_point.name, exc
-            )
+            ) from None
         # load coroutines
         log = []
         for coro_name, coro in getmembers(module):
diff --git a/cylc/flow/main_loop/health_check.py b/cylc/flow/main_loop/health_check.py
index b488c878c0e..a470fd3cb88 100644
--- a/cylc/flow/main_loop/health_check.py
+++ b/cylc/flow/main_loop/health_check.py
@@ -56,4 +56,4 @@ def _check_contact_file(scheduler):
         raise CylcError(
             '%s: contact file corrupted/modified and may be left'
             % workflow_files.get_contact_file_path(scheduler.workflow)
-        )
+        ) from None
diff --git a/cylc/flow/network/__init__.py b/cylc/flow/network/__init__.py
index 916b129e244..315a57a23ba 100644
--- a/cylc/flow/network/__init__.py
+++ b/cylc/flow/network/__init__.py
@@ -78,7 +78,7 @@ def get_location(workflow: str) -> Tuple[str, int, int]:
         contact = load_contact_file(workflow)
     except (IOError, ValueError, ServiceFileError):
         # Contact file does not exist or corrupted, workflow should be dead
-        raise WorkflowStopped(workflow)
+        raise WorkflowStopped(workflow) from None
 
     host = contact[ContactFileFields.HOST]
     host = get_fqdn_by_host(host)
@@ -179,13 +179,13 @@ def _socket_bind(self, min_port, max_port, srv_prv_key_loc=None):
                 f"Failed to find server's public "
                 f"key in "
                 f"{srv_prv_key_info.full_key_path}."
-            )
+            ) from None
         except OSError:
             raise ServiceFileError(
                 f"IO error opening server's private "
                 f"key from "
                 f"{srv_prv_key_info.full_key_path}."
-            )
+            ) from None
         if server_private_key is None:  # this can't be caught by exception
             raise ServiceFileError(
                 f"Failed to find server's private "
@@ -204,7 +204,9 @@ def _socket_bind(self, min_port, max_port, srv_prv_key_loc=None):
                 self.port = self.socket.bind_to_random_port(
                     'tcp://*', min_port, max_port)
         except (zmq.error.ZMQError, zmq.error.ZMQBindError) as exc:
-            raise CylcError(f'could not start Cylc ZMQ server: {exc}')
+            raise CylcError(
+                f'could not start Cylc ZMQ server: {exc}'
+            ) from None
 
     # Keeping srv_public_key_loc as optional arg so as to not break interface
     def _socket_connect(self, host, port, srv_public_key_loc=None):
@@ -237,7 +239,7 @@ def _socket_connect(self, host, port, srv_public_key_loc=None):
             client_public_key, client_priv_key = zmq.auth.load_certificate(
                 client_priv_key_info.full_key_path)
         except (OSError, ValueError):
-            raise ClientError(error_msg)
+            raise ClientError(error_msg) from None
         if client_priv_key is None:  # this can't be caught by exception
             raise ClientError(error_msg)
         self.socket.curve_publickey = client_public_key
@@ -256,7 +258,8 @@ def _socket_connect(self, host, port, srv_public_key_loc=None):
             self.socket.curve_serverkey = server_public_key
         except (OSError, ValueError):  # ValueError raised w/ no public key
             raise ClientError(
-                "Failed to load the workflow's public key, so cannot connect.")
+                "Failed to load the workflow's public key, so cannot connect."
+            ) from None
 
         self.socket.connect(f'tcp://{host}:{port}')
 
diff --git a/cylc/flow/network/client.py b/cylc/flow/network/client.py
index e7e26954d56..099ef8bc0ff 100644
--- a/cylc/flow/network/client.py
+++ b/cylc/flow/network/client.py
@@ -326,7 +326,7 @@ async def async_request(
             raise ClientError(
                 error.get('message'),  # type: ignore
                 error.get('traceback'),  # type: ignore
-            )
+            ) from None
 
     def get_header(self) -> dict:
         """Return "header" data to attach to each request for traceability.
diff --git a/cylc/flow/network/resolvers.py b/cylc/flow/network/resolvers.py
index 11eafd8bea5..fc9b67eeef5 100644
--- a/cylc/flow/network/resolvers.py
+++ b/cylc/flow/network/resolvers.py
@@ -750,7 +750,7 @@ async def _mutation_mapper(
         try:
             meth = COMMANDS[command]
         except KeyError:
-            raise ValueError(f"Command '{command}' not found")
+            raise ValueError(f"Command '{command}' not found") from None
 
         try:
             # Initiate the command. Validation may be performed at this point,
diff --git a/cylc/flow/network/schema.py b/cylc/flow/network/schema.py
index 70e40232c1d..b962b582b70 100644
--- a/cylc/flow/network/schema.py
+++ b/cylc/flow/network/schema.py
@@ -271,7 +271,7 @@ def field_name_from_type(
     try:
         return NODE_MAP[named_type.name]
     except KeyError:
-        raise ValueError(f"'{named_type.name}' is not a node type")
+        raise ValueError(f"'{named_type.name}' is not a node type") from None
 
 
 def get_resolvers(info: 'ResolveInfo') -> 'BaseResolvers':
diff --git a/cylc/flow/network/ssh_client.py b/cylc/flow/network/ssh_client.py
index d1dd4fb6da4..d2ed0dd33e9 100644
--- a/cylc/flow/network/ssh_client.py
+++ b/cylc/flow/network/ssh_client.py
@@ -90,7 +90,7 @@ async def async_request(
                 f"Command exceeded the timeout {timeout}s. "
                 "This could be due to network problems. "
                 "Check the workflow log."
-            )
+            ) from None
 
     def prepare_command(
         self, command: str, args: Optional[dict], timeout: Union[float, str]
diff --git a/cylc/flow/param_expand.py b/cylc/flow/param_expand.py
index 0707a46e1a3..6ab104404e4 100644
--- a/cylc/flow/param_expand.py
+++ b/cylc/flow/param_expand.py
@@ -195,8 +195,9 @@ def _expand_name(self, results, tmpl, params, spec_vals=None):
             try:
                 results.append((tmpl % current_values, current_values))
             except KeyError as exc:
-                raise ParamExpandError('parameter %s is not '
-                                       'defined.' % str(exc.args[0]))
+                raise ParamExpandError(
+                    'parameter %s is not ' 'defined.' % str(exc.args[0])
+                ) from None
         else:
             for param_val in params[0][1]:
                 spec_vals[params[0][0]] = param_val
@@ -306,8 +307,8 @@ def expand_parent_params(self, parent, param_values, origin):
                     used[item] = param_values[item]
                 except KeyError:
                     raise ParamExpandError(
-                        "parameter '%s' undefined in '%s'" % (
-                            item, origin))
+                        "parameter '%s' undefined in '%s'" % (item, origin)
+                    ) from None
 
         # For each parameter substitute the param_tmpl_cfg.
         tmpl = tmpl.format(**self.param_tmpl_cfg)
@@ -425,8 +426,9 @@ def _expand_graph(self, line, all_params,
                 try:
                     repl = tmpl % param_values
                 except KeyError as exc:
-                    raise ParamExpandError('parameter %s is not '
-                                           'defined.' % str(exc.args[0]))
+                    raise ParamExpandError(
+                        'parameter %s is not ' 'defined.' % str(exc.args[0])
+                    ) from None
                 line = line.replace('<' + p_group + '>', repl)
             if line:
                 line_set.add(line)
diff --git a/cylc/flow/parsec/config.py b/cylc/flow/parsec/config.py
index 19f937d8e5b..29944c03b30 100644
--- a/cylc/flow/parsec/config.py
+++ b/cylc/flow/parsec/config.py
@@ -150,10 +150,12 @@ def get(self, keys: Optional[Iterable[str]] = None, sparse: bool = False):
                         # setting not present in __MANY__ section:
                         key in self.spec.get(*parents)
                     ):
-                        raise ItemNotFoundError(itemstr(parents, key))
+                        raise ItemNotFoundError(
+                            itemstr(parents, key)
+                        ) from None
                     raise InvalidConfigError(
                         itemstr(parents, key), self.spec.name
-                    )
+                    ) from None
                 else:
                     parents.append(key)
 
diff --git a/cylc/flow/parsec/empysupport.py b/cylc/flow/parsec/empysupport.py
index b4164894e0f..e3dc5e28df4 100644
--- a/cylc/flow/parsec/empysupport.py
+++ b/cylc/flow/parsec/empysupport.py
@@ -66,7 +66,7 @@ def empyprocess(
         raise EmPyError(
             str(exc),
             lines={'<template>': flines[max(lineno - 4, 0): lineno]},
-        )
+        ) from None
     finally:
         interpreter.shutdown()
         xworkflow = xtempl.getvalue()
diff --git a/cylc/flow/parsec/fileparse.py b/cylc/flow/parsec/fileparse.py
index 57b3449561a..f4de85aee3c 100644
--- a/cylc/flow/parsec/fileparse.py
+++ b/cylc/flow/parsec/fileparse.py
@@ -493,8 +493,10 @@ def read_and_proc(
             try:
                 from cylc.flow.parsec.empysupport import empyprocess
             except ImportError:
-                raise ParsecError('EmPy Python package must be installed '
-                                  'to process file: ' + fpath)
+                raise ParsecError(
+                    'EmPy Python package must be installed '
+                    'to process file: ' + fpath
+                ) from None
             flines = empyprocess(
                 fpath, flines, fdir, template_vars
             )
@@ -513,8 +515,10 @@ def read_and_proc(
             try:
                 from cylc.flow.parsec.jinja2support import jinja2process
             except ImportError:
-                raise ParsecError('Jinja2 Python package must be installed '
-                                  'to process file: ' + fpath)
+                raise ParsecError(
+                    'Jinja2 Python package must be installed '
+                    'to process file: ' + fpath
+                ) from None
             flines = jinja2process(
                 fpath, flines, fdir, template_vars
             )
diff --git a/cylc/flow/parsec/jinja2support.py b/cylc/flow/parsec/jinja2support.py
index 34f92c11519..74335358421 100644
--- a/cylc/flow/parsec/jinja2support.py
+++ b/cylc/flow/parsec/jinja2support.py
@@ -75,7 +75,7 @@ def load(
         try:
             mdict = __import__(name, fromlist=['*']).__dict__
         except ImportError:
-            raise TemplateNotFound(name)
+            raise TemplateNotFound(name) from None
 
         # inject module dict into the context of an empty template
         def root_render_func(context, *args, **kwargs):
@@ -297,12 +297,12 @@ def jinja2process(
             exc,
             lines=get_error_lines(fpath, flines),
             filename=filename
-        )
+        ) from None
     except Exception as exc:
         raise Jinja2Error(
             exc,
             lines=get_error_lines(fpath, flines),
-        )
+        ) from None
 
     # Ignore blank lines (lone Jinja2 statements leave blank lines behind)
     return [line for line in lines if line.strip()]
diff --git a/cylc/flow/parsec/upgrade.py b/cylc/flow/parsec/upgrade.py
index 0f75732f6d5..3ef53b006f6 100644
--- a/cylc/flow/parsec/upgrade.py
+++ b/cylc/flow/parsec/upgrade.py
@@ -110,7 +110,7 @@ def get_item(self, keys):
                 raise UpgradeError(
                     f'{self.show_keys(keys[:-1], True)}'
                     f' ("{keys[-2]}" should be a [section] not a setting)'
-                )
+                ) from None
         return item
 
     def put_item(self, keys, val):
diff --git a/cylc/flow/parsec/validate.py b/cylc/flow/parsec/validate.py
index 18c19596a63..12e2163a894 100644
--- a/cylc/flow/parsec/validate.py
+++ b/cylc/flow/parsec/validate.py
@@ -280,7 +280,7 @@ def coerce_float(cls, value, keys):
         try:
             return float(value)
         except ValueError as exc:
-            raise IllegalValueError('float', keys, value, exc=exc)
+            raise IllegalValueError('float', keys, value, exc=exc) from None
 
     @classmethod
     def coerce_float_list(cls, value, keys):
@@ -309,7 +309,7 @@ def coerce_int(cls, value, keys):
         try:
             return int(value)
         except ValueError as exc:
-            raise IllegalValueError('int', keys, value, exc=exc)
+            raise IllegalValueError('int', keys, value, exc=exc) from None
 
     @classmethod
     def coerce_int_list(cls, value, keys):
@@ -480,13 +480,17 @@ def expand_list(cls, values, keys, type_):
                 try:
                     lvalues.append(type_(item))
                 except ValueError as exc:
-                    raise IllegalValueError('list', keys, item, exc=exc)
+                    raise IllegalValueError(
+                        'list', keys, item, exc=exc
+                    ) from None
             else:
                 # mult * val
                 try:
                     lvalues += int(mult) * [type_(val)]
                 except ValueError as exc:
-                    raise IllegalValueError('list', keys, item, exc=exc)
+                    raise IllegalValueError(
+                        'list', keys, item, exc=exc
+                    ) from None
         return lvalues
 
     @classmethod
@@ -832,7 +836,9 @@ def coerce_cycle_point(cls, value, keys):
                     parser.parse(value)
                 return value
             except IsodatetimeError as exc:
-                raise IllegalValueError('cycle point', keys, value, exc=exc)
+                raise IllegalValueError(
+                    'cycle point', keys, value, exc=exc
+                ) from None
         try:
             TimePointParser().parse(value)
         except IsodatetimeError as exc:
@@ -841,7 +847,9 @@ def coerce_cycle_point(cls, value, keys):
                 details = {'msg': "Invalid cycle point"}
             else:
                 details = {'exc': exc}
-            raise IllegalValueError('cycle point', keys, value, **details)
+            raise IllegalValueError(
+                'cycle point', keys, value, **details
+            ) from None
         return value
 
     @classmethod
@@ -883,7 +891,7 @@ def coerce_cycle_point_format(cls, value, keys):
             except IsodatetimeError as exc:
                 raise IllegalValueError(
                     'cycle point format', keys, value, exc=exc
-                )
+                ) from None
             return value
         if 'X' in value:
             for i in range(1, 101):
@@ -898,7 +906,9 @@ def coerce_cycle_point_format(cls, value, keys):
         try:
             dumper.dump(test_timepoint, value)
         except IsodatetimeError as exc:
-            raise IllegalValueError('cycle point format', keys, value, exc=exc)
+            raise IllegalValueError(
+                'cycle point format', keys, value, exc=exc
+            ) from None
         return value
 
     @classmethod
@@ -932,7 +942,7 @@ def coerce_cycle_point_time_zone(cls, value, keys):
         except ValueError as exc:  # not IsodatetimeError as too specific
             raise IllegalValueError(
                 'cycle point time zone format', keys, value, exc=exc
-            )
+            ) from None
         return value
 
     @classmethod
@@ -951,7 +961,9 @@ def coerce_interval(cls, value, keys):
         try:
             interval = DurationParser().parse(value)
         except IsodatetimeError as exc:
-            raise IllegalValueError("ISO 8601 interval", keys, value, exc=exc)
+            raise IllegalValueError(
+                "ISO 8601 interval", keys, value, exc=exc
+            ) from None
         return DurationFloat(interval.get_seconds())
 
     @classmethod
@@ -1012,8 +1024,11 @@ def coerce_parameter_list(cls, value, keys):
                 except ValueError:
                     if can_only_be == int:
                         raise IllegalValueError(
-                            'parameter', keys, value,
-                            'mixing int range and str')
+                            'parameter',
+                            keys,
+                            value,
+                            'mixing int range and str',
+                        ) from None
                     can_only_be = str
                 items.append(item)
             else:
@@ -1248,7 +1263,8 @@ def coerce_interval(cls, value, keys):
                 interval = DurationParser().parse(str(DurationFloat(value)))
             except IsodatetimeError as exc:
                 raise IllegalValueError(
-                    "ISO 8601 interval", keys, value, exc=exc)
+                    "ISO 8601 interval", keys, value, exc=exc
+                ) from None
         return DurationFloat(interval.get_seconds())
 
 
diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py
index aa097a2e680..da3d6c93cde 100644
--- a/cylc/flow/pathutil.py
+++ b/cylc/flow/pathutil.py
@@ -272,7 +272,7 @@ def make_symlink_dir(path: Union[Path, str], target: Union[Path, str]) -> bool:
         except OSError as exc:
             raise WorkflowFilesError(
                 f"Failed to remove broken symlink {path}\n{exc}"
-            )
+            ) from None
     try:
         target.mkdir(parents=True, exist_ok=False)
     except FileExistsError:
@@ -280,7 +280,7 @@ def make_symlink_dir(path: Union[Path, str], target: Union[Path, str]) -> bool:
             f"Symlink dir target already exists: ({path} ->) {target}\n"
             "Tip: in future, use 'cylc clean' instead of manually deleting "
             "workflow run dirs."
-        )
+        ) from None
 
     # This is needed in case share and share/cycle have the same symlink dir:
     if path.exists():
@@ -291,7 +291,7 @@ def make_symlink_dir(path: Union[Path, str], target: Union[Path, str]) -> bool:
         path.symlink_to(target)
         return True
     except OSError as exc:
-        raise WorkflowFilesError(f"Error when symlinking\n{exc}")
+        raise WorkflowFilesError(f"Error when symlinking\n{exc}") from None
 
 
 def remove_dir_and_target(path: Union[Path, str]) -> None:
diff --git a/cylc/flow/prerequisite.py b/cylc/flow/prerequisite.py
index b388934e2de..449a5009656 100644
--- a/cylc/flow/prerequisite.py
+++ b/cylc/flow/prerequisite.py
@@ -195,7 +195,8 @@ def _conditional_is_satisfied(self):
                 err_msg += (
                     " (could be unmatched parentheses in the graph string?)")
             raise TriggerExpressionError(
-                '"%s":\n%s' % (self.get_raw_conditional_expression(), err_msg))
+                '"%s":\n%s' % (self.get_raw_conditional_expression(), err_msg)
+            ) from None
         return res
 
     def satisfy_me(self, outputs: Iterable['Tokens']) -> 'Set[Tokens]':
diff --git a/cylc/flow/scripts/broadcast.py b/cylc/flow/scripts/broadcast.py
index c7e5d2a4f3b..8cfda85bdd8 100755
--- a/cylc/flow/scripts/broadcast.py
+++ b/cylc/flow/scripts/broadcast.py
@@ -360,7 +360,7 @@ async def run(options: 'Values', workflow_id):
                 # TODO validate showtask?
                 raise InputError(
                     'TASK_ID_GLOB must be in the format: cycle/task'
-                )
+                ) from None
         result = await pclient.async_request('graphql', query_kwargs)
         for wflow in result['workflows']:
             settings = wflow['broadcasts']
diff --git a/cylc/flow/scripts/cat_log.py b/cylc/flow/scripts/cat_log.py
index 688f51ddd35..e53524cc648 100755
--- a/cylc/flow/scripts/cat_log.py
+++ b/cylc/flow/scripts/cat_log.py
@@ -495,7 +495,7 @@ def _main(
                     raise InputError(
                         f"--rotation {rotation_number} invalid "
                         f"(max value is {len(logs) - 1})"
-                    )
+                    ) from None
             else:
                 raise InputError('Log file not found.')
         else:
diff --git a/cylc/flow/scripts/clean.py b/cylc/flow/scripts/clean.py
index 2166a98f055..fffd27752d7 100644
--- a/cylc/flow/scripts/clean.py
+++ b/cylc/flow/scripts/clean.py
@@ -157,7 +157,7 @@ def parse_timeout(opts: 'Values') -> None:
                 raise InputError(
                     f"Invalid timeout: {opts.remote_timeout}. Must be "
                     "an ISO 8601 duration or number of seconds."
-                )
+                ) from None
         opts.remote_timeout = str(timeout)
 
 
diff --git a/cylc/flow/scripts/cylc.py b/cylc/flow/scripts/cylc.py
index 4ca411f8e9b..209512264a9 100644
--- a/cylc/flow/scripts/cylc.py
+++ b/cylc/flow/scripts/cylc.py
@@ -612,7 +612,7 @@ def pycoverage(cmd_args):  # pragma: no cover
             'Could not initiate coverage, likely because Cylc was not '
             'installed in editable mode.'
             '\n\n*****************************\n\n'
-        )
+        ) from None
 
     # start the coverage running
     cov.start()
diff --git a/cylc/flow/scripts/dump.py b/cylc/flow/scripts/dump.py
index f891c943d77..f47ee8d2df8 100755
--- a/cylc/flow/scripts/dump.py
+++ b/cylc/flow/scripts/dump.py
@@ -296,4 +296,5 @@ async def dump(workflow_id, options, write=print):
                         write(', '.join(values))
     except Exception as exc:
         raise CylcError(
-            json.dumps(workflows, indent=4) + '\n' + str(exc) + '\n')
+            json.dumps(workflows, indent=4) + '\n' + str(exc) + '\n'
+        ) from None
diff --git a/cylc/flow/scripts/ext_trigger.py b/cylc/flow/scripts/ext_trigger.py
index c76bbb86712..775d6eaa0da 100755
--- a/cylc/flow/scripts/ext_trigger.py
+++ b/cylc/flow/scripts/ext_trigger.py
@@ -145,7 +145,7 @@ def main(
             LOG.exception(exc)
             LOG.info(MSG_SEND_FAILED, i_try + 1, max_n_tries)
             if i_try == max_n_tries - 1:  # final attempt
-                raise CylcError('send failed')
+                raise CylcError('send failed') from None
             LOG.info(MSG_SEND_RETRY, retry_intvl_secs, options.comms_timeout)
             sleep(retry_intvl_secs)
         else:
diff --git a/cylc/flow/scripts/get_workflow_contact.py b/cylc/flow/scripts/get_workflow_contact.py
index 9903aec6abb..66194cce732 100755
--- a/cylc/flow/scripts/get_workflow_contact.py
+++ b/cylc/flow/scripts/get_workflow_contact.py
@@ -49,10 +49,10 @@ def main(parser: COP, options: 'Values', workflow_id: str) -> None:
     )
     try:
         data = load_contact_file(workflow_id)
-    except ServiceFileError:
+    except ServiceFileError as exc:
         raise CylcError(
             f"{workflow_id}: cannot get contact info, workflow not running?"
-        )
+        ) from exc
     else:
         for key, value in sorted(data.items()):
             print("%s=%s" % (key, value))
diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py
index d045ab70838..c452d3be8f5 100755
--- a/cylc/flow/scripts/lint.py
+++ b/cylc/flow/scripts/lint.py
@@ -805,7 +805,7 @@ def get_pyproject_toml(dir_: Path) -> Dict[str, Any]:
         try:
             loadeddata = toml_loads(tomlfile.read_text())
         except TOMLDecodeError as exc:
-            raise CylcError(f'pyproject.toml did not load: {exc}')
+            raise CylcError(f'pyproject.toml did not load: {exc}') from None
 
         _tool, _cylc, _lint = LINT_TABLE
         try:
diff --git a/cylc/flow/scripts/report_timings.py b/cylc/flow/scripts/report_timings.py
index b6d796ad732..362f40953c2 100755
--- a/cylc/flow/scripts/report_timings.py
+++ b/cylc/flow/scripts/report_timings.py
@@ -257,7 +257,7 @@ def _check_imports(self):
             raise CylcError(
                 'Cannot import pandas - summary unavailable.'
                 ' try: pip install cylc-flow[report-timings]'
-            )
+            ) from None
         else:
             del pandas
 
@@ -402,5 +402,5 @@ def _check_imports(self):
         except ImportError:
             raise CylcError(
                 'Cannot import matplotlib - HTML summary unavailable.'
-            )
+            ) from None
         super(HTMLTimingSummary, self)._check_imports()
diff --git a/cylc/flow/scripts/validate.py b/cylc/flow/scripts/validate.py
index a224302c1b0..f3a77087069 100755
--- a/cylc/flow/scripts/validate.py
+++ b/cylc/flow/scripts/validate.py
@@ -181,7 +181,8 @@ async def run(
             continue
         except Exception as exc:
             raise WorkflowConfigError(
-                'failed to instantiate task %s: %s' % (name, exc))
+                'failed to instantiate task %s: %s' % (name, exc)
+            ) from None
 
         # force trigger evaluation now
         try:
@@ -189,17 +190,21 @@ async def run(
         except TriggerExpressionError as exc:
             err = str(exc)
             if '@' in err:
-                print(f"ERROR, {name}: xtriggers can't be in conditional"
-                      f" expressions: {err}",
-                      file=sys.stderr)
+                print(
+                    f"ERROR, {name}: xtriggers can't be in conditional"
+                    f" expressions: {err}",
+                    file=sys.stderr,
+                )
             else:
-                print('ERROR, %s: bad trigger: %s' % (name, err),
-                      file=sys.stderr)
-            raise WorkflowConfigError("ERROR: bad trigger")
+                print(
+                    'ERROR, %s: bad trigger: %s' % (name, err), file=sys.stderr
+                )
+            raise WorkflowConfigError("ERROR: bad trigger") from None
         except Exception as exc:
             print(str(exc), file=sys.stderr)
             raise WorkflowConfigError(
-                '%s: failed to evaluate triggers.' % name)
+                '%s: failed to evaluate triggers.' % name
+            ) from None
         if cylc.flow.flags.verbosity > 0:
             print('  + %s ok' % itask.identity)
 
diff --git a/cylc/flow/scripts/workflow_state.py b/cylc/flow/scripts/workflow_state.py
index f6350110223..cea5ab906ab 100755
--- a/cylc/flow/scripts/workflow_state.py
+++ b/cylc/flow/scripts/workflow_state.py
@@ -179,7 +179,7 @@ def __init__(
         try:
             tokens = Tokens(self.id_)
         except ValueError as exc:
-            raise InputError(exc)
+            raise InputError(exc) from None
 
         self.workflow_id_raw = tokens.workflow_id
         self.selector = (
@@ -404,7 +404,8 @@ def main(parser: COP, options: 'Values', *ids: str) -> None:
                 options.depr_point = os.environ["CYLC_TASK_CYCLE_POINT"]
             except KeyError:
                 raise InputError(
-                    "--task-point: $CYLC_TASK_CYCLE_POINT is not defined")
+                    "--task-point: $CYLC_TASK_CYCLE_POINT is not defined"
+                ) from None
 
         if options.depr_point is not None:
             id_ += f"//{options.depr_point}"
diff --git a/cylc/flow/task_id.py b/cylc/flow/task_id.py
index dbe003222b6..50075f92b9b 100644
--- a/cylc/flow/task_id.py
+++ b/cylc/flow/task_id.py
@@ -92,7 +92,8 @@ def get_standardised_point_string(cls, point_string):
         except PointParsingError as exc:
             # (This is only needed to raise a clearer error message).
             raise ValueError(
-                "Invalid cycle point: %s (%s)" % (point_string, exc))
+                "Invalid cycle point: %s (%s)" % (point_string, exc)
+            ) from None
         return point_string
 
     @classmethod
diff --git a/cylc/flow/time_parser.py b/cylc/flow/time_parser.py
index 932a62677aa..c3d242d6883 100644
--- a/cylc/flow/time_parser.py
+++ b/cylc/flow/time_parser.py
@@ -255,7 +255,7 @@ def parse_recurrence(
             except CylcMissingContextPointError:
                 raise CylcMissingFinalCyclePointError(
                     "This workflow requires a final cycle point."
-                )
+                ) from None
 
             exclusion_points = []
             # Convert the exclusion strings to ISO8601 points
diff --git a/cylc/flow/tui/data.py b/cylc/flow/tui/data.py
index 90344e2462b..948d1906fe9 100644
--- a/cylc/flow/tui/data.py
+++ b/cylc/flow/tui/data.py
@@ -408,11 +408,11 @@ def online_mutate(mutation, selection):
     except WorkflowStopped:
         raise Exception(
             f'Cannot peform command {mutation} on a stopped workflow'
-        )
+        ) from None
     except (ClientError, ClientTimeout) as exc:
         raise Exception(
             f'Error connecting to workflow: {exc}'
-        )
+        ) from None
 
     request_string = generate_mutation(mutation, variables)
     client(
diff --git a/cylc/flow/util.py b/cylc/flow/util.py
index 74a11d0505a..5b7d5df06f8 100644
--- a/cylc/flow/util.py
+++ b/cylc/flow/util.py
@@ -281,7 +281,7 @@ def _eval(expr, **variables):
                 error_class,
                 f'{exc.msg}: {exc.text}',
                 {'expr': expr}
-            )
+            ) from None
 
         # check against whitelisted types
         try:
@@ -302,7 +302,7 @@ def _eval(expr, **variables):
                     'error_node': error_node,
                     'error_type': error_node.__class__.__name__,
                 },
-            )
+            ) from None
 
         # run the expresion
         # Note: this may raise runtime errors
diff --git a/cylc/flow/workflow_db_mgr.py b/cylc/flow/workflow_db_mgr.py
index 8e57d19292b..c3f3ea9a263 100644
--- a/cylc/flow/workflow_db_mgr.py
+++ b/cylc/flow/workflow_db_mgr.py
@@ -701,7 +701,9 @@ def _get_last_run_version(cls, pri_dao: CylcWorkflowDAO) -> Version:
                 [cls.KEY_CYLC_VERSION]
             ).fetchone()[0]
         except (TypeError, OperationalError):
-            raise ServiceFileError(f"{INCOMPAT_MSG}, or is corrupted.")
+            raise ServiceFileError(
+                f"{INCOMPAT_MSG}, or is corrupted."
+            ) from None
         return parse_version(last_run_ver)
 
     @classmethod
diff --git a/cylc/flow/workflow_files.py b/cylc/flow/workflow_files.py
index 023d481324c..ab390ec10aa 100644
--- a/cylc/flow/workflow_files.py
+++ b/cylc/flow/workflow_files.py
@@ -426,7 +426,7 @@ def _is_process_running(
         raise CylcError(
             f'Attempt to determine whether workflow is running on {host}'
             ' timed out after 10 seconds.'
-        )
+        ) from None
 
     if proc.returncode == 2:
         # the psutil call failed to gather metrics on the process
@@ -505,7 +505,7 @@ def detect_old_contact_file(
         # this can happen if contact file is from an outdated version of Cylc
         raise ServiceFileError(
             f'Found contact file with incomplete data:\n{exc}.'
-        )
+        ) from None
 
     # check if the workflow process is running ...
     # NOTE: can raise CylcError
@@ -631,7 +631,7 @@ def load_contact_file(id_: str, run_dir=None) -> Dict[str, str]:
         with open(path) as f:
             file_content = f.read()
     except IOError:
-        raise ServiceFileError("Couldn't load contact file")
+        raise ServiceFileError("Couldn't load contact file") from None
     data: Dict[str, str] = {}
     for line in file_content.splitlines():
         key, value = [item.strip() for item in line.split("=", 1)]
@@ -914,7 +914,7 @@ def infer_latest_run(
     try:
         id_ = str(path.relative_to(cylc_run_dir))
     except ValueError:
-        raise ValueError(f"{path} is not in the cylc-run directory")
+        raise ValueError(f"{path} is not in the cylc-run directory") from None
 
     if not path.exists():
         raise InputError(
diff --git a/cylc/flow/xtrigger_mgr.py b/cylc/flow/xtrigger_mgr.py
index 4f33ec7f3cc..60bd9b87504 100644
--- a/cylc/flow/xtrigger_mgr.py
+++ b/cylc/flow/xtrigger_mgr.py
@@ -250,12 +250,12 @@ def _validate(
         try:
             func = get_xtrig_func(fctx.mod_name, fctx.func_name, fdir)
         except (ImportError, AttributeError) as exc:
-            raise XtriggerConfigError(label, sig_str, exc)
+            raise XtriggerConfigError(label, sig_str, exc) from None
         try:
             sig = signature(func)
         except TypeError as exc:
             # not callable
-            raise XtriggerConfigError(label, sig_str, exc)
+            raise XtriggerConfigError(label, sig_str, exc) from None
 
         sig = cls._handle_sequential_kwarg(label, fctx, sig)
 
@@ -269,7 +269,7 @@ def _validate(
                     label, fctx, err
                 )
             else:
-                raise err
+                raise err from None
 
         # Specific xtrigger.validate(), if available.
         # Note arg string templating has not been done at this point.
@@ -293,7 +293,7 @@ def _validate(
                     raise XtriggerConfigError(
                         label, sig_str,
                         f"Illegal template in xtrigger: {match}",
-                    )
+                    ) from None
 
         # check for deprecated template variables
         deprecated_variables = template_vars & {
@@ -364,7 +364,7 @@ def _try_xtrig_validate_func(
         except Exception as exc:  # Note: catch all errors
             if not isinstance(exc, WorkflowConfigError):
                 LOG.exception(exc)
-            raise XtriggerConfigError(label, signature_str, exc)
+            raise XtriggerConfigError(label, signature_str, exc) from None
 
     # BACK COMPAT: workflow_state_backcompat
     # from: 8.0.0
@@ -390,7 +390,7 @@ def _try_workflow_state_backcompat(
             bound_args = sig.bind(*fctx.func_args, **fctx.func_kwargs)
         except TypeError:
             # failed signature check for backcompat function
-            raise err  # original signature check error
+            raise
 
         old_sig_str = fctx.get_signature()
         upg_sig_str = "workflow_state({})".format(
diff --git a/cylc/flow/xtriggers/wall_clock.py b/cylc/flow/xtriggers/wall_clock.py
index f000319d1ff..cea8d0b3026 100644
--- a/cylc/flow/xtriggers/wall_clock.py
+++ b/cylc/flow/xtriggers/wall_clock.py
@@ -75,4 +75,6 @@ def validate(args: Dict[str, Any]):
     try:
         interval_parse(args["offset"])
     except (ValueError, AttributeError):
-        raise WorkflowConfigError(f"Invalid offset: {args['offset']}")
+        raise WorkflowConfigError(
+            f"Invalid offset: {args['offset']}"
+        ) from None

From 2f43b4e1a5ea4483a960b7059509d8d06a4142f9 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Wed, 19 Jun 2024 17:17:45 +0100
Subject: [PATCH 04/53] lint: address PERF101

---
 cylc/flow/network/multi.py | 2 +-
 cylc/flow/scheduler.py     | 2 +-
 cylc/flow/task_pool.py     | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cylc/flow/network/multi.py b/cylc/flow/network/multi.py
index 2b9ea418976..9c190f68799 100644
--- a/cylc/flow/network/multi.py
+++ b/cylc/flow/network/multi.py
@@ -235,7 +235,7 @@ def _report(
     """
     try:
         ret: List[Tuple[Optional[str], Optional[str], bool]] = []
-        for _mutation_name, mutation_response in response.items():
+        for mutation_response in response.values():
             # extract the result of each mutation result in the response
             success, msg = mutation_response['result'][0]['response']
             out = None
diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py
index ff593648e7b..48fe6df62d1 100644
--- a/cylc/flow/scheduler.py
+++ b/cylc/flow/scheduler.py
@@ -917,7 +917,7 @@ def process_queued_task_messages(self) -> None:
         # Remaining unprocessed messages have no corresponding task proxy.
         # For example, if I manually set a running task to succeeded, the
         # proxy can be removed, but the orphaned job still sends messages.
-        for _id, tms in messages.items():
+        for tms in messages.values():
             warn = "Undeliverable task messages received and ignored:"
             for _, msg in tms:
                 warn += f'\n  {msg.job_id}: {msg.severity} - "{msg.message}"'
diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py
index 21893103f86..53f6f354eb5 100644
--- a/cylc/flow/task_pool.py
+++ b/cylc/flow/task_pool.py
@@ -870,8 +870,8 @@ def get_tasks(self) -> List[TaskProxy]:
         if self.active_tasks_changed:
             self.active_tasks_changed = False
             self._active_tasks_list = []
-            for _, itask_id_map in self.active_tasks.items():
-                for __, itask in itask_id_map.items():
+            for itask_id_map in self.active_tasks.values():
+                for itask in itask_id_map.values():
                     self._active_tasks_list.append(itask)
         return self._active_tasks_list
 

From b188a44045faeb64508e8c1398c91692f57d2b95 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Wed, 19 Jun 2024 17:23:35 +0100
Subject: [PATCH 05/53] lint: address PERF402

---
 cylc/flow/resources.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/cylc/flow/resources.py b/cylc/flow/resources.py
index d81b7f7211b..956ec27e8aa 100644
--- a/cylc/flow/resources.py
+++ b/cylc/flow/resources.py
@@ -18,7 +18,7 @@
 
 from contextlib import suppress
 from pathlib import Path
-from random import shuffle
+from random import choice
 import shutil
 import sys
 from typing import Optional
@@ -205,12 +205,9 @@ def get_api_key() -> str:
     load over a larger number of keys to prevent hitting the cap with group
     sessions.
     """
-    keys = []
     with open((TUTORIAL_DIR / 'api-keys'), 'r') as api_keys:
-        for api_key in api_keys:
-            keys.append(api_key)
-    shuffle(keys)
-    return keys[0].strip()
+        return choice(list(api_keys)).strip()  # nosec
+        # (the randomness of this choice is not a security concern)
 
 
 def set_api_key(tgt):

From e28e0c0a753f4d9fa11fa18fb60cff318eea9799 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Wed, 10 Jul 2024 09:56:16 +0100
Subject: [PATCH 06/53] Lint.job runner specific timeout (#6137)

* Add S014: Use Execution Time Limit in preference to directives.

* response to review

* remove wallclock limit from the directives example for SGE

* Apply suggestions from code review
---
 changes.d/6137.feat.md                       |  1 +
 cylc/flow/job_runner_handlers/loadleveler.py |  5 +-
 cylc/flow/job_runner_handlers/lsf.py         |  8 ++-
 cylc/flow/job_runner_handlers/moab.py        |  6 ++-
 cylc/flow/job_runner_handlers/pbs.py         | 10 ++--
 cylc/flow/job_runner_handlers/sge.py         |  6 +--
 cylc/flow/job_runner_handlers/slurm.py       |  6 ++-
 cylc/flow/scripts/lint.py                    | 55 ++++++++++++++++++++
 tests/unit/scripts/test_lint.py              |  2 +
 9 files changed, 85 insertions(+), 14 deletions(-)
 create mode 100644 changes.d/6137.feat.md

diff --git a/changes.d/6137.feat.md b/changes.d/6137.feat.md
new file mode 100644
index 00000000000..d947999de99
--- /dev/null
+++ b/changes.d/6137.feat.md
@@ -0,0 +1 @@
+New Cylc lint rule: S014: Don't use job runner specific execution time limit directives, use execution time limit.
\ No newline at end of file
diff --git a/cylc/flow/job_runner_handlers/loadleveler.py b/cylc/flow/job_runner_handlers/loadleveler.py
index ee8203b2b47..d097ff85faf 100644
--- a/cylc/flow/job_runner_handlers/loadleveler.py
+++ b/cylc/flow/job_runner_handlers/loadleveler.py
@@ -83,6 +83,7 @@ class LoadlevelerHandler():
         re.compile("^llsubmit: Processed command file through Submit Filter:")]
     SUBMIT_CMD_TMPL = "llsubmit '%(job)s'"
     VACATION_SIGNAL = "USR1"
+    TIME_LIMIT_DIRECTIVE = "wall_clock_limit"
 
     def format_directives(self, job_conf):
         """Format the job directives for a job file."""
@@ -96,8 +97,8 @@ def format_directives(self, job_conf):
         directives["output"] = job_file_path + ".out"
         directives["error"] = job_file_path + ".err"
         if (job_conf["execution_time_limit"] and
-                directives.get("wall_clock_limit") is None):
-            directives["wall_clock_limit"] = "%d,%d" % (
+                directives.get(self.TIME_LIMIT_DIRECTIVE) is None):
+            directives[self.TIME_LIMIT_DIRECTIVE] = "%d,%d" % (
                 job_conf["execution_time_limit"] + 60,
                 job_conf["execution_time_limit"])
         for key, value in list(job_conf["directives"].items()):
diff --git a/cylc/flow/job_runner_handlers/lsf.py b/cylc/flow/job_runner_handlers/lsf.py
index a465c9b7924..534d1205a9d 100644
--- a/cylc/flow/job_runner_handlers/lsf.py
+++ b/cylc/flow/job_runner_handlers/lsf.py
@@ -70,6 +70,7 @@ class LSFHandler():
     POLL_CMD = "bjobs"
     REC_ID_FROM_SUBMIT_OUT = re.compile(r"^Job <(?P<id>\d+)>")
     SUBMIT_CMD_TMPL = "bsub"
+    TIME_LIMIT_DIRECTIVE = "-W"
 
     @classmethod
     def format_directives(cls, job_conf):
@@ -82,8 +83,11 @@ def format_directives(cls, job_conf):
         )
         directives["-o"] = job_file_path + ".out"
         directives["-e"] = job_file_path + ".err"
-        if job_conf["execution_time_limit"] and directives.get("-W") is None:
-            directives["-W"] = str(math.ceil(
+        if (
+            job_conf["execution_time_limit"]
+            and directives.get(cls.TIME_LIMIT_DIRECTIVE) is None
+        ):
+            directives[cls.TIME_LIMIT_DIRECTIVE] = str(math.ceil(
                 job_conf["execution_time_limit"] / 60))
         for key, value in list(job_conf["directives"].items()):
             directives[key] = value
diff --git a/cylc/flow/job_runner_handlers/moab.py b/cylc/flow/job_runner_handlers/moab.py
index 839d246ccbc..ec068d48420 100644
--- a/cylc/flow/job_runner_handlers/moab.py
+++ b/cylc/flow/job_runner_handlers/moab.py
@@ -78,6 +78,7 @@ class MoabHandler:
     POLL_CMD = "checkjob"
     REC_ID_FROM_SUBMIT_OUT = re.compile(r"""\A\s*(?P<id>\S+)\s*\Z""")
     SUBMIT_CMD_TMPL = "msub '%(job)s'"
+    TIME_LIMIT_DIRECTIVE = "-l walltime"
 
     def format_directives(self, job_conf):
         """Format the job directives for a job file."""
@@ -91,8 +92,9 @@ def format_directives(self, job_conf):
         directives["-o"] = job_file_path + ".out"
         directives["-e"] = job_file_path + ".err"
         if (job_conf["execution_time_limit"] and
-                directives.get("-l walltime") is None):
-            directives["-l walltime"] = "%d" % job_conf["execution_time_limit"]
+                directives.get(self.TIME_LIMIT_DIRECTIVE) is None):
+            directives[self.TIME_LIMIT_DIRECTIVE] = "%d" % job_conf[
+                "execution_time_limit"]
         # restartable?
         directives.update(job_conf["directives"])
         lines = []
diff --git a/cylc/flow/job_runner_handlers/pbs.py b/cylc/flow/job_runner_handlers/pbs.py
index aa264311fc4..ac0d6c47a00 100644
--- a/cylc/flow/job_runner_handlers/pbs.py
+++ b/cylc/flow/job_runner_handlers/pbs.py
@@ -84,6 +84,7 @@ class PBSHandler:
     POLL_CANT_CONNECT_ERR = "Connection refused"
     REC_ID_FROM_SUBMIT_OUT = re.compile(r"^\s*(?P<id>\d+)", re.M)
     SUBMIT_CMD_TMPL = "qsub '%(job)s'"
+    TIME_LIMIT_DIRECTIVE = "-l walltime"
 
     def format_directives(self, job_conf):
         """Format the job directives for a job file."""
@@ -105,9 +106,12 @@ def format_directives(self, job_conf):
 
         directives["-o"] = job_file_path + ".out"
         directives["-e"] = job_file_path + ".err"
-        if (job_conf["execution_time_limit"] and
-                directives.get("-l walltime") is None):
-            directives["-l walltime"] = "%d" % job_conf["execution_time_limit"]
+        if (
+            job_conf["execution_time_limit"]
+            and directives.get(self.TIME_LIMIT_DIRECTIVE) is None
+        ):
+            directives[self.TIME_LIMIT_DIRECTIVE] = "%d" % job_conf[
+                "execution_time_limit"]
         for key, value in list(job_conf["directives"].items()):
             directives[key] = value
         lines = []
diff --git a/cylc/flow/job_runner_handlers/sge.py b/cylc/flow/job_runner_handlers/sge.py
index 33f7d5a26d7..c7c50956fb9 100644
--- a/cylc/flow/job_runner_handlers/sge.py
+++ b/cylc/flow/job_runner_handlers/sge.py
@@ -37,7 +37,6 @@
                -cwd =
                -q = foo
                -l h_data = 1024M
-               -l h_rt = 24:00:00
 
 These are written to the top of the job script like this:
 
@@ -76,6 +75,7 @@ class SGEHandler:
     POLL_CMD = "qstat"
     REC_ID_FROM_SUBMIT_OUT = re.compile(r"\D+(?P<id>\d+)\D+")
     SUBMIT_CMD_TMPL = "qsub '%(job)s'"
+    TIME_LIMIT_DIRECTIVE = "-l h_rt"
 
     def format_directives(self, job_conf):
         """Format the job directives for a job file."""
@@ -88,8 +88,8 @@ def format_directives(self, job_conf):
         directives['-o'] = job_file_path + ".out"
         directives['-e'] = job_file_path + ".err"
         if (job_conf["execution_time_limit"] and
-                directives.get("-l h_rt") is None):
-            directives["-l h_rt"] = "%d:%02d:%02d" % (
+                directives.get(self.TIME_LIMIT_DIRECTIVE) is None):
+            directives[self.TIME_LIMIT_DIRECTIVE] = "%d:%02d:%02d" % (
                 job_conf["execution_time_limit"] / 3600,
                 (job_conf["execution_time_limit"] / 60) % 60,
                 job_conf["execution_time_limit"] % 60)
diff --git a/cylc/flow/job_runner_handlers/slurm.py b/cylc/flow/job_runner_handlers/slurm.py
index 4ec6be20471..33df2ebc926 100644
--- a/cylc/flow/job_runner_handlers/slurm.py
+++ b/cylc/flow/job_runner_handlers/slurm.py
@@ -135,6 +135,8 @@ class SLURMHandler():
     #  Separator between het job directive sections
     SEP_HETJOB = "#SBATCH hetjob"
 
+    TIME_LIMIT_DIRECTIVE = "--time"
+
     @classmethod
     def filter_poll_many_output(cls, out):
         """Return list of job IDs extracted from job poll stdout.
@@ -161,8 +163,8 @@ def format_directives(cls, job_conf):
         directives['--output'] = job_file_path.replace('%', '%%') + ".out"
         directives['--error'] = job_file_path.replace('%', '%%') + ".err"
         if (job_conf["execution_time_limit"] and
-                directives.get("--time") is None):
-            directives["--time"] = "%d:%02d" % (
+                directives.get(cls.TIME_LIMIT_DIRECTIVE) is None):
+            directives[cls.TIME_LIMIT_DIRECTIVE] = "%d:%02d" % (
                 job_conf["execution_time_limit"] / 60,
                 job_conf["execution_time_limit"] % 60)
         for key, value in list(job_conf['directives'].items()):
diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py
index d045ab70838..1561ad743f3 100755
--- a/cylc/flow/scripts/lint.py
+++ b/cylc/flow/scripts/lint.py
@@ -50,6 +50,7 @@
    max-line-length = 130        # Max line length for linting
 """
 import functools
+import pkgutil
 from pathlib import Path
 import re
 import sys
@@ -78,6 +79,8 @@
 from cylc.flow import LOG
 from cylc.flow.exceptions import CylcError
 import cylc.flow.flags
+from cylc.flow import job_runner_handlers
+from cylc.flow.job_runner_mgr import JobRunnerManager
 from cylc.flow.loggingutil import set_timestamps
 from cylc.flow.option_parsers import (
     CylcOptionParser as COP,
@@ -162,6 +165,34 @@
 }
 
 
+def get_wallclock_directives():
+    """Get a set of directives equivalent to execution time limit"""
+    job_runner_manager = JobRunnerManager()
+    directives = {}
+    for module in pkgutil.iter_modules(job_runner_handlers.__path__):
+        directive = getattr(
+            job_runner_manager._get_sys(module.name),
+            'TIME_LIMIT_DIRECTIVE',
+            None
+        )
+        if directive:
+            directives[module.name] = directive
+    return directives
+
+
+WALLCLOCK_DIRECTIVES = get_wallclock_directives()
+
+
+def check_wallclock_directives(line: str) -> Union[Dict[str, str], bool]:
+    """Check for job runner specific directives
+    equivelent to exection time limit.
+    """
+    for directive in set(WALLCLOCK_DIRECTIVES.values()):
+        if line.strip().startswith(directive):
+            return {'directive': line.strip()}
+    return False
+
+
 def check_jinja2_no_shebang(
     line: str,
     file: Path,
@@ -533,6 +564,30 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]:
     'S013': {
         'short': 'Items should be indented in 4 space blocks.',
         FUNCTION: check_indentation
+    },
+    'S014': {
+        'short': (
+            'Use ``[runtime][TASK]execution time limit``'
+            ' rather than job runner directive: ``{directive}``.'
+        ),
+        'rst': (
+            'Using ``[runtime][TASK]execution time limit`` is'
+            ' recommended in preference to using job runner'
+            ' directives because it allows Cylc to retain awareness'
+            ' of whether the job should have finished, even if contact'
+            ' with the target job runner\'s platform has been lost.'
+            ' \n\nThe following directives are considered equivelent to'
+            ' execution time limit:\n * '
+        )
+        + '\n * '.join((
+            f'``{directive}`` ({job_runner})'
+            for job_runner, directive in WALLCLOCK_DIRECTIVES.items()
+        )) + (
+            '\n\n.. note:: Using ``execution time limit`` which'
+            ' is automatically translated to the job runner\'s timeout'
+            ' directive can make your workflow more portable.'
+        ),
+        FUNCTION: check_wallclock_directives,
     }
 }
 # Subset of deprecations which are tricky (impossible?) to scrape from the
diff --git a/tests/unit/scripts/test_lint.py b/tests/unit/scripts/test_lint.py
index 0f5f02648bd..527a33bdfb5 100644
--- a/tests/unit/scripts/test_lint.py
+++ b/tests/unit/scripts/test_lint.py
@@ -178,6 +178,8 @@
 something\t
     [[bar]]
         platform = $(some-script foo)
+            [[[directives]]]
+                -l walltime = 666
     [[baz]]
         platform = `no backticks`
 """ + (

From 5605265c97177fc19f975a4e8e9122d58c609972 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Wed, 10 Jul 2024 17:32:30 +0100
Subject: [PATCH 07/53] Correct changelog [skip ci]

---
 CHANGES.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 061d935a010..6ff82d9c920 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -15,8 +15,6 @@ $ towncrier create <PR-number>.<break|feat|fix>.md --content "Short description"
 
 ### 🔧 Fixes
 
-[#6178](https://github.com/cylc/cylc-flow/pull/6178) - Fix an issue where Tui could hang when closing.
-
 [#6186](https://github.com/cylc/cylc-flow/pull/6186) - Fixed bug where using flow numbers with `cylc set` would not work correctly.
 
 [#6200](https://github.com/cylc/cylc-flow/pull/6200) - Fixed bug where a stalled paused workflow would be incorrectly reported as running, not paused
@@ -33,6 +31,8 @@ $ towncrier create <PR-number>.<break|feat|fix>.md --content "Short description"
 
 [#6176](https://github.com/cylc/cylc-flow/pull/6176) - Fix bug where jobs which fail to submit are not shown in GUI/TUI if submission retries are set.
 
+[#6178](https://github.com/cylc/cylc-flow/pull/6178) - Fix an issue where Tui could hang when closing.
+
 ## __cylc-8.3.0 (Released 2024-06-18)__
 
 ### ⚠ Breaking Changes

From 497a74e769293f20ebd40619aa7818d1df14d6f3 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 30 Jan 2024 12:02:39 +0000
Subject: [PATCH 08/53] example: extending workflow

---
 .../etc/examples/extending-workflow/.validate | 80 +++++++++++++++++++
 .../etc/examples/extending-workflow/index.rst | 77 ++++++++++++++++++
 .../extending-workflow/simple/flow.cylc       | 24 ++++++
 3 files changed, 181 insertions(+)
 create mode 100755 cylc/flow/etc/examples/extending-workflow/.validate
 create mode 100644 cylc/flow/etc/examples/extending-workflow/index.rst
 create mode 100644 cylc/flow/etc/examples/extending-workflow/simple/flow.cylc

diff --git a/cylc/flow/etc/examples/extending-workflow/.validate b/cylc/flow/etc/examples/extending-workflow/.validate
new file mode 100755
index 00000000000..43c810372ce
--- /dev/null
+++ b/cylc/flow/etc/examples/extending-workflow/.validate
@@ -0,0 +1,80 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -eux
+
+test_simple () {
+    local ID
+    ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)"
+    
+    # lint
+    cylc lint ./simple
+    
+    # copy into a temp directory
+    local SRC_DIR
+    SRC_DIR="$(mktemp -d)"
+    cp simple/flow.cylc "$SRC_DIR"
+    
+    # speed things up with simulation mode
+    cat >>"${SRC_DIR}/flow.cylc" <<__HERE__
+    [runtime]
+        [[root]]
+            [[[simulation]]]
+                default run length = PT0S
+__HERE__
+    
+    # start the workflow
+    cylc vip \
+        --check-circular \
+        --no-run-name \
+        --no-detach \
+        --workflow-name "$ID" \
+        --mode=simulation \
+        "$SRC_DIR"
+    
+    # it should have reached the 2002 cycle
+    grep '2002/a' "${HOME}/cylc-run/${ID}/log/scheduler/log"
+    if grep '2003/a' "${HOME}/cylc-run/${ID}/log/scheduler/log"; then
+        exit 1
+    fi
+    
+    # edit the "stop after cycle point"
+    sed -i \
+        's/stop after cycle point.*/stop after cycle point = 2004/' \
+        "${SRC_DIR}/flow.cylc"
+    
+    # continue the run
+    cylc vr \
+        --no-detach \
+        --mode=simulation \
+        --yes \
+        "$ID"
+    
+    # it should have reached the 2004 cycle
+    grep '2004/a' "${HOME}/cylc-run/${ID}/log/scheduler/log"
+    if grep '2005/a' "${HOME}/cylc-run/${ID}/log/scheduler/log"; then
+        exit 1
+    fi
+    
+    # clean up
+    cylc clean "$ID"
+    
+    rm -r "${SRC_DIR}"
+}
+
+
+test_simple
diff --git a/cylc/flow/etc/examples/extending-workflow/index.rst b/cylc/flow/etc/examples/extending-workflow/index.rst
new file mode 100644
index 00000000000..c5bc023dc11
--- /dev/null
+++ b/cylc/flow/etc/examples/extending-workflow/index.rst
@@ -0,0 +1,77 @@
+Extending Workflow
+==================
+
+.. cylc-scope:: flow.cylc[scheduling]
+
+Sometimes we may run a workflow to :term:`completion <workflow completion>`,
+but subsequently wish to run it for a few more cycles.
+
+With Cylc 7 this was often done by changing the `final cycle point` and
+restarting the workflow. This approach worked, but was a little awkward.
+It's possible with Cylc 8, but we would recommend moving away from this
+pattern instead.
+
+The recommended approach to this problem (Cylc 6+) is to use the
+`stop after cycle point` rather than the `final cycle point`.
+
+The `stop after cycle point` tells Cylc to **stop** after the workflow passes
+the specified point, whereas the `final cycle point` tells Cylc that the
+workflow **finishes** at the specified point.
+
+When a workflow **finishes**, it is a little awkward to restart as you have to
+tell Cylc which tasks to continue on from. The `stop after cycle point`
+solution avoids this issue.
+
+
+Example
+-------
+
+.. admonition:: Get a copy of this example
+   :class: hint
+
+   .. code-block:: console
+
+      $ cylc get-resources examples/extending-workflow/simple
+
+This workflow will stop at the end of the ``2002`` cycle:
+
+.. literalinclude:: simple/flow.cylc
+   :language: cylc
+
+After it has run and shut down, change the `stop after cycle point` to
+the desired value and restart it. E.g:
+
+.. code-block:: bash
+
+   # install and run the workflow:
+   cylc vip
+
+   # then later edit "stop after cycle point" to "2004"
+
+   # then reinstall and restart the workflow:
+   cylc vr
+
+The workflow will continue from where it left off and run until the end of the
+``2004`` cycle. Because the workflow never hit the `final cycle point` it
+never "finished" so no special steps are required to restart the workflow.
+
+You can also set the `stop after cycle point` when you start the workflow:
+
+.. code-block:: bash
+
+   cylc play --stop-cycle-point=2020 myworkflow
+
+Or change it at any point whilst the workflow is running:
+
+.. code-block:: bash
+
+   cylc stop myworkflow//2030  # change the stop after cycle point to 2030
+
+.. note::
+
+   If you set the `stop after cycle point` on the command line, this value will
+   take precedence over the one in the workflow configuration. Use
+   ``cylc play --stop-cycle-point=reload`` to restart the workflow using the
+   `stop after cycle point` configured in the workflow configuration.
+
+.. cylc-scope::
diff --git a/cylc/flow/etc/examples/extending-workflow/simple/flow.cylc b/cylc/flow/etc/examples/extending-workflow/simple/flow.cylc
new file mode 100644
index 00000000000..2de2f28f2ea
--- /dev/null
+++ b/cylc/flow/etc/examples/extending-workflow/simple/flow.cylc
@@ -0,0 +1,24 @@
+[meta]
+    title = "Basic extendable workflow"
+    description = """
+        Use the "stop after cycle point" rather than the "final cycle point"
+        to allow this workflow to be easily extended at a later date.
+    """
+
+[scheduler]
+    # use the year for the cycle point
+    # (strip off the month, day, hour and minute)
+    cycle point format = CCYY
+
+[scheduling]
+    initial cycle point = 2000
+    stop after cycle point = 2002  # stop after two years of simulated time
+    [[graph]]
+        P1Y = """
+            z[-P1Y] => a
+            a => z
+        """
+
+[runtime]
+    [[a]]
+    [[z]]

From f625c524b46e42058d466654f4efcc888f669ec4 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 16 Jul 2024 13:19:52 +0100
Subject: [PATCH 09/53] doc: add link to task comms section

---
 cylc/flow/cfgspec/globalcfg.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
index db8c7bdd5ff..d6182f8f7ea 100644
--- a/cylc/flow/cfgspec/globalcfg.py
+++ b/cylc/flow/cfgspec/globalcfg.py
@@ -1311,7 +1311,7 @@ def default_for(
                 The means by which task progress messages are reported back to
                 the running workflow.
 
-                Options:
+                ..rubric:: Options:
 
                 zmq
                    Direct client-server TCP communication via network ports
@@ -1320,6 +1320,8 @@ def default_for(
                 ssh
                    Use non-interactive ssh for task communications
 
+                For more information, see :ref:`TaskComms`.
+
                 .. versionchanged:: 8.0.0
 
                    {REPLACES}``global.rc[hosts][<host>]task communication

From 3ff5b4e5731276dbffccb798b48cf12d41b5b270 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Tue, 25 Jun 2024 10:34:45 +0100
Subject: [PATCH 10/53] Allow the Symlinking for log/job

response to review: ensure cylc clean works.
---
 changes.d/feat.6168.md                      |  1 +
 cylc/flow/cfgspec/globalcfg.py              | 65 +++++----------------
 cylc/flow/pathutil.py                       | 10 +++-
 cylc/flow/workflow_files.py                 |  7 ++-
 tests/functional/cylc-clean/00-basic.t      |  6 +-
 tests/functional/cylc-clean/01-remote.t     |  1 +
 tests/functional/cylc-install/01-symlinks.t | 11 +++-
 tests/functional/remote/04-symlink-dirs.t   | 20 ++++++-
 tests/unit/test_pathutil.py                 |  2 +
 9 files changed, 68 insertions(+), 55 deletions(-)
 create mode 100644 changes.d/feat.6168.md

diff --git a/changes.d/feat.6168.md b/changes.d/feat.6168.md
new file mode 100644
index 00000000000..84ccc73b236
--- /dev/null
+++ b/changes.d/feat.6168.md
@@ -0,0 +1 @@
+Allow symlinking log/job separately from log
diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
index d6182f8f7ea..94b0e347962 100644
--- a/cylc/flow/cfgspec/globalcfg.py
+++ b/cylc/flow/cfgspec/globalcfg.py
@@ -30,6 +30,7 @@
 from cylc.flow.exceptions import GlobalConfigError
 from cylc.flow.hostuserutil import get_user_home
 from cylc.flow.network.client_factory import CommsMeth
+from cylc.flow.pathutil import SYMLINKABLE_LOCATIONS
 from cylc.flow.parsec.config import (
     ConfigNode as Conf,
     ParsecConfig,
@@ -1148,55 +1149,21 @@ def default_for(
 
                     .. versionadded:: 8.0.0
                 """)
-                Conf('log', VDR.V_STRING, None, desc="""
-                    Alternative location for the log dir.
-
-                    If specified the workflow log directory will be created in
-                    ``<this-path>/cylc-run/<workflow-id>/log`` and a
-                    symbolic link will be created from
-                    ``$HOME/cylc-run/<workflow-id>/log``. If not specified
-                    the workflow log directory will be created in
-                    ``$HOME/cylc-run/<workflow-id>/log``.
-
-                    .. versionadded:: 8.0.0
-                """)
-                Conf('share', VDR.V_STRING, None, desc="""
-                    Alternative location for the share dir.
-
-                    If specified the workflow share directory will be
-                    created in ``<this-path>/cylc-run/<workflow-id>/share``
-                    and a symbolic link will be created from
-                    ``<$HOME/cylc-run/<workflow-id>/share``. If not specified
-                    the workflow share directory will be created in
-                    ``$HOME/cylc-run/<workflow-id>/share``.
-
-                    .. versionadded:: 8.0.0
-                """)
-                Conf('share/cycle', VDR.V_STRING, None, desc="""
-                    Alternative directory for the share/cycle dir.
-
-                    If specified the workflow share/cycle directory
-                    will be created in
-                    ``<this-path>/cylc-run/<workflow-id>/share/cycle``
-                    and a symbolic link will be created from
-                    ``$HOME/cylc-run/<workflow-id>/share/cycle``. If not
-                    specified the workflow share/cycle directory will be
-                    created in ``$HOME/cylc-run/<workflow-id>/share/cycle``.
-
-                    .. versionadded:: 8.0.0
-                """)
-                Conf('work', VDR.V_STRING, None, desc="""
-                    Alternative directory for the work dir.
-
-                    If specified the workflow work directory will be created in
-                    ``<this-path>/cylc-run/<workflow-id>/work`` and a
-                    symbolic link will be created from
-                    ``$HOME/cylc-run/<workflow-id>/work``. If not specified
-                    the workflow work directory will be created in
-                    ``$HOME/cylc-run/<workflow-id>/work``.
-
-                    .. versionadded:: 8.0.0
-                """)
+                for folder, versionadded in SYMLINKABLE_LOCATIONS.items():
+                    Conf(folder, VDR.V_STRING, None, desc=f"""
+                        Alternative location for the log dir.
+
+                        If specified the workflow {folder} directory will
+                        be created in
+                        ``<this-path>/cylc-run/<workflow-id>/{folder}``
+                        and a symbolic link will be created from
+                        ``$HOME/cylc-run/<workflow-id>/{folder}``.
+                        If not specified the workflow log directory will
+                        be created in
+                        ``$HOME/cylc-run/<workflow-id>/{folder}``.
+
+                        .. versionadded:: {versionadded}
+                    """)
     with Conf('platforms', desc='''
         Platforms allow you to define compute resources available at your
         site.
diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py
index da3d6c93cde..a3c7f9d2b73 100644
--- a/cylc/flow/pathutil.py
+++ b/cylc/flow/pathutil.py
@@ -45,6 +45,14 @@
 """Matches relative paths that are explicit (starts with ./)"""
 
 SHELL_ENV_VARS = re.compile(r'\$[^$/]*')
+SYMLINKABLE_LOCATIONS: Dict[str, str] = {
+    # Location: Version Added
+    'log': '8.0.0',
+    'log/job': '8.4.0',
+    'share': '8.0.0',
+    'share/cycle': '8.0.0',
+    'work': '8.0.0'
+}
 
 
 def expand_path(*args: Union[Path, str]) -> str:
@@ -234,7 +242,7 @@ def get_dirs_to_symlink(
     if base_dir:
         dirs_to_symlink['run'] = os.path.join(
             base_dir, 'cylc-run', workflow_id)
-    for dir_ in ['log', 'share', 'share/cycle', 'work']:
+    for dir_ in SYMLINKABLE_LOCATIONS.keys():
         link = symlink_conf[install_target].get(dir_, None)
         if (not link) or link == base_dir:
             continue
diff --git a/cylc/flow/workflow_files.py b/cylc/flow/workflow_files.py
index ab390ec10aa..0aac6bc174a 100644
--- a/cylc/flow/workflow_files.py
+++ b/cylc/flow/workflow_files.py
@@ -203,6 +203,11 @@ class LogDir:
         DB = 'db'
         """The public database"""
 
+        JOB = 'job'
+        """The job log directory."""
+
+    LOG_JOB_DIR = os.path.join(LogDir.DIRNAME, LogDir.JOB)
+
     SHARE_DIR = 'share'
     """Workflow share directory."""
 
@@ -259,7 +264,7 @@ class Install:
     """Reserved filenames that cannot be used as run names."""
 
     SYMLINK_DIRS = frozenset([
-        SHARE_CYCLE_DIR, SHARE_DIR, LogDir.DIRNAME, WORK_DIR, ''
+        SHARE_CYCLE_DIR, SHARE_DIR, LogDir.DIRNAME, LOG_JOB_DIR, WORK_DIR, ''
     ])
     """The paths of the symlink dirs that may be set in
     global.cylc[install][symlink dirs], relative to the run dir
diff --git a/tests/functional/cylc-clean/00-basic.t b/tests/functional/cylc-clean/00-basic.t
index 1f4cec84984..7c880217aae 100644
--- a/tests/functional/cylc-clean/00-basic.t
+++ b/tests/functional/cylc-clean/00-basic.t
@@ -33,6 +33,7 @@ create_test_global_config "" "
         [[[localhost]]]
             run = ${TEST_DIR}/${SYM_NAME}/run
             log = ${TEST_DIR}/${SYM_NAME}/log
+            log/job = ${TEST_DIR}/${SYM_NAME}/log
             share = ${TEST_DIR}/${SYM_NAME}/share
             share/cycle = ${TEST_DIR}/${SYM_NAME}/cycle
             work = ${TEST_DIR}/${SYM_NAME}/work
@@ -68,8 +69,9 @@ ${TEST_DIR}/${SYM_NAME}/log/cylc-run/${CYLC_TEST_REG_BASE}
 |   \`-- cylc-clean
 |       \`-- ${TEST_NAME_BASE}
 |           \`-- log
-|               \`-- install
-|                   \`-- ${INSTALL_LOG_FILE}
+|               |-- install
+|               |   \`-- ${INSTALL_LOG_FILE}
+|               \`-- job
 \`-- leave-me-alone
 ${TEST_DIR}/${SYM_NAME}/run/cylc-run/${CYLC_TEST_REG_BASE}
 \`-- ${FUNCTIONAL_DIR}
diff --git a/tests/functional/cylc-clean/01-remote.t b/tests/functional/cylc-clean/01-remote.t
index 67148478eb0..db9669d91ec 100644
--- a/tests/functional/cylc-clean/01-remote.t
+++ b/tests/functional/cylc-clean/01-remote.t
@@ -37,6 +37,7 @@ create_test_global_config "" "
         [[[${CYLC_TEST_INSTALL_TARGET}]]]
             run = ${TEST_DIR}/${SYM_NAME}/run
             log = ${TEST_DIR}/${SYM_NAME}/other
+            log/job = ${TEST_DIR}/${SYM_NAME}/job
             share = ${TEST_DIR}/${SYM_NAME}/other
             share/cycle = ${TEST_DIR}/${SYM_NAME}/cycle
             work = ${TEST_DIR}/${SYM_NAME}/other
diff --git a/tests/functional/cylc-install/01-symlinks.t b/tests/functional/cylc-install/01-symlinks.t
index 8fbcc66c1ca..2f55b01d8b0 100644
--- a/tests/functional/cylc-install/01-symlinks.t
+++ b/tests/functional/cylc-install/01-symlinks.t
@@ -24,7 +24,7 @@ if [[ -z ${TMPDIR:-} || -z ${USER:-} || $TMPDIR/$USER == "$HOME" ]]; then
     skip_all '"TMPDIR" or "USER" not defined or "TMPDIR"/"USER" is "HOME"'
 fi
 
-set_test_number 30
+set_test_number 32
 
 create_test_global_config "" "
 [install]
@@ -33,6 +33,7 @@ create_test_global_config "" "
         run = \$TMPDIR/\$USER/test_cylc_symlink/cylctb_tmp_run_dir
         share = \$TMPDIR/\$USER/test_cylc_symlink/
         log = \$TMPDIR/\$USER/test_cylc_symlink/
+        log/job = \$TMPDIR/\$USER/test_cylc_symlink/job_log_dir
         share/cycle = \$TMPDIR/\$USER/test_cylc_symlink/cylctb_tmp_share_dir
         work = \$TMPDIR/\$USER/test_cylc_symlink/
 "
@@ -53,6 +54,10 @@ TEST_SYM="${TEST_NAME_BASE}-share-cycle-glblcfg"
 run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/share/cycle")" \
     = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_share_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/share/cycle"
 
+TEST_SYM="${TEST_NAME_BASE}-log-job-glblcfg"
+run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/log/job")" \
+    = "$TMPDIR/${USER}/test_cylc_symlink/job_log_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/log/job"
+
 for DIR in 'work' 'share' 'log'; do
     TEST_SYM="${TEST_NAME_BASE}-${DIR}-glbcfg"
     run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/${DIR}")" \
@@ -137,6 +142,10 @@ TEST_SYM="${TEST_NAME}-share-cycle"
 run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/share/cycle")" \
     = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_share_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/share/cycle"
 
+TEST_SYM="${TEST_NAME}-log-job"
+run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/log/job")" \
+    = "$TMPDIR/${USER}/test_cylc_symlink/job_log_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/log/job"
+
 for DIR in 'work' 'share' 'log'; do
     TEST_SYM="${TEST_NAME}-${DIR}"
     run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/${DIR}")" \
diff --git a/tests/functional/remote/04-symlink-dirs.t b/tests/functional/remote/04-symlink-dirs.t
index 28ab6049694..1c19f90c510 100644
--- a/tests/functional/remote/04-symlink-dirs.t
+++ b/tests/functional/remote/04-symlink-dirs.t
@@ -24,7 +24,7 @@ if [[ -z ${TMPDIR:-} || -z ${USER:-} || $TMPDIR/$USER == "$HOME" ]]; then
     skip_all '"TMPDIR" or "USER" not defined or "TMPDIR"/"USER" is "HOME"'
 fi
 
-set_test_number 12
+set_test_number 14
 
 create_test_global_config "" "
 [install]
@@ -33,12 +33,14 @@ create_test_global_config "" "
             run = \$TMPDIR/\$USER/cylctb_tmp_run_dir
             share = \$TMPDIR/\$USER
             log = \$TMPDIR/\$USER
+            log/job = \$TMPDIR/\$USER/cylctb_tmp_log_job_dir
             share/cycle = \$TMPDIR/\$USER/cylctb_tmp_share_dir
             work = \$TMPDIR/\$USER
         [[[$CYLC_TEST_INSTALL_TARGET]]]
             run = \$TMPDIR/\$USER/test_cylc_symlink/ctb_tmp_run_dir
             share = \$TMPDIR/\$USER/test_cylc_symlink/
             log = \$TMPDIR/\$USER/test_cylc_symlink/
+            log/job = \$TMPDIR/\$USER/cylctb_tmp_log_job_dir
             share/cycle = \$TMPDIR/\$USER/test_cylc_symlink/ctb_tmp_share_dir
             work = \$TMPDIR/\$USER/test_cylc_symlink/
 "
@@ -66,6 +68,14 @@ else
     fail "$TEST_SYM.localhost"
 fi
 
+TEST_SYM="${TEST_NAME_BASE}-log/job-symlink-exists-ok"
+if [[ $(readlink "$HOME/cylc-run/${WORKFLOW_NAME}/log/job") == \
+"$TMPDIR/$USER/cylctb_tmp_log_job_dir/cylc-run/${WORKFLOW_NAME}/log/job" ]]; then
+    ok "$TEST_SYM.localhost"
+else
+    fail "$TEST_SYM.localhost"
+fi
+
 for DIR in 'work' 'share' 'log'; do
     TEST_SYM="${TEST_NAME_BASE}-${DIR}-symlink-exists-ok"
     if [[ $(readlink "$HOME/cylc-run/${WORKFLOW_NAME}/${DIR}") == \
@@ -94,6 +104,14 @@ else
     fail "${TEST_NAME_BASE}-share/cycle-symlink-exists-ok.remotehost"
 fi
 
+# shellcheck disable=SC2016
+LINK="$(${SSH} "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"/log/job'"')"
+if [[ "$LINK" == *"/test_cylc_symlink/cylctb_tmp_log_job_dir/cylc-run/${WORKFLOW_NAME}/log/job" ]]; then
+    ok "${TEST_NAME_BASE}-log/job-symlink-exists-ok.remotehost"
+else
+    fail "${TEST_NAME_BASE}-log/job-symlink-exists-ok.remotehost"
+fi
+
 for DIR in 'work' 'share' 'log'; do
 # shellcheck disable=SC2016
     LINK="$(${SSH} "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"/$DIR'"')"
diff --git a/tests/unit/test_pathutil.py b/tests/unit/test_pathutil.py
index 6ed9e9ec120..24cc3999400 100644
--- a/tests/unit/test_pathutil.py
+++ b/tests/unit/test_pathutil.py
@@ -203,6 +203,7 @@ def test_make_workflow_run_tree(
                     run = $DEE
                     work = $DAH
                     log = $DUH
+                    log/job =$RAY
                     share = $DOH
                     share/cycle = $DAH
             ''',
@@ -210,6 +211,7 @@ def test_make_workflow_run_tree(
                 'run': '$DEE/cylc-run/morpheus',
                 'work': '$DAH/cylc-run/morpheus/work',
                 'log': '$DUH/cylc-run/morpheus/log',
+                'log/job': '$RAY/cylc-run/morpheus/log/job',
                 'share': '$DOH/cylc-run/morpheus/share',
                 'share/cycle': '$DAH/cylc-run/morpheus/share/cycle'
             },

From 44e846a8113b883fcb7212aaea4eff486b3621c5 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Thu, 27 Jun 2024 11:36:21 +0100
Subject: [PATCH 11/53] Provided a single source of truth for symlinkable dirs

Ensure that test actually creates symlinks to be cleaned.
---
 cylc/flow/pathutil.py                     |  8 ++++----
 cylc/flow/workflow_files.py               |  5 ++---
 tests/functional/cylc-clean/00-basic.t    | 14 ++++++++++++--
 tests/functional/cylc-clean/01-remote.t   | 16 +++++++++++-----
 tests/functional/remote/04-symlink-dirs.t |  2 +-
 5 files changed, 30 insertions(+), 15 deletions(-)

diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py
index a3c7f9d2b73..5311c0ea6ac 100644
--- a/cylc/flow/pathutil.py
+++ b/cylc/flow/pathutil.py
@@ -47,11 +47,11 @@
 SHELL_ENV_VARS = re.compile(r'\$[^$/]*')
 SYMLINKABLE_LOCATIONS: Dict[str, str] = {
     # Location: Version Added
-    'log': '8.0.0',
-    'log/job': '8.4.0',
     'share': '8.0.0',
     'share/cycle': '8.0.0',
-    'work': '8.0.0'
+    'log': '8.0.0',
+    'log/job': '8.4.0',
+    'work': '8.0.0',
 }
 
 
@@ -242,7 +242,7 @@ def get_dirs_to_symlink(
     if base_dir:
         dirs_to_symlink['run'] = os.path.join(
             base_dir, 'cylc-run', workflow_id)
-    for dir_ in SYMLINKABLE_LOCATIONS.keys():
+    for dir_ in SYMLINKABLE_LOCATIONS:
         link = symlink_conf[install_target].get(dir_, None)
         if (not link) or link == base_dir:
             continue
diff --git a/cylc/flow/workflow_files.py b/cylc/flow/workflow_files.py
index 0aac6bc174a..95626125f89 100644
--- a/cylc/flow/workflow_files.py
+++ b/cylc/flow/workflow_files.py
@@ -59,6 +59,7 @@
     is_remote_host,
 )
 from cylc.flow.pathutil import (
+    SYMLINKABLE_LOCATIONS,
     expand_path,
     get_cylc_run_dir,
     get_workflow_run_dir,
@@ -263,9 +264,7 @@ class Install:
     RESERVED_NAMES = frozenset([FLOW_FILE, SUITE_RC, *RESERVED_DIRNAMES])
     """Reserved filenames that cannot be used as run names."""
 
-    SYMLINK_DIRS = frozenset([
-        SHARE_CYCLE_DIR, SHARE_DIR, LogDir.DIRNAME, LOG_JOB_DIR, WORK_DIR, ''
-    ])
+    SYMLINK_DIRS = frozenset(list(SYMLINKABLE_LOCATIONS) + [''])
     """The paths of the symlink dirs that may be set in
     global.cylc[install][symlink dirs], relative to the run dir
     ('' represents the run dir)."""
diff --git a/tests/functional/cylc-clean/00-basic.t b/tests/functional/cylc-clean/00-basic.t
index 7c880217aae..8fffd672e26 100644
--- a/tests/functional/cylc-clean/00-basic.t
+++ b/tests/functional/cylc-clean/00-basic.t
@@ -33,7 +33,7 @@ create_test_global_config "" "
         [[[localhost]]]
             run = ${TEST_DIR}/${SYM_NAME}/run
             log = ${TEST_DIR}/${SYM_NAME}/log
-            log/job = ${TEST_DIR}/${SYM_NAME}/log
+            log/job = ${TEST_DIR}/${SYM_NAME}/job
             share = ${TEST_DIR}/${SYM_NAME}/share
             share/cycle = ${TEST_DIR}/${SYM_NAME}/cycle
             work = ${TEST_DIR}/${SYM_NAME}/work
@@ -56,6 +56,10 @@ cmp_ok "${TEST_NAME}.stdout" <<< "${TEST_DIR}/${SYM_NAME}/run/cylc-run/${WORKFLO
 INSTALL_LOG_FILE=$(ls "${TEST_DIR}/${SYM_NAME}/log/cylc-run/${WORKFLOW_NAME}/log/install")
 TEST_NAME="test-dir-tree-pre-clean"
 run_ok "${TEST_NAME}" tree --noreport --charset=ascii "${TEST_DIR}/${SYM_NAME}/"*"/cylc-run/${CYLC_TEST_REG_BASE}"
+# cat ${TEST_NAME}.stdout >&2
+# echo "${CYLC_TEST_REG_BASE}" >&2
+# echo ${TEST_DIR}/${SYM_NAME} >&2
+# sleep 600
 # Note: backticks need to be escaped in the heredoc
 cmp_ok "${TEST_NAME}.stdout" << __TREE__
 ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${CYLC_TEST_REG_BASE}
@@ -64,6 +68,12 @@ ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${CYLC_TEST_REG_BASE}
         \`-- ${TEST_NAME_BASE}
             \`-- share
                 \`-- cycle
+${TEST_DIR}/${SYM_NAME}/job/cylc-run/${CYLC_TEST_REG_BASE}
+\`-- ${FUNCTIONAL_DIR}
+    \`-- cylc-clean
+        \`-- ${TEST_NAME_BASE}
+            \`-- log
+                \`-- job
 ${TEST_DIR}/${SYM_NAME}/log/cylc-run/${CYLC_TEST_REG_BASE}
 |-- ${FUNCTIONAL_DIR}
 |   \`-- cylc-clean
@@ -71,7 +81,7 @@ ${TEST_DIR}/${SYM_NAME}/log/cylc-run/${CYLC_TEST_REG_BASE}
 |           \`-- log
 |               |-- install
 |               |   \`-- ${INSTALL_LOG_FILE}
-|               \`-- job
+|               \`-- job -> ${TEST_DIR}/${SYM_NAME}/job/cylc-run/${WORKFLOW_NAME}/log/job
 \`-- leave-me-alone
 ${TEST_DIR}/${SYM_NAME}/run/cylc-run/${CYLC_TEST_REG_BASE}
 \`-- ${FUNCTIONAL_DIR}
diff --git a/tests/functional/cylc-clean/01-remote.t b/tests/functional/cylc-clean/01-remote.t
index db9669d91ec..330fc6c69ff 100644
--- a/tests/functional/cylc-clean/01-remote.t
+++ b/tests/functional/cylc-clean/01-remote.t
@@ -82,16 +82,22 @@ ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${CYLC_TEST_REG_BASE}
 |           \`-- share
 |               \`-- cycle
 \`-- leave-me-alone
+${TEST_DIR}/${SYM_NAME}/job/cylc-run/${CYLC_TEST_REG_BASE}
+\`-- ${FUNCTIONAL_DIR}
+    \`-- cylc-clean
+        \`-- ${TEST_NAME_BASE}
+            \`-- log
+                \`-- job
+                    \`-- 1
+                        \`-- santa
+                            |-- 01
+                            \`-- NN -> 01
 ${TEST_DIR}/${SYM_NAME}/other/cylc-run/${CYLC_TEST_REG_BASE}
 \`-- ${FUNCTIONAL_DIR}
     \`-- cylc-clean
         \`-- ${TEST_NAME_BASE}
             |-- log
-            |   \`-- job
-            |       \`-- 1
-            |           \`-- santa
-            |               |-- 01
-            |               \`-- NN -> 01
+            |   \`-- job -> ${TEST_DIR}/${SYM_NAME}/job/cylc-run/${WORKFLOW_NAME}/log/job
             |-- share
             |   \`-- cycle -> ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${WORKFLOW_NAME}/share/cycle
             \`-- work
diff --git a/tests/functional/remote/04-symlink-dirs.t b/tests/functional/remote/04-symlink-dirs.t
index 1c19f90c510..720c2f0fa34 100644
--- a/tests/functional/remote/04-symlink-dirs.t
+++ b/tests/functional/remote/04-symlink-dirs.t
@@ -106,7 +106,7 @@ fi
 
 # shellcheck disable=SC2016
 LINK="$(${SSH} "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"/log/job'"')"
-if [[ "$LINK" == *"/test_cylc_symlink/cylctb_tmp_log_job_dir/cylc-run/${WORKFLOW_NAME}/log/job" ]]; then
+if [[ "$LINK" == *"/cylctb_tmp_log_job_dir/cylc-run/${WORKFLOW_NAME}/log/job" ]]; then
     ok "${TEST_NAME_BASE}-log/job-symlink-exists-ok.remotehost"
 else
     fail "${TEST_NAME_BASE}-log/job-symlink-exists-ok.remotehost"

From 152e794343193fe114de5699c06a58df40dc77a9 Mon Sep 17 00:00:00 2001
From: Hilary James Oliver <hilary.j.oliver@gmail.com>
Date: Thu, 1 Aug 2024 11:47:11 +1200
Subject: [PATCH 12/53] Update tests/functional/cylc-clean/00-basic.t [skip ci]

Co-authored-by: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
---
 tests/functional/cylc-clean/00-basic.t | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tests/functional/cylc-clean/00-basic.t b/tests/functional/cylc-clean/00-basic.t
index 8fffd672e26..72106b87198 100644
--- a/tests/functional/cylc-clean/00-basic.t
+++ b/tests/functional/cylc-clean/00-basic.t
@@ -56,10 +56,6 @@ cmp_ok "${TEST_NAME}.stdout" <<< "${TEST_DIR}/${SYM_NAME}/run/cylc-run/${WORKFLO
 INSTALL_LOG_FILE=$(ls "${TEST_DIR}/${SYM_NAME}/log/cylc-run/${WORKFLOW_NAME}/log/install")
 TEST_NAME="test-dir-tree-pre-clean"
 run_ok "${TEST_NAME}" tree --noreport --charset=ascii "${TEST_DIR}/${SYM_NAME}/"*"/cylc-run/${CYLC_TEST_REG_BASE}"
-# cat ${TEST_NAME}.stdout >&2
-# echo "${CYLC_TEST_REG_BASE}" >&2
-# echo ${TEST_DIR}/${SYM_NAME} >&2
-# sleep 600
 # Note: backticks need to be escaped in the heredoc
 cmp_ok "${TEST_NAME}.stdout" << __TREE__
 ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${CYLC_TEST_REG_BASE}

From 7dc77e9d068fa9ab670f9b1abed2ad2fde42cbda Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Thu, 1 Aug 2024 16:24:40 +0100
Subject: [PATCH 13/53] Fix changelog fragment name

---
 changes.d/{feat.6168.md => 6168.feat.md} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename changes.d/{feat.6168.md => 6168.feat.md} (100%)

diff --git a/changes.d/feat.6168.md b/changes.d/6168.feat.md
similarity index 100%
rename from changes.d/feat.6168.md
rename to changes.d/6168.feat.md

From 6976854e1e643f3863c393ab21af864153e9fe30 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Tue, 6 Aug 2024 14:36:19 +0100
Subject: [PATCH 14/53] Remove cached prerequisite satisfaction

This manually cached property has the possibility to drift out of sync with individual `Prerequisite` satisfaction states, especially with `cylc remove` work in progress
---
 cylc/flow/task_state.py | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/cylc/flow/task_state.py b/cylc/flow/task_state.py
index ebb3dbc985b..c26b84c0414 100644
--- a/cylc/flow/task_state.py
+++ b/cylc/flow/task_state.py
@@ -195,9 +195,6 @@ class TaskState:
             Time string of latest update time.
         .xtriggers (dict):
             xtriggers as {trigger (str): satisfied (boolean), ...}.
-        ._is_satisfied (boolean):
-            Are prerequisites satisfied?
-            Reset None to force re-evaluation when a prereq gets satisfied.
         ._suicide_is_satisfied (boolean):
             Are prerequisites to trigger suicide satisfied?
             Reset None to force re-evaluation when a prereq gets satisfied.
@@ -217,7 +214,6 @@ class TaskState:
         "suicide_prerequisites",
         "time_updated",
         "xtriggers",
-        "_is_satisfied",
         "_suicide_is_satisfied",
     ]
 
@@ -229,7 +225,6 @@ def __init__(self, tdef, point, status, is_held):
         self.is_updated = False
         self.time_updated = None
 
-        self._is_satisfied = None
         self._suicide_is_satisfied = None
 
         # Prerequisites.
@@ -324,7 +319,6 @@ def satisfy_me(
             if yep:
                 valid = valid.union(yep)
                 continue
-            self._is_satisfied = None
             self._suicide_is_satisfied = None
         return valid
 
@@ -340,14 +334,10 @@ def set_all_satisfied(self):
         """Set all my prerequisites satisfied."""
         for p in self.prerequisites:
             p.set_satisfied()
-        self._is_satisfied = True
 
     def prerequisites_all_satisfied(self):
         """Return True if (non-suicide) prerequisites are fully satisfied."""
-        if self._is_satisfied is None:
-            self._is_satisfied = all(
-                preq.is_satisfied() for preq in self.prerequisites)
-        return self._is_satisfied
+        return all(preq.is_satisfied() for preq in self.prerequisites)
 
     def prerequisites_are_not_all_satisfied(self):
         """Return True if (any) prerequisites are not fully satisfied."""
@@ -380,7 +370,6 @@ def set_prerequisites_all_satisfied(self):
         """Set prerequisites to all satisfied."""
         for prereq in self.prerequisites:
             prereq.set_satisfied()
-        self._is_satisfied = None
 
     def get_resolved_dependencies(self):
         """Return a list of dependencies which have been met for this task.
@@ -467,7 +456,6 @@ def _add_prerequisites(self, point, tdef):
         """Add task prerequisites."""
         # Triggers for sequence_i only used if my cycle point is a
         # valid member of sequence_i's sequence of cycle points.
-        self._is_satisfied = None
         self._suicide_is_satisfied = None
 
         # Use dicts to avoid generating duplicate prerequisites from sequences

From f658e9351f77b2fee39ea08cdce3e0c309f0ee13 Mon Sep 17 00:00:00 2001
From: Paul Earnshaw <pdearnshaw+github@gmail.com>
Date: Tue, 6 Aug 2024 15:40:02 +0100
Subject: [PATCH 15/53] Made jinja2 subsitution regex less greedy

---
 cylc/flow/etc/syntax/cylc.lang | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cylc/flow/etc/syntax/cylc.lang b/cylc/flow/etc/syntax/cylc.lang
index 0270c7f3cfc..c3f43da2c95 100644
--- a/cylc/flow/etc/syntax/cylc.lang
+++ b/cylc/flow/etc/syntax/cylc.lang
@@ -219,7 +219,7 @@
       <end>#\}</end>
     </context>
     <context id="jinja2-substitution" style-ref="template">
-      <match>\{\{.*\}\}</match>
+      <match>\{\{.*?\}\}</match>
     </context>
     <context id="jinja2-block" style-ref="template" end-at-line-end="false">
       <start>\{%</start>

From 7d282bce2d61b8595e7e08569739a0b282df3f7d Mon Sep 17 00:00:00 2001
From: Paul Earnshaw <pdearnshaw+github@gmail.com>
Date: Tue, 6 Aug 2024 15:46:42 +0100
Subject: [PATCH 16/53] Added name to CONTRIBUTING.md

---
 CONTRIBUTING.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 30e81f26f93..adc342dcab5 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -95,6 +95,7 @@ requests_).
  - Mark Dawson
  - Diquan Jabbour
  - Shixian Sheng
+ - Paul Earnshaw
 <!-- end-shortlog -->
 
 (All contributors are identifiable with email addresses in the git version

From 2c2839e79a117913e7c9853166511f0d9187d877 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Tue, 6 Aug 2024 15:54:51 +0100
Subject: [PATCH 17/53] Remove cached suicide prerequisite satisfaction

---
 cylc/flow/task_state.py | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/cylc/flow/task_state.py b/cylc/flow/task_state.py
index c26b84c0414..06ee8ae6e97 100644
--- a/cylc/flow/task_state.py
+++ b/cylc/flow/task_state.py
@@ -195,9 +195,6 @@ class TaskState:
             Time string of latest update time.
         .xtriggers (dict):
             xtriggers as {trigger (str): satisfied (boolean), ...}.
-        ._suicide_is_satisfied (boolean):
-            Are prerequisites to trigger suicide satisfied?
-            Reset None to force re-evaluation when a prereq gets satisfied.
     """
 
     # Memory optimization - constrain possible attributes to this list.
@@ -214,7 +211,6 @@ class TaskState:
         "suicide_prerequisites",
         "time_updated",
         "xtriggers",
-        "_suicide_is_satisfied",
     ]
 
     def __init__(self, tdef, point, status, is_held):
@@ -225,8 +221,6 @@ def __init__(self, tdef, point, status, is_held):
         self.is_updated = False
         self.time_updated = None
 
-        self._suicide_is_satisfied = None
-
         # Prerequisites.
         self.prerequisites: List[Prerequisite] = []
         self.suicide_prerequisites: List[Prerequisite] = []
@@ -319,7 +313,6 @@ def satisfy_me(
             if yep:
                 valid = valid.union(yep)
                 continue
-            self._suicide_is_satisfied = None
         return valid
 
     def xtriggers_all_satisfied(self):
@@ -346,10 +339,7 @@ def prerequisites_are_not_all_satisfied(self):
 
     def suicide_prerequisites_all_satisfied(self):
         """Return True if all suicide prerequisites are satisfied."""
-        if self._suicide_is_satisfied is None:
-            self._suicide_is_satisfied = all(
-                preq.is_satisfied() for preq in self.suicide_prerequisites)
-        return self._suicide_is_satisfied
+        return all(preq.is_satisfied() for preq in self.suicide_prerequisites)
 
     def prerequisites_get_target_points(self):
         """Return a list of cycle points targeted by each prerequisite."""
@@ -456,7 +446,6 @@ def _add_prerequisites(self, point, tdef):
         """Add task prerequisites."""
         # Triggers for sequence_i only used if my cycle point is a
         # valid member of sequence_i's sequence of cycle points.
-        self._suicide_is_satisfied = None
 
         # Use dicts to avoid generating duplicate prerequisites from sequences
         # with coincident cycle points.

From 80fa008e18a6feb8ecb21123bde16c048fd1b599 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Tue, 6 Aug 2024 16:08:28 +0100
Subject: [PATCH 18/53] Tidy

---
 cylc/flow/task_pool.py  |  2 +-
 cylc/flow/task_state.py | 21 +++++++++------------
 2 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py
index 78a9d0c6f9f..72e8065d4ca 100644
--- a/cylc/flow/task_pool.py
+++ b/cylc/flow/task_pool.py
@@ -1990,7 +1990,7 @@ def _set_prereqs_itask(
 
         """
         if prereqs == ["all"]:
-            itask.state.set_all_satisfied()
+            itask.state.set_prerequisites_all_satisfied()
         else:
             # Attempt to set the given presrequisites.
             # Log any that aren't valid for the task.
diff --git a/cylc/flow/task_state.py b/cylc/flow/task_state.py
index 06ee8ae6e97..a4adb53d374 100644
--- a/cylc/flow/task_state.py
+++ b/cylc/flow/task_state.py
@@ -309,10 +309,9 @@ def satisfy_me(
         """
         valid: Set[Tokens] = set()
         for prereq in (*self.prerequisites, *self.suicide_prerequisites):
-            yep = prereq.satisfy_me(outputs)
-            if yep:
-                valid = valid.union(yep)
-                continue
+            valid.update(
+                prereq.satisfy_me(outputs)
+            )
         return valid
 
     def xtriggers_all_satisfied(self):
@@ -323,11 +322,6 @@ def external_triggers_all_satisfied(self):
         """Return True if all external triggers are satisfied."""
         return all(self.external_triggers.values())
 
-    def set_all_satisfied(self):
-        """Set all my prerequisites satisfied."""
-        for p in self.prerequisites:
-            p.set_satisfied()
-
     def prerequisites_all_satisfied(self):
         """Return True if (non-suicide) prerequisites are fully satisfied."""
         return all(preq.is_satisfied() for preq in self.prerequisites)
@@ -349,9 +343,12 @@ def prerequisites_get_target_points(self):
             for point in prerequisite.get_target_points()
         }
 
-    def prerequisites_eval_all(self):
-        """Set all prerequisites to satisfied."""
-        # (Validation: will abort on illegal trigger expressions.)
+    def prerequisites_eval_all(self) -> None:
+        """Evaluate satisifaction of all prerequisites and
+        suicide prerequisites.
+
+        Provides validation - will abort on illegal trigger expressions.
+        """
         for preqs in [self.prerequisites, self.suicide_prerequisites]:
             for preq in preqs:
                 preq.is_satisfied()

From be934fd8d67611f2673b4dad5103534652dca096 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Thu, 8 Aug 2024 12:24:46 +0100
Subject: [PATCH 19/53] Fix small bug in logging force-satisfied prereqs

---
 cylc/flow/prerequisite.py       |  2 +-
 tests/unit/test_prerequisite.py | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/cylc/flow/prerequisite.py b/cylc/flow/prerequisite.py
index 449a5009656..915397ee224 100644
--- a/cylc/flow/prerequisite.py
+++ b/cylc/flow/prerequisite.py
@@ -295,4 +295,4 @@ def get_resolved_dependencies(self):
         """
         return [f'{point}/{name}' for
                 (point, name, _), satisfied in self.satisfied.items() if
-                satisfied == self.DEP_STATE_SATISFIED]
+                satisfied]
diff --git a/tests/unit/test_prerequisite.py b/tests/unit/test_prerequisite.py
index 410176e508c..1514ad07f66 100644
--- a/tests/unit/test_prerequisite.py
+++ b/tests/unit/test_prerequisite.py
@@ -16,6 +16,7 @@
 
 import pytest
 
+from cylc.flow.cycling.integer import IntegerPoint
 from cylc.flow.cycling.loader import ISO8601_CYCLING_TYPE, get_point
 from cylc.flow.prerequisite import Prerequisite
 from cylc.flow.id import Tokens
@@ -102,3 +103,16 @@ def test_get_target_points(prereq):
         get_point('2000'),
         get_point('2001'),
     }
+
+
+def test_get_resolved_dependencies():
+    prereq = Prerequisite(IntegerPoint('2'))
+    prereq.satisfied[('1', 'a', 'x')] = 'satisfied naturally'
+    prereq.satisfied[('1', 'b', 'x')] = False
+    prereq.satisfied[('1', 'c', 'x')] = 'satisfied from database'
+    prereq.satisfied[('1', 'd', 'x')] = 'force satisfied'
+    assert prereq.get_resolved_dependencies() == [
+        '1/a',
+        '1/c',
+        '1/d',
+    ]

From 8c337c83daa9739d4220d48e5beb702855755232 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Thu, 8 Aug 2024 12:19:21 +0100
Subject: [PATCH 20/53] Rework `Prerequisite` class to ensure caching of
 satisfaction state is handled automatically

---
 cylc/flow/data_store_mgr.py              |  17 +-
 cylc/flow/prerequisite.py                | 222 ++++++++++++++---------
 cylc/flow/task_pool.py                   |  21 ++-
 cylc/flow/task_proxy.py                  |  35 ++--
 cylc/flow/task_state.py                  |  40 ++--
 cylc/flow/task_trigger.py                |  52 +++---
 cylc/flow/taskdef.py                     |  14 +-
 cylc/flow/workflow_db_mgr.py             |   2 +-
 tests/integration/scripts/test_set.py    |   8 +-
 tests/integration/test_data_store_mgr.py |  38 ++--
 tests/integration/test_task_pool.py      |  18 +-
 tests/unit/test_prerequisite.py          |  97 ++++++----
 tests/unit/test_task_state.py            |   3 +-
 13 files changed, 342 insertions(+), 225 deletions(-)

diff --git a/cylc/flow/data_store_mgr.py b/cylc/flow/data_store_mgr.py
index b98a055f882..459abf77e56 100644
--- a/cylc/flow/data_store_mgr.py
+++ b/cylc/flow/data_store_mgr.py
@@ -116,6 +116,7 @@
 if TYPE_CHECKING:
     from cylc.flow.cycling import PointBase
     from cylc.flow.flow_mgr import FlowNums
+    from cylc.flow.prerequisite import Prerequisite
 
 EDGES = 'edges'
 FAMILIES = 'families'
@@ -1444,7 +1445,7 @@ def apply_task_proxy_db_history(self):
             prereq_ids.add(f'{relative_id}/{flow_nums_str}')
 
         # Batch load prerequisites of tasks according to flow.
-        prereqs_map = {}
+        prereqs_map: Dict[str, dict] = {}
         for (
                 cycle, name, prereq_name,
                 prereq_cycle, prereq_output, satisfied
@@ -1458,16 +1459,14 @@ def apply_task_proxy_db_history(self):
             ] = satisfied if satisfied != '0' else False
 
         for ikey, prereqs in prereqs_map.items():
+            itask_prereq: Prerequisite
             for itask_prereq in (
-                    self.db_load_task_proxies[ikey][0].state.prerequisites
+                self.db_load_task_proxies[ikey][0].state.prerequisites
             ):
-                for key in itask_prereq.satisfied.keys():
-                    try:
-                        itask_prereq.satisfied[key] = prereqs[key]
-                    except KeyError:
-                        # This prereq is not in the DB: new dependencies
-                        # added to an already-spawned task before restart.
-                        itask_prereq.satisfied[key] = False
+                for key in itask_prereq:
+                    itask_prereq[key] = prereqs.get(key, False)
+                    # (False if prereq is not in the DB: new dependencies
+                    # added to an already-spawned task before restart.)
 
         # Extract info from itasks to data-store.
         for task_info in self.db_load_task_proxies.values():
diff --git a/cylc/flow/prerequisite.py b/cylc/flow/prerequisite.py
index 915397ee224..ad608af78a9 100644
--- a/cylc/flow/prerequisite.py
+++ b/cylc/flow/prerequisite.py
@@ -18,20 +18,66 @@
 
 import math
 import re
-from typing import Iterable, Set, TYPE_CHECKING
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    ItemsView,
+    Iterable,
+    Iterator,
+    List,
+    NamedTuple,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+)
+
+# BACK COMPAT: typing_extensions.Literal
+# FROM: Python 3.7
+# TO: Python 3.8
+from typing_extensions import Literal
 
 from cylc.flow.cycling.loader import get_point
+from cylc.flow.data_messages_pb2 import PbCondition, PbPrerequisite
 from cylc.flow.exceptions import TriggerExpressionError
-from cylc.flow.data_messages_pb2 import (
-    PbPrerequisite,
-    PbCondition,
-)
 from cylc.flow.id import quick_relative_detokenise
 
+
 if TYPE_CHECKING:
+    from cylc.flow.cycling import PointBase
     from cylc.flow.id import Tokens
 
 
+AnyPrereqMessage = Tuple[Union['PointBase', str, int], str, str]
+
+
+class PrereqMessage(NamedTuple):
+    """A message pertaining to a Prerequisite."""
+    point: str
+    task: str
+    output: str
+
+    def get_id(self) -> str:
+        """Get the relative ID of the task in this prereq message."""
+        return quick_relative_detokenise(self.point, self.task)
+
+    @staticmethod
+    def coerce(tuple_: AnyPrereqMessage) -> 'PrereqMessage':
+        """Coerce a tuple to a PrereqMessage."""
+        if isinstance(tuple_, PrereqMessage):
+            return tuple_
+        point, task, output = tuple_
+        return PrereqMessage(point=str(point), task=task, output=output)
+
+
+SatisfiedState = Literal[
+    'satisfied naturally',
+    'satisfied from database',
+    'force satisfied',
+    False
+]
+
+
 class Prerequisite:
     """The concrete result of an abstract logical trigger expression.
 
@@ -46,70 +92,83 @@ class Prerequisite:
 
     # Memory optimization - constrain possible attributes to this list.
     __slots__ = (
-        "satisfied",
+        "_satisfied",
         "_all_satisfied",
         "conditional_expression",
         "point",
     )
 
     # Extracts T from "foo.T succeeded" etc.
-    SATISFIED_TEMPLATE = 'bool(self.satisfied[("%s", "%s", "%s")])'
+    SATISFIED_TEMPLATE = 'bool(self._satisfied[("%s", "%s", "%s")])'
     MESSAGE_TEMPLATE = r'%s/%s %s'
 
-    DEP_STATE_SATISFIED = 'satisfied naturally'
-    DEP_STATE_OVERRIDDEN = 'force satisfied'
-    DEP_STATE_UNSATISFIED = False
-
-    def __init__(self, point):
+    def __init__(self, point: 'PointBase'):
         # The cycle point to which this prerequisite belongs.
         # cylc.flow.cycling.PointBase
         self.point = point
 
         # Dictionary of messages pertaining to this prerequisite.
         # {('point string', 'task name', 'output'): DEP_STATE_X, ...}
-        self.satisfied = {}
+        self._satisfied: Dict[PrereqMessage, SatisfiedState] = {}
 
         # Expression present only when conditions are used.
         # '1/foo failed & 1/bar succeeded'
-        self.conditional_expression = None
+        self.conditional_expression: Optional[str] = None
 
         # The cached state of this prerequisite:
         # * `None` (no cached state)
         # * `True` (prerequisite satisfied)
         # * `False` (prerequisite unsatisfied).
-        self._all_satisfied = None
+        self._all_satisfied: Optional[bool] = None
 
-    def instantaneous_hash(self):
+    def instantaneous_hash(self) -> int:
         """Generate a hash of this prerequisite in its current state.
 
-        Note not using `__hash__` because Prerequisite objects are mutable.
+        NOTE: Is not affected by any change in satisfaction state.
+
+        (Not defining `self.__hash__()` because Prerequisite objects
+        are mutable.)
         """
         return hash((
             self.point,
             self.conditional_expression,
-            tuple(self.satisfied.keys()),
+            tuple(self._satisfied.keys()),
         ))
 
-    def add(self, name, point, output, pre_initial=False):
+    def __getitem__(self, key: AnyPrereqMessage) -> SatisfiedState:
+        """Return the satisfaction state of a dependency.
+
+        Args:
+            key: Tuple of (point, name, output) for a task.
+        """
+        return self._satisfied[PrereqMessage.coerce(key)]
+
+    def __setitem__(
+        self,
+        key: AnyPrereqMessage,
+        value: Union[SatisfiedState, bool] = False,
+    ) -> None:
         """Register an output with this prerequisite.
 
         Args:
-            name (str): The name of the task to which the output pertains.
-            point (str/cylc.flow.cycling.PointBase): The cycle point at which
-                this dependent output should appear.
-            output (str): String representing the output e.g. "succeeded".
-            pre_initial (bool): this is a pre-initial dependency.
+            key: Tuple of (point, name, output) for a prerequisite task.
+            value: Dependency satisfaction state (for pre-initial dependencies
+                this should be True).
 
         """
-        message = (str(point), name, output)
+        key = PrereqMessage.coerce(key)
+        if value is True:
+            value = 'satisfied naturally'
+        self._satisfied[key] = value
+        if not (self._all_satisfied and value):
+            # Force later recalculation of cached satisfaction state:
+            self._all_satisfied = None
 
-        # Add a new prerequisite as satisfied if pre-initial, else unsatisfied.
-        if pre_initial:
-            self.satisfied[message] = self.DEP_STATE_SATISFIED
-        else:
-            self.satisfied[message] = self.DEP_STATE_UNSATISFIED
-        if self._all_satisfied is not None:
-            self._all_satisfied = False
+    def __iter__(self) -> Iterator[PrereqMessage]:
+        return iter(self._satisfied)
+
+    def items(self) -> ItemsView[PrereqMessage, SatisfiedState]:
+        return self._satisfied.items()
 
     def get_raw_conditional_expression(self):
         """Return a representation of this prereq as a string.
@@ -120,7 +179,7 @@ def get_raw_conditional_expression(self):
         expr = self.conditional_expression
         if not expr:
             return None
-        for message in self.satisfied:
+        for message in self._satisfied:
             expr = expr.replace(self.SATISFIED_TEMPLATE % message,
                                 self.MESSAGE_TEMPLATE % message)
         return expr
@@ -134,21 +193,19 @@ def set_condition(self, expr):
             # is a substring of another: foo | xfoo => bar.
             # Add 'foo' to the 'satisfied' dict before 'xfoo'.
             >>> preq = Prerequisite(1)
-            >>> preq.satisfied = {
-            ...    ('1', 'foo', 'succeeded'): False,
-            ...    ('1', 'xfoo', 'succeeded'): False
-            ... }
+            >>> preq[(1, 'foo', 'succeeded')] = False
+            >>> preq[(1, 'xfoo', 'succeeded')] = False
             >>> preq.set_condition("1/foo succeeded|1/xfoo succeeded")
             >>> expr = preq.conditional_expression
             >>> expr.split('|')  # doctest: +NORMALIZE_WHITESPACE
-            ['bool(self.satisfied[("1", "foo", "succeeded")])',
-            'bool(self.satisfied[("1", "xfoo", "succeeded")])']
+            ['bool(self._satisfied[("1", "foo", "succeeded")])',
+            'bool(self._satisfied[("1", "xfoo", "succeeded")])']
 
         """
         self._all_satisfied = None
         if '|' in expr:
             # Make a Python expression so we can eval() the logic.
-            for message in self.satisfied:
+            for message in self._satisfied:
                 # Use '\b' in case one task name is a substring of another
                 # and escape special chars ('.', timezone '+') in task IDs.
                 expr = re.sub(
@@ -166,25 +223,23 @@ def is_satisfied(self):
 
         """
         if self._all_satisfied is not None:
+            # Cached value.
             return self._all_satisfied
-        else:
-            # No cached value.
-            if self.satisfied == {}:
-                # No prerequisites left after pre-initial simplification.
-                return True
-            if self.conditional_expression:
-                # Trigger expression with at least one '|': use eval.
-                self._all_satisfied = self._conditional_is_satisfied()
-            else:
-                self._all_satisfied = all(self.satisfied.values())
-            return self._all_satisfied
+        if self._satisfied == {}:
+            # No prerequisites left after pre-initial simplification.
+            return True
+        self._all_satisfied = self._eval_satisfied()
+        return self._all_satisfied
 
-    def _conditional_is_satisfied(self):
+    def _eval_satisfied(self) -> bool:
         """Evaluate the prerequisite's condition expression.
 
         Does not cache the result.
 
         """
+        if not self.conditional_expression:
+            return all(self._satisfied.values())
+
         try:
             res = eval(self.conditional_expression)  # nosec
             # * the expression is constructed internally
@@ -208,20 +263,18 @@ def satisfy_me(self, outputs: Iterable['Tokens']) -> 'Set[Tokens]':
         """
         valid = set()
         for output in outputs:
-            prereq = (output['cycle'], output['task'], output['task_sel'])
-            if prereq not in self.satisfied:
+            prereq = PrereqMessage(
+                output['cycle'], output['task'], output['task_sel']
+            )
+            if prereq not in self._satisfied:
                 continue
             valid.add(output)
-            self.satisfied[prereq] = self.DEP_STATE_SATISFIED
-            if self.conditional_expression is None:
-                self._all_satisfied = all(self.satisfied.values())
-            else:
-                self._all_satisfied = self._conditional_is_satisfied()
+            self[prereq] = 'satisfied naturally'
         return valid
 
-    def api_dump(self):
+    def api_dump(self) -> Optional[PbPrerequisite]:
         """Return list of populated Protobuf data objects."""
-        if not self.satisfied:
+        if not self._satisfied:
             return None
         if self.conditional_expression:
             expr = (
@@ -230,26 +283,22 @@ def api_dump(self):
         else:
             expr = ' & '.join(
                 self.MESSAGE_TEMPLATE % s_msg
-                for s_msg in self.satisfied
+                for s_msg in self._satisfied
             )
         conds = []
-        num_length = math.ceil(len(self.satisfied) / 10)
-        for ind, message_tuple in enumerate(sorted(self.satisfied)):
-            point, name = message_tuple[0:2]
-            t_id = quick_relative_detokenise(point, name)
+        num_length = math.ceil(len(self._satisfied) / 10)
+        for ind, message_tuple in enumerate(sorted(self._satisfied)):
+            t_id = message_tuple.get_id()
             char = 'c%.{0}d'.format(num_length) % ind
             c_msg = self.MESSAGE_TEMPLATE % message_tuple
-            c_val = self.satisfied[message_tuple]
-            c_bool = bool(c_val)
-            if c_bool is False:
-                c_val = "unsatisfied"
+            c_val = self._satisfied[message_tuple]
             conds.append(
                 PbCondition(
                     task_proxy=t_id,
                     expr_alias=char,
-                    req_state=message_tuple[2],
-                    satisfied=c_bool,
-                    message=c_val,
+                    req_state=message_tuple.output,
+                    satisfied=bool(c_val),
+                    message=(c_val or 'unsatisfied'),
                 )
             )
             expr = expr.replace(c_msg, char)
@@ -260,24 +309,23 @@ def api_dump(self):
             cycle_points=sorted(self.iter_target_point_strings()),
         )
 
-    def set_satisfied(self):
+    def set_satisfied(self) -> None:
         """Force this prerequisite into the satisfied state.
 
         State can be overridden by calling `self.satisfy_me`.
 
         """
-        for message in self.satisfied:
-            if not self.satisfied[message]:
-                self.satisfied[message] = self.DEP_STATE_OVERRIDDEN
-        if self.conditional_expression is None:
-            self._all_satisfied = True
+        for message in self._satisfied:
+            if not self._satisfied[message]:
+                self._satisfied[message] = 'force satisfied'
+        if self.conditional_expression:
+            self._all_satisfied = self._eval_satisfied()
         else:
-            self._all_satisfied = self._conditional_is_satisfied()
+            self._all_satisfied = True
 
     def iter_target_point_strings(self):
         yield from {
-            message[0]
-            for message in self.satisfied
+            message.point for message in self._satisfied
         }
 
     def get_target_points(self):
@@ -287,12 +335,14 @@ def get_target_points(self):
             get_point(p) for p in self.iter_target_point_strings()
         ]
 
-    def get_resolved_dependencies(self):
+    def get_resolved_dependencies(self) -> List[str]:
         """Return a list of satisfied dependencies.
 
         E.G: ['1/foo', '2/bar']
 
         """
-        return [f'{point}/{name}' for
-                (point, name, _), satisfied in self.satisfied.items() if
-                satisfied]
+        return [
+            msg.get_id()
+            for msg, satisfied in self._satisfied.items()
+            if satisfied
+        ]
diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py
index 72e8065d4ca..2f953c9812d 100644
--- a/cylc/flow/task_pool.py
+++ b/cylc/flow/task_pool.py
@@ -89,11 +89,12 @@
     from cylc.flow.config import WorkflowConfig
     from cylc.flow.cycling import IntervalBase, PointBase
     from cylc.flow.data_store_mgr import DataStoreMgr
-    from cylc.flow.taskdef import TaskDef
+    from cylc.flow.flow_mgr import FlowMgr, FlowNums
+    from cylc.flow.prerequisite import SatisfiedState
     from cylc.flow.task_events_mgr import TaskEventsManager
-    from cylc.flow.xtrigger_mgr import XtriggerManager
+    from cylc.flow.taskdef import TaskDef
     from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager
-    from cylc.flow.flow_mgr import FlowMgr, FlowNums
+    from cylc.flow.xtrigger_mgr import XtriggerManager
 
 
 Pool = Dict['PointBase', Dict[str, TaskProxy]]
@@ -439,7 +440,7 @@ def check_task_output(
         task: str,
         output_msg: str,
         flow_nums: 'FlowNums',
-    ) -> Union[str, bool]:
+    ) -> 'SatisfiedState':
         """Returns truthy if the specified output is satisfied in the DB."""
         for task_outputs, task_flow_nums in (
             self.workflow_db_mgr.pri_dao.select_task_outputs(task, cycle)
@@ -563,16 +564,16 @@ def load_db_task_pool_for_restart(self, row_idx, row):
                 ] = satisfied if satisfied != '0' else False
 
             for itask_prereq in itask.state.prerequisites:
-                for key in itask_prereq.satisfied.keys():
-                    try:
-                        itask_prereq.satisfied[key] = sat[key]
-                    except KeyError:
+                for key in itask_prereq:
+                    if key in sat:
+                        itask_prereq[key] = sat[key]
+                    else:
                         # This prereq is not in the DB: new dependencies
                         # added to an already-spawned task before restart.
                         # Look through task outputs to see if is has been
                         # satisfied
                         prereq_cycle, prereq_task, prereq_output_msg = key
-                        itask_prereq.satisfied[key] = (
+                        itask_prereq[key] = (
                             self.check_task_output(
                                 prereq_cycle,
                                 prereq_task,
@@ -1995,7 +1996,7 @@ def _set_prereqs_itask(
             # Attempt to set the given presrequisites.
             # Log any that aren't valid for the task.
             presus = self._standardise_prereqs(prereqs)
-            unmatched = itask.satisfy_me(list(presus.keys()))
+            unmatched = itask.satisfy_me(presus.keys())
             for task_msg in unmatched:
                 LOG.warning(
                     f"{itask.identity} does not depend on"
diff --git a/cylc/flow/task_proxy.py b/cylc/flow/task_proxy.py
index 4e7b60d6e0a..27332ac9316 100644
--- a/cylc/flow/task_proxy.py
+++ b/cylc/flow/task_proxy.py
@@ -21,14 +21,15 @@
 from fnmatch import fnmatchcase
 from time import time
 from typing import (
+    TYPE_CHECKING,
     Any,
     Callable,
     Counter as TypingCounter,
     Dict,
+    Iterable,
     List,
     Optional,
     Set,
-    TYPE_CHECKING,
     Tuple,
 )
 
@@ -53,10 +54,12 @@
 
 if TYPE_CHECKING:
     from cylc.flow.cycling import PointBase
+    from cylc.flow.flow_mgr import FlowNums
+    from cylc.flow.id import Tokens
+    from cylc.flow.prerequisite import PrereqMessage, SatisfiedState
     from cylc.flow.simulation import ModeSettings
     from cylc.flow.task_action_timer import TaskActionTimer
     from cylc.flow.taskdef import TaskDef
-    from cylc.flow.id import Tokens
 
 
 class TaskProxy:
@@ -322,7 +325,11 @@ def __str__(self) -> str:
             f"{id_}{stringify_flow_nums(self.flow_nums)}:{self.state}"
         )
 
-    def copy_to_reload_successor(self, reload_successor, check_output):
+    def copy_to_reload_successor(
+        self,
+        reload_successor: 'TaskProxy',
+        check_output: Callable[[str, str, str, 'FlowNums'], 'SatisfiedState'],
+    ):
         """Copy attributes to successor on reload of this task proxy."""
         self.reload_successor = reload_successor
         reload_successor.submit_num = self.submit_num
@@ -346,10 +353,10 @@ def copy_to_reload_successor(self, reload_successor, check_output):
         # pre-reload state of prerequisites that still exist post-reload.
 
         # Get all prereq states, e.g. {('1', 'c', 'succeeded'): False, ...}
-        pre_reload = {
+        pre_reload: Dict[PrereqMessage, SatisfiedState] = {
             k: v
             for pre in self.state.prerequisites
-            for (k, v) in pre.satisfied.items()
+            for (k, v) in pre.items()
         }
         # Use them to update the new prerequisites.
         # - unchanged prerequisites will keep their pre-reload state.
@@ -357,16 +364,12 @@ def copy_to_reload_successor(self, reload_successor, check_output):
         # - added prerequisites will be recorded as unsatisfied
         #   NOTE: even if the corresponding output was completed pre-reload!
         for pre in reload_successor.state.prerequisites:
-            for k in pre.satisfied.keys():
-                try:
-                    pre.satisfied[k] = pre_reload[k]
-                except KeyError:
-                    # Look through task outputs to see if is has been
-                    # satisfied
-                    pre.satisfied[k] = check_output(
-                        *k,
-                        self.flow_nums,
-                    )
+            for k in pre:
+                pre[k] = pre_reload.get(
+                    k,
+                    # Else look thru task outputs to see if it's been satisfied
+                    check_output(*k, self.flow_nums)
+                )
 
         reload_successor.state.xtriggers.update({
             # copy across any special "_cylc" xtriggers which were added
@@ -544,7 +547,7 @@ def state_reset(
         return False
 
     def satisfy_me(
-        self, task_messages: 'List[Tokens]'
+        self, task_messages: 'Iterable[Tokens]'
     ) -> 'Set[Tokens]':
         """Try to satisfy my prerequisites with given output messages.
 
diff --git a/cylc/flow/task_state.py b/cylc/flow/task_state.py
index a4adb53d374..a6a6bc1258f 100644
--- a/cylc/flow/task_state.py
+++ b/cylc/flow/task_state.py
@@ -17,13 +17,24 @@
 """Task state related logic."""
 
 
-from typing import List, Iterable, Set, TYPE_CHECKING
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    Iterable,
+    List,
+    Set,
+)
+
 from cylc.flow.prerequisite import Prerequisite
 from cylc.flow.task_outputs import TaskOutputs
 from cylc.flow.wallclock import get_current_time_string
 
+
 if TYPE_CHECKING:
+    from cylc.flow.cycling import PointBase
     from cylc.flow.id import Tokens
+    from cylc.flow.prerequisite import PrereqMessage
+    from cylc.flow.taskdef import TaskDef
 
 
 # Task status names and meanings.
@@ -439,15 +450,15 @@ def is_gte(self, status):
         return (TASK_STATUSES_ORDERED.index(self.status) >=
                 TASK_STATUSES_ORDERED.index(status))
 
-    def _add_prerequisites(self, point, tdef):
+    def _add_prerequisites(self, point: 'PointBase', tdef: 'TaskDef'):
         """Add task prerequisites."""
         # Triggers for sequence_i only used if my cycle point is a
         # valid member of sequence_i's sequence of cycle points.
 
         # Use dicts to avoid generating duplicate prerequisites from sequences
         # with coincident cycle points.
-        prerequisites = {}
-        suicide_prerequisites = {}
+        prerequisites: Dict[int, Prerequisite] = {}
+        suicide_prerequisites: Dict[int, Prerequisite] = {}
 
         for sequence, dependencies in tdef.dependencies.items():
             if not sequence.is_valid(point):
@@ -470,8 +481,9 @@ def _add_prerequisites(self, point, tdef):
             if adjusted:
                 p_prev = max(adjusted)
                 cpre = Prerequisite(point)
-                cpre.add(tdef.name, p_prev, TASK_STATUS_SUCCEEDED,
-                         p_prev < tdef.start_point)
+                cpre[(p_prev, tdef.name, TASK_STATUS_SUCCEEDED)] = (
+                    p_prev < tdef.start_point
+                )
                 cpre.set_condition(tdef.name)
                 prerequisites[cpre.instantaneous_hash()] = cpre
 
@@ -497,13 +509,9 @@ def _add_xtriggers(self, point, tdef):
             for xtrig_label in xtrig_labels:
                 self.add_xtrigger(xtrig_label)
 
-    def get_unsatisfied_prerequisites(self):
-        unsat = []
-        for prereq in self.prerequisites:
-            if prereq.is_satisfied():
-                continue
-            for key, val in prereq.satisfied.items():
-                if val:
-                    continue
-                unsat.append(key)
-        return unsat
+    def get_unsatisfied_prerequisites(self) -> List['PrereqMessage']:
+        return [
+            key
+            for prereq in self.prerequisites if not prereq.is_satisfied()
+            for key, satisfied in prereq.items() if not satisfied
+        ]
diff --git a/cylc/flow/task_trigger.py b/cylc/flow/task_trigger.py
index d56c2a593b4..820b5eb8a18 100644
--- a/cylc/flow/task_trigger.py
+++ b/cylc/flow/task_trigger.py
@@ -14,11 +14,22 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from typing import TYPE_CHECKING, Tuple
+
 from cylc.flow.cycling.loader import (
-    get_point, get_point_relative, get_interval)
+    get_interval,
+    get_point,
+    get_point_relative,
+)
 from cylc.flow.prerequisite import Prerequisite
 from cylc.flow.task_qualifiers import ALT_QUALIFIERS
 
+
+if TYPE_CHECKING:
+    from cylc.flow.cycling import PointBase
+    from cylc.flow.taskdef import TaskDef
+
+
 # Task trigger names (e.g. foo:fail => bar).
 # Can use "foo:fail => bar" or "foo:failed => bar", etc.
 
@@ -176,23 +187,21 @@ class Dependency:
 
     def __init__(self, exp, task_triggers, suicide):
         self._exp = exp
-        self.task_triggers = tuple(task_triggers)  # More memory efficient.
+        self.task_triggers: Tuple[
+            TaskTrigger
+        ] = tuple(task_triggers)  # More memory efficient.
         self.suicide = suicide
 
-    def get_prerequisite(self, point, tdef):
+    def get_prerequisite(
+        self, point: 'PointBase', tdef: 'TaskDef'
+    ) -> Prerequisite:
         """Generate a Prerequisite object from this dependency.
 
         Args:
-            point (cylc.flow.cycling.PointBase): The cycle point at which to
-                generate the Prerequisite for.
-            tdef (cylc.flow.taskdef.TaskDef): The TaskDef of the dependent
-                task.
-
-        Returns:
-            cylc.flow.prerequisite.Prerequisite
+            point: The cycle point at which to generate the Prerequisite for.
+            tdef: The TaskDef of the dependent task.
 
         """
-        # Create Prerequisite.
         cpre = Prerequisite(point)
 
         # Loop over TaskTrigger instances.
@@ -213,21 +222,22 @@ def get_prerequisite(self, point, tdef):
                              tdef.max_future_prereq_offset)):
                         tdef.max_future_prereq_offset = (
                             prereq_offset)
-                cpre.add(
-                    task_trigger.task_name,
+                cpre[(
                     task_trigger.get_point(point),
-                    task_trigger.output,
-                    (
-                        (prereq_offset_point < tdef.start_point) &
-                        (point >= tdef.start_point)
-                    )
+                    task_trigger.task_name,
+                    task_trigger.output
+                )] = (
+                    (prereq_offset_point < tdef.start_point) &
+                    (point >= tdef.start_point)
                 )
             else:
                 # Trigger is within the same cycle point.
                 # Register task message with Prerequisite object.
-                cpre.add(task_trigger.task_name,
-                         task_trigger.get_point(point),
-                         task_trigger.output)
+                cpre[(
+                    task_trigger.get_point(point),
+                    task_trigger.task_name,
+                    task_trigger.output,
+                )] = False
         cpre.set_condition(self.get_expression(point))
         return cpre
 
diff --git a/cylc/flow/taskdef.py b/cylc/flow/taskdef.py
index 448844c8cc3..762324c9d75 100644
--- a/cylc/flow/taskdef.py
+++ b/cylc/flow/taskdef.py
@@ -17,20 +17,22 @@
 """Task definition."""
 
 from collections import deque
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Dict, List
 
 import cylc.flow.flags
 from cylc.flow.exceptions import TaskDefError
 from cylc.flow.task_id import TaskID
 from cylc.flow.task_outputs import (
+    SORT_ORDERS,
+    TASK_OUTPUT_FAILED,
     TASK_OUTPUT_SUBMITTED,
     TASK_OUTPUT_SUCCEEDED,
-    TASK_OUTPUT_FAILED,
-    SORT_ORDERS
 )
 
+
 if TYPE_CHECKING:
-    from cylc.flow.cycling import PointBase
+    from cylc.flow.cycling import PointBase, SequenceBase
+    from cylc.flow.task_trigger import Dependency
 
 
 def generate_graph_children(tdef, point):
@@ -145,7 +147,7 @@ def __init__(self, name, rtcfg, run_mode, start_point, initial_point):
         self.start_point = start_point
         self.initial_point = initial_point
 
-        self.sequences = []
+        self.sequences: List[SequenceBase] = []
         self.used_in_offset_trigger = False
 
         # some defaults
@@ -155,7 +157,7 @@ def __init__(self, name, rtcfg, run_mode, start_point, initial_point):
 
         self.expiration_offset = None
         self.namespace_hierarchy = []
-        self.dependencies = {}
+        self.dependencies: Dict[SequenceBase, List[Dependency]] = {}
         self.outputs = {}  # {output: (message, is_required)}
         self.graph_children = {}
         self.graph_parents = {}
diff --git a/cylc/flow/workflow_db_mgr.py b/cylc/flow/workflow_db_mgr.py
index c3f3ea9a263..d45108ca587 100644
--- a/cylc/flow/workflow_db_mgr.py
+++ b/cylc/flow/workflow_db_mgr.py
@@ -479,7 +479,7 @@ def put_task_pool(self, pool: 'TaskPool') -> None:
         for itask in pool.get_tasks():
             for prereq in itask.state.prerequisites:
                 for (p_cycle, p_name, p_output), satisfied_state in (
-                    prereq.satisfied.items()
+                    prereq.items()
                 ):
                     self.put_insert_task_prerequisites(itask, {
                         "flow_nums": serialise_set(itask.flow_nums),
diff --git a/tests/integration/scripts/test_set.py b/tests/integration/scripts/test_set.py
index 22cd44bbbce..af368aab0d5 100644
--- a/tests/integration/scripts/test_set.py
+++ b/tests/integration/scripts/test_set.py
@@ -20,8 +20,10 @@
 """
 
 from cylc.flow.cycling.integer import IntegerPoint
+from cylc.flow.data_messages_pb2 import PbTaskProxy
 from cylc.flow.data_store_mgr import TASK_PROXIES
-from cylc.flow.task_state import TASK_STATUS_WAITING, TASK_STATUS_SUCCEEDED
+from cylc.flow.scheduler import Scheduler
+from cylc.flow.task_state import TASK_STATUS_SUCCEEDED, TASK_STATUS_WAITING
 
 
 async def test_set_parentless_spawning(
@@ -106,11 +108,11 @@ async def test_data_store(
             'a': {'outputs': {'x': 'xyz'}},
         },
     })
-    schd = scheduler(id_)
+    schd: Scheduler = scheduler(id_)
     async with start(schd):
         await schd.update_data_structure()
         data = schd.data_store_mgr.data[schd.tokens.id]
-        task_a = data[TASK_PROXIES][
+        task_a: PbTaskProxy = data[TASK_PROXIES][
             schd.pool.get_task(IntegerPoint('1'), 'a').tokens.id
         ]
 
diff --git a/tests/integration/test_data_store_mgr.py b/tests/integration/test_data_store_mgr.py
index 906b1ac052d..e9c4e7616b6 100644
--- a/tests/integration/test_data_store_mgr.py
+++ b/tests/integration/test_data_store_mgr.py
@@ -14,18 +14,21 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from typing import Iterable, List, cast
+
 import pytest
-from typing import TYPE_CHECKING
 
+from cylc.flow.data_messages_pb2 import PbPrerequisite, PbTaskProxy
 from cylc.flow.data_store_mgr import (
     EDGES,
     FAMILY_PROXIES,
     JOBS,
-    TASKS,
     TASK_PROXIES,
-    WORKFLOW
+    TASKS,
+    WORKFLOW,
 )
 from cylc.flow.id import Tokens
+from cylc.flow.scheduler import Scheduler
 from cylc.flow.task_state import (
     TASK_STATUS_FAILED,
     TASK_STATUS_SUCCEEDED,
@@ -33,9 +36,6 @@
 )
 from cylc.flow.wallclock import get_current_time_string
 
-if TYPE_CHECKING:
-    from cylc.flow.scheduler import Scheduler
-
 
 # NOTE: These tests mutate the data store, so running them in isolation may
 # see failures when they actually pass if you run the whole file
@@ -88,6 +88,18 @@ def ext_id(schd):
     return f'~{schd.owner}/{schd.workflow}//{int_id(None)}'
 
 
+def get_pb_prereqs(schd: Scheduler) -> List[PbPrerequisite]:
+    """Get all protobuf prerequisites from the data store task proxies."""
+    return [
+        p
+        for t in cast(
+            'Iterable[PbTaskProxy]',
+            schd.data_store_mgr.updated[TASK_PROXIES].values()
+        )
+        for p in t.prerequisites
+    ]
+
+
 @pytest.fixture(scope='module')
 async def harness(mod_flow, mod_scheduler, mod_start):
     flow_def = {
@@ -296,6 +308,7 @@ async def test_update_data_structure(harness):
 
 def test_delta_task_prerequisite(harness):
     """Test delta_task_prerequisites."""
+    schd: Scheduler
     schd, data = harness
     schd.pool.set_prereqs_and_outputs(
         [t.identity for t in schd.pool.get_tasks()],
@@ -303,20 +316,15 @@ def test_delta_task_prerequisite(harness):
         [],
         "all"
     )
-    assert all({
-        p.satisfied
-        for t in schd.data_store_mgr.updated[TASK_PROXIES].values()
-        for p in t.prerequisites})
+    assert all(p.satisfied for p in get_pb_prereqs(schd))
     for itask in schd.pool.get_tasks():
         # set prereqs as not-satisfied
         for prereq in itask.state.prerequisites:
             prereq._all_satisfied = False
-            prereq.satisfied = {key: False for key in prereq.satisfied}
+            for key in prereq:
+                prereq._satisfied[key] = False
         schd.data_store_mgr.delta_task_prerequisite(itask)
-    assert not any({
-        p.satisfied
-        for t in schd.data_store_mgr.updated[TASK_PROXIES].values()
-        for p in t.prerequisites})
+    assert not any(p.satisfied for p in get_pb_prereqs(schd))
 
 
 async def test_absolute_graph_edges(flow, scheduler, start):
diff --git a/tests/integration/test_task_pool.py b/tests/integration/test_task_pool.py
index beba9075bd3..a38f33704ac 100644
--- a/tests/integration/test_task_pool.py
+++ b/tests/integration/test_task_pool.py
@@ -33,6 +33,7 @@
 from cylc.flow import commands
 from cylc.flow.cycling.integer import IntegerPoint
 from cylc.flow.cycling.iso8601 import ISO8601Point
+from cylc.flow.data_messages_pb2 import PbPrerequisite
 from cylc.flow.data_store_mgr import TASK_PROXIES
 from cylc.flow.task_events_mgr import TaskEventsManager
 from cylc.flow.task_outputs import (
@@ -699,7 +700,7 @@ async def test_restart_prereqs(
         }
     }
     id_ = flow(conf)
-    schd = scheduler(id_, run_mode='simulation', paused_start=False)
+    schd: Scheduler = scheduler(id_, paused_start=False)
 
     async with start(schd):
         # Release tasks 1/a and 1/b
@@ -737,7 +738,7 @@ async def test_restart_prereqs(
         ][0]
         assert sorted(
             (
-                p.satisfied
+                p._satisfied
                 for p in task_z.state.prerequisites
             ),
             key=lambda d: tuple(d.keys())[0],
@@ -822,7 +823,7 @@ async def test_reload_prereqs(
         }
     }
     id_ = flow(conf)
-    schd = scheduler(id_, run_mode='simulation', paused_start=False)
+    schd: Scheduler = scheduler(id_, paused_start=False)
 
     async with start(schd):
         # Release tasks 1/a and 1/b
@@ -849,7 +850,7 @@ async def test_reload_prereqs(
         ][0]
         assert sorted(
             (
-                p.satisfied
+                p._satisfied
                 for p in task_z.state.prerequisites
             ),
             key=lambda d: tuple(d.keys())[0],
@@ -857,6 +858,7 @@ async def test_reload_prereqs(
 
 
 async def _test_restart_prereqs_sat():
+    schd: Scheduler
     # YIELD: the workflow has now started...
     schd = yield
     await schd.update_data_structure()
@@ -891,7 +893,7 @@ async def _test_restart_prereqs_sat():
     assert sorted(
         (*key, satisfied)
         for prereq in task_c.state.prerequisites
-        for key, satisfied in prereq.satisfied.items()
+        for key, satisfied in prereq.items()
     ) == [
         ('1', 'a', 'succeeded', 'satisfied naturally'),
         ('1', 'b', 'succeeded', 'satisfied from database')
@@ -902,7 +904,7 @@ async def _test_restart_prereqs_sat():
     tasks = (
         schd.data_store_mgr.data[schd.data_store_mgr.workflow_id][TASK_PROXIES]
     )
-    task_c_prereqs = tasks[
+    task_c_prereqs: List[PbPrerequisite] = tasks[
         schd.data_store_mgr.id_.duplicate(cycle='1', task='c').id
     ].prerequisites
     assert sorted(
@@ -1414,7 +1416,7 @@ def set_prereqs(prereqs):
     async with start(schd) as log:
         # Invalid: task name wildcard:
         set_prereqs(["2040/*"])
-        assert log_filter(log, contains='Invalid prerequisite task name' )
+        assert log_filter(log, contains='Invalid prerequisite task name')
 
         # Invalid: cycle point wildcard.
         set_prereqs(["*/foo"])
@@ -2085,7 +2087,7 @@ async def test_set_future_flow(flow, scheduler, start, log_filter):
         # set b:succeeded in flow 2 and check downstream spawning
         schd.pool.set_prereqs_and_outputs(['1/b'], prereqs=[], outputs=[], flow=[2])
         assert schd.pool.get_task(IntegerPoint("1"), "c1") is None, '1/c1 (flow 2) should not be spawned after 1/b:succeeded'
-        assert schd.pool.get_task(IntegerPoint("1"), "c2") is not None, '1/c2 (flow 2) should be spawned after 1/b:succeeded' 
+        assert schd.pool.get_task(IntegerPoint("1"), "c2") is not None, '1/c2 (flow 2) should be spawned after 1/b:succeeded'
 
 
 async def test_trigger_queue(one, run, db_select, complete):
diff --git a/tests/unit/test_prerequisite.py b/tests/unit/test_prerequisite.py
index 1514ad07f66..105e5e85401 100644
--- a/tests/unit/test_prerequisite.py
+++ b/tests/unit/test_prerequisite.py
@@ -26,35 +26,15 @@
 def prereq(set_cycling_type):
     set_cycling_type(ISO8601_CYCLING_TYPE, "Z")
     prereq = Prerequisite(get_point('2000'))
-    prereq.add(
-        'a',
-        '1999',
-        'succeeded',
-        True
-    )
-    prereq.add(
-        'b',
-        '2000',
-        'succeeded',
-        False
-    )
-    prereq.add(
-        'c',
-        '2000',
-        'succeeded',
-        False
-    )
-    prereq.add(
-        'd',
-        '2001',
-        'custom',
-        False
-    )
+    prereq[(1999, 'a', 'succeeded')] = True
+    prereq[(2000, 'b', 'succeeded')] = False
+    prereq[(2000, 'c', 'succeeded')] = False
+    prereq[(2001, 'd', 'custom')] = False
     return prereq
 
 
-def test_satisfied(prereq):
-    assert prereq.satisfied == {
+def test_satisfied(prereq: Prerequisite):
+    assert prereq._satisfied == {
         # the pre-initial dependency should be marked as satisfied
         ('1999', 'a', 'succeeded'): 'satisfied naturally',
         # all others should not
@@ -62,12 +42,18 @@ def test_satisfied(prereq):
         ('2000', 'c', 'succeeded'): False,
         ('2001', 'd', 'custom'): False,
     }
+    # No cached satisfaction state yet:
+    assert prereq._all_satisfied is None
+    # Calling self.is_satisfied() should cache the result:
+    assert not prereq.is_satisfied()
+    assert prereq._all_satisfied is False
+
     # mark two prerequisites as satisfied
     prereq.satisfy_me([
         Tokens('2000/b:succeeded', relative=True),
         Tokens('2000/c:succeeded', relative=True),
     ])
-    assert prereq.satisfied == {
+    assert prereq._satisfied == {
         # the pre-initial dependency should be marked as satisfied
         ('1999', 'a', 'succeeded'): 'satisfied naturally',
         # the two newly-satisfied dependency should be satisfied
@@ -76,9 +62,13 @@ def test_satisfied(prereq):
         # the remaining dependency should not
         ('2001', 'd', 'custom'): False,
     }
+    # Should have reset cached satisfaction state:
+    assert prereq._all_satisfied is None
+    assert not prereq.is_satisfied()
+
     # mark all prereqs as satisfied
     prereq.set_satisfied()
-    assert prereq.satisfied == {
+    assert prereq._satisfied == {
         # the pre-initial dependency should be marked as satisfied
         ('1999', 'a', 'succeeded'): 'satisfied naturally',
         # the two newly-satisfied dependency should be satisfied
@@ -87,6 +77,49 @@ def test_satisfied(prereq):
         # the remaining dependency should be marked as forse-satisfied
         ('2001', 'd', 'custom'): 'force satisfied',
     }
+    # Should have set cached satisfaction state as must be true now:
+    assert prereq._all_satisfied is True
+    assert prereq.is_satisfied()
+
+
+def test_getitem_setitem(prereq: Prerequisite):
+    msg = ('2000', 'b', 'succeeded')
+    # __getitem__:
+    assert prereq[msg] is False
+
+    # __setitem__:
+    prereq[msg] = True
+    assert prereq[msg] == 'satisfied naturally'
+    prereq[msg] = 'force satisfied'
+    assert prereq[msg] == 'force satisfied'
+    # coercion of cycle point
+    assert prereq[(2000, 'b', 'succeeded')] == 'force satisfied'
+    assert prereq[(get_point('2000'), 'b', 'succeeded')] == 'force satisfied'
+
+
+def test_iter(prereq: Prerequisite):
+    assert list(prereq) == [
+        ('1999', 'a', 'succeeded'),
+        ('2000', 'b', 'succeeded'),
+        ('2000', 'c', 'succeeded'),
+        ('2001', 'd', 'custom'),
+    ]
+    assert [p.task for p in prereq] == ['a', 'b', 'c', 'd']
+
+
+def test_items(prereq: Prerequisite):
+    assert list(prereq.items()) == [
+        (('1999', 'a', 'succeeded'), 'satisfied naturally'),
+        (('2000', 'b', 'succeeded'), False),
+        (('2000', 'c', 'succeeded'), False),
+        (('2001', 'd', 'custom'), False),
+    ]
+
+
+def test_set_condition(prereq: Prerequisite):
+    assert not prereq.is_satisfied()
+    prereq.set_condition('1999/a succeeded | 2000/b succeeded')
+    assert prereq.is_satisfied()
 
 
 def test_iter_target_point_strings(prereq):
@@ -107,10 +140,10 @@ def test_get_target_points(prereq):
 
 def test_get_resolved_dependencies():
     prereq = Prerequisite(IntegerPoint('2'))
-    prereq.satisfied[('1', 'a', 'x')] = 'satisfied naturally'
-    prereq.satisfied[('1', 'b', 'x')] = False
-    prereq.satisfied[('1', 'c', 'x')] = 'satisfied from database'
-    prereq.satisfied[('1', 'd', 'x')] = 'force satisfied'
+    prereq[('1', 'a', 'x')] = True
+    prereq[('1', 'b', 'x')] = False
+    prereq[('1', 'c', 'x')] = 'satisfied from database'
+    prereq[('1', 'd', 'x')] = 'force satisfied'
     assert prereq.get_resolved_dependencies() == [
         '1/a',
         '1/c',
diff --git a/tests/unit/test_task_state.py b/tests/unit/test_task_state.py
index e655c74b7bb..4b5d796a043 100644
--- a/tests/unit/test_task_state.py
+++ b/tests/unit/test_task_state.py
@@ -100,7 +100,7 @@ def test_task_prereq_duplicates(set_cycling_type):
 
     tstate = TaskState(tdef, IntegerPoint("1"), TASK_STATUS_WAITING, False)
 
-    prereqs = [p.satisfied for p in tstate.prerequisites]
+    prereqs = [p._satisfied for p in tstate.prerequisites]
 
     assert prereqs == [{("1", "a", "succeeded"): False}]
 
@@ -118,4 +118,3 @@ def test_task_state_order():
     assert tstate.is_gte(TASK_STATUS_SUBMITTED)
     assert not tstate.is_gt(TASK_STATUS_RUNNING)
     assert not tstate.is_gte(TASK_STATUS_RUNNING)
-

From 8a4366850b467efb4d5529ca9072a6cb645a25f7 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Thu, 8 Aug 2024 12:29:04 +0100
Subject: [PATCH 21/53] Fix potential type error

---
 cylc/flow/task_pool.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py
index 2f953c9812d..aeee7505e66 100644
--- a/cylc/flow/task_pool.py
+++ b/cylc/flow/task_pool.py
@@ -1209,16 +1209,14 @@ def log_unsatisfied_prereqs(self) -> bool:
             task_point = itask.point
             if self.stop_point and task_point > self.stop_point:
                 continue
-            for point, task, msg in (
-                itask.state.get_unsatisfied_prerequisites()
-            ):
-                if get_point(point) > self.stop_point:
+            for pr in itask.state.get_unsatisfied_prerequisites():
+                if self.stop_point and get_point(pr.point) > self.stop_point:
                     continue
                 if itask.identity not in unsat:
                     unsat[itask.identity] = []
                 unsat[itask.identity].append(
-                    f"{point}/{task}:"
-                    f"{self.config.get_taskdef(task).get_output(msg)}"
+                    f"{pr.get_id()}:"
+                    f"{self.config.get_taskdef(pr.task).get_output(pr.output)}"
                 )
         if unsat:
             LOG.warning(

From 34c9fa5a57adae74dbdcd60d3416f3f32988db81 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 1 Aug 2024 13:20:15 +0100
Subject: [PATCH 22/53] examples: extending workflow

* Fix incorrect heading level
* Add a note about scheduling tasks to run at the
  "stop after cycle point".
---
 .../etc/examples/extending-workflow/index.rst | 32 +++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/cylc/flow/etc/examples/extending-workflow/index.rst b/cylc/flow/etc/examples/extending-workflow/index.rst
index c5bc023dc11..036438ff466 100644
--- a/cylc/flow/etc/examples/extending-workflow/index.rst
+++ b/cylc/flow/etc/examples/extending-workflow/index.rst
@@ -1,5 +1,5 @@
 Extending Workflow
-==================
+------------------
 
 .. cylc-scope:: flow.cylc[scheduling]
 
@@ -24,7 +24,7 @@ solution avoids this issue.
 
 
 Example
--------
+^^^^^^^
 
 .. admonition:: Get a copy of this example
    :class: hint
@@ -74,4 +74,32 @@ Or change it at any point whilst the workflow is running:
    ``cylc play --stop-cycle-point=reload`` to restart the workflow using the
    `stop after cycle point` configured in the workflow configuration.
 
+
+Running Tasks At The `stop after cycle point`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you have tasks that you want to run before the workflow shuts down at the
+`stop after cycle point`, use the recurrence ``R1/<cycle-point>`` to schedudule
+them, e.g:
+
+.. code-block:: cylc
+
+   #!Jinja2
+
+   {% set stop_cycle = '3000' %}
+
+   [scheduling]
+      initial cycle point = 2000
+      stop after cycle point = {{ stop_cycle }}
+      [[graph]]
+         R1/{{ stop_cycle }} = """
+            # these tasks will run *before* the workflow shuts down
+            z => run_me => and_me
+         """
+
+When the workflow is subsequently restarted with a later
+`stop after cycle point`, these tasks will be re-scheduled at the new
+stop point.
+
+
 .. cylc-scope::

From aa4835621c5ae305a27f18ff3bc95166f86a7882 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 13 Aug 2024 13:51:35 +0100
Subject: [PATCH 23/53] schema: minor docs changes

* Consistent string quotation.
* Consistent termination punctuation.
* Correct some ReST literals (double backquotes).
* Correct one incorrectly closed markdown literal.
---
 cylc/flow/network/schema.py | 139 +++++++++++++++++++-----------------
 1 file changed, 72 insertions(+), 67 deletions(-)

diff --git a/cylc/flow/network/schema.py b/cylc/flow/network/schema.py
index b962b582b70..8e20225d4b6 100644
--- a/cylc/flow/network/schema.py
+++ b/cylc/flow/network/schema.py
@@ -577,9 +577,10 @@ def resolve_mapping_to_list(root, info, **args):
 # Types:
 class NodeMeta(ObjectType):
     class Meta:
-        description = """
-Meta data fields,
-including custom fields in a generic user-defined dump"""
+        description = sstrip("""
+            Meta data fields, including custom fields in a generic user-defined
+            dump.
+        """)
     title = String(default_value=None)
     description = String(default_value=None)
     URL = String(default_value=None)
@@ -588,7 +589,7 @@ class Meta:
 
 class TimeZone(ObjectType):
     class Meta:
-        description = """Time zone info."""
+        description = 'Time zone info.'
     hours = Int()
     minutes = Int()
     string_basic = String()
@@ -597,7 +598,7 @@ class Meta:
 
 class Workflow(ObjectType):
     class Meta:
-        description = """Global workflow info."""
+        description = 'Global workflow info.'
     id = ID()  # noqa: A003 (required for definition)
     name = String(
         description='The workflow ID with the ~user/ prefix removed.',
@@ -730,7 +731,7 @@ class Meta:
     latest_state_tasks = GenericScalar(
         states=graphene.List(
             String,
-            description="List of task states to show",
+            description="List of task states to show.",
             default_value=TASK_STATUSES_ORDERED),
         resolver=resolve_state_tasks,
         description='The latest tasks to have entered each task state.',
@@ -769,8 +770,9 @@ class Meta:
         ids=graphene.List(
             ID,
             description=sstrip('''
-                Node IDs, cycle point and/or-just family/task namespace:
-                    ["1234/foo", "1234/FAM", "*/FAM"]
+                Node IDs, cycle point and/or-just family/task namespace.
+
+                E.g: `["1234/foo", "1234/FAM", "*/FAM"]`
             '''),
             default_value=[]
         ),
@@ -858,7 +860,7 @@ class Meta:
     )
     task_proxy = Field(
         lambda: TaskProxy,
-        description="The TaskProxy of the task which submitted this job",
+        description="The TaskProxy of the task which submitted this job.",
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
         delta_type=DELTA_TYPE_DEFAULT,
@@ -919,7 +921,7 @@ class Meta:
     )
     runtime = Field(
         Runtime,
-        description="The task's `[runtime`] section.",
+        description="The task's `[runtime]` section.",
     )
     mean_elapsed_time = Float(
         description="The task's average runtime."
@@ -955,7 +957,7 @@ class Meta:
 
 class PollTask(ObjectType):
     class Meta:
-        description = """Polling task edge"""
+        description = 'Polling task edge.'
     local_proxy = ID()
     workflow = String()
     remote_proxy = ID()
@@ -965,11 +967,11 @@ class Meta:
 
 class Condition(ObjectType):
     class Meta:
-        description = """Prerequisite conditions."""
+        description = 'Prerequisite conditions.'
     task_id = String()
     task_proxy = Field(
         lambda: TaskProxy,
-        description="""Associated Task Proxy""",
+        description='Associated Task Proxy.',
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
         delta_type=DELTA_TYPE_DEFAULT,
@@ -985,18 +987,19 @@ def resolve_task_id(root, info, **args):
 
 class Prerequisite(ObjectType):
     class Meta:
-        description = """Task prerequisite."""
+        description = 'Task prerequisite.'
     expression = String()
     conditions = graphene.List(
         Condition,
-        description="""Condition monomers of a task prerequisites.""")
+        description='Condition monomers of a task prerequisites.'
+    )
     cycle_points = graphene.List(String)
     satisfied = Boolean()
 
 
 class Output(ObjectType):
     class Meta:
-        description = """Task output"""
+        description = 'Task output.'
     label = String()
     message = String()
     satisfied = Boolean()
@@ -1004,16 +1007,16 @@ class Meta:
 
 
 class OutputLabel(String):
-    """Task output, e.g. "succeeded"."""
+    """Task output, e.g. `succeeded`."""
 
 
 class PrerequisiteString(String):
-    """A task prerequisite, e.g. "2040/foo:succeeded"."""
+    """A task prerequisite, e.g. `2040/foo:succeeded`."""
 
 
 class XTrigger(ObjectType):
     class Meta:
-        description = """Task trigger"""
+        description = 'Task trigger.'
     id = String()  # noqa: A003 (required for definition)
     label = String()
     message = String()
@@ -1023,7 +1026,7 @@ class Meta:
 
 class TaskProxy(ObjectType):
     class Meta:
-        description = """Task cycle instance."""
+        description = 'Task cycle instance.'
     id = ID()  # noqa: A003 (required for schema definition)
     task = Field(
         Task,
@@ -1073,7 +1076,7 @@ class Meta:
         '''),
     )
     depth = Int(
-        description='The family inheritance depth',
+        description='The family inheritance depth.',
     )
     graph_depth = Int(
         description=sstrip('''
@@ -1158,7 +1161,7 @@ class Meta:
 
 class Family(ObjectType):
     class Meta:
-        description = """Task definition, static fields"""
+        description = 'Task definition, static fields.'
     id = ID()  # noqa: A003 (required for schema definition)
     name = String()
     meta = Field(NodeMeta)
@@ -1166,7 +1169,7 @@ class Meta:
     depth = Int()
     proxies = graphene.List(
         lambda: FamilyProxy,
-        description="""Associated cycle point proxies""",
+        description='Associated cycle point proxies.',
         args=PROXY_ARGS,
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
@@ -1174,7 +1177,7 @@ class Meta:
         resolver=get_nodes_by_ids)
     parents = graphene.List(
         lambda: Family,
-        description="""Family definition parent.""",
+        description='Family definition parent.',
         args=DEF_ARGS,
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
@@ -1182,7 +1185,7 @@ class Meta:
         resolver=get_nodes_by_ids)
     child_tasks = graphene.List(
         Task,
-        description="""Descendant definition tasks.""",
+        description='Descendant definition tasks.',
         args=DEF_ARGS,
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
@@ -1190,7 +1193,7 @@ class Meta:
         resolver=get_nodes_by_ids)
     child_families = graphene.List(
         lambda: Family,
-        description="""Descendant desc families.""",
+        description='Descendant desc families.',
         args=DEF_ARGS,
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
@@ -1198,7 +1201,7 @@ class Meta:
         resolver=get_nodes_by_ids)
     first_parent = Field(
         lambda: Family,
-        description="""Family first parent.""",
+        description='Family first parent.',
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
         delta_type=DELTA_TYPE_DEFAULT,
@@ -1207,14 +1210,14 @@ class Meta:
 
 class FamilyProxy(ObjectType):
     class Meta:
-        description = """Family composite."""
+        description = 'Family composite.'
     id = ID()  # noqa: A003 (required for schema definition)
     cycle_point = String()
     # name & namespace for filtering/sorting
     name = String()
     family = Field(
         Family,
-        description="""Family definition""",
+        description='Family definition.',
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
         delta_type=DELTA_TYPE_DEFAULT,
@@ -1238,7 +1241,7 @@ class Meta:
     )
     child_tasks = graphene.List(
         TaskProxy,
-        description="""Descendant task proxies.""",
+        description='Descendant task proxies.',
         args=PROXY_ARGS,
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
@@ -1246,7 +1249,7 @@ class Meta:
         resolver=get_nodes_by_ids)
     child_families = graphene.List(
         lambda: FamilyProxy,
-        description="""Descendant family proxies.""",
+        description='Descendant family proxies.',
         args=PROXY_ARGS,
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
@@ -1254,7 +1257,7 @@ class Meta:
         resolver=get_nodes_by_ids)
     first_parent = Field(
         lambda: FamilyProxy,
-        description="""Task first parent.""",
+        description='Task first parent.',
         args=PROXY_ARGS,
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
@@ -1262,7 +1265,7 @@ class Meta:
         resolver=get_node_by_id)
     ancestors = graphene.List(
         lambda: FamilyProxy,
-        description="""First parent ancestors.""",
+        description='First parent ancestors.',
         args=PROXY_ARGS,
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
@@ -1283,7 +1286,7 @@ def resolve_type(cls, instance, info):
 
 class Edge(ObjectType):
     class Meta:
-        description = """Dependency edge task/family proxies"""
+        description = 'Dependency edge task/family proxies.'
     id = ID()  # noqa: A003 (required for schema definition)
     source = ID()
     source_node = Field(
@@ -1305,7 +1308,7 @@ class Meta:
 
 class Edges(ObjectType):
     class Meta:
-        description = """Dependency edge"""
+        description = 'Dependency edge.'
     edges = graphene.List(
         Edge,
         args=EDGE_ARGS,
@@ -1320,19 +1323,19 @@ class Meta:
 
 class NodesEdges(ObjectType):
     class Meta:
-        description = """Related Nodes & Edges."""
+        description = 'Related Nodes & Edges.'
     nodes = graphene.List(
         TaskProxy,
-        description="""Task nodes from and including root.""")
+        description='Task nodes from and including root.')
     edges = graphene.List(
         Edge,
-        description="""Edges associated with the nodes.""")
+        description='Edges associated with the nodes.')
 
 
 # Query declaration
 class Queries(ObjectType):
     class Meta:
-        description = """Multi-Workflow root level queries."""
+        description = 'Multi-Workflow root level queries.'
     workflows = graphene.List(
         Workflow,
         description=Workflow._meta.description,
@@ -1420,7 +1423,7 @@ class Meta:
 # Generic containers
 class GenericResponse(ObjectType):
     class Meta:
-        description = """Container for command queued response"""
+        description = 'Container for command queued response.'
 
     result = GenericScalar()
 
@@ -1455,6 +1458,7 @@ async def mutator(
             method). If None, uses mutation class name converted to snake_case.
         workflows: List of workflow IDs.
         exworkflows: List of workflow IDs.
+
     """
     if command is None:
         command = to_snake_case(info.field_name)
@@ -1567,13 +1571,13 @@ class TaskState(InputObjectType):
 
     status = TaskStatus()
     is_held = Boolean(description=sstrip('''
-        If a task is held no new job submissions will be made
+        If a task is held no new job submissions will be made.
     '''))
     is_queued = Boolean(description=sstrip('''
-        Task is queued for job submission
+        Task is queued for job submission.
     '''))
     is_runahead = Boolean(description=sstrip('''
-        Task is runahead limited
+        Task is runahead limited.
     '''))
 
 
@@ -1792,7 +1796,7 @@ class Arguments:
         event_time = String(default_value=None)
         messages = graphene.List(
             graphene.List(String),
-            description="""List in the form `[[severity, message], ...]`.""",
+            description='List in the form `[[severity, message], ...]`.',
             default_value=None
         )
 
@@ -1820,6 +1824,7 @@ class Resume(Mutation):
     class Meta:
         description = sstrip('''
             Resume a paused workflow.
+
             See also the opposite command `pause`.
 
             Valid for: paused workflows.
@@ -1952,8 +1957,8 @@ class Meta:
             trigger event. When an incoming message satisfies
             a task's external trigger the message ID is broadcast
             to all downstream tasks in the cycle point as
-            ``$CYLC_EXT_TRIGGER_ID``.  Tasks can use
-            ``$CYLC_EXT_TRIGGER_ID``, for example,  to
+            `$CYLC_EXT_TRIGGER_ID`.  Tasks can use
+            `$CYLC_EXT_TRIGGER_ID`, for example,  to
             identify a new data file that the external
             triggering system is responding to.
 
@@ -2113,9 +2118,9 @@ class Meta:
             Setting outputs contributes to the task's completion, sets the
             corresponding prerequisites of child tasks, and sets any implied
             outputs:
-             - ``started`` implies ``submitted``.
-             - ``succeeded`` and ``failed`` imply ``started``.
-             - custom outputs and ``expired`` do not imply any other outputs.
+             - `started` implies `submitted`.
+             - `succeeded` and `failed` imply `started`.
+             - custom outputs and `expired` do not imply any other outputs.
 
             Valid for: paused, running, stopping workflows.
         """)
@@ -2224,7 +2229,7 @@ def delta_subs(root, info: 'ResolveInfo', **args) -> AsyncGenerator[Any, None]:
 
 class Pruned(ObjectType):
     class Meta:
-        description = """WFS Nodes/Edges that have been removed."""
+        description = 'WFS Nodes/Edges that have been removed.'
     workflow = String()
     families = graphene.List(String, default_value=[])
     family_proxies = graphene.List(String, default_value=[])
@@ -2246,7 +2251,7 @@ class Delta(Interface):
 
     families = graphene.List(
         Family,
-        description="""Family definitions.""",
+        description='Family definitions.',
         args=DEF_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2255,7 +2260,7 @@ class Delta(Interface):
     )
     family_proxies = graphene.List(
         FamilyProxy,
-        description="""Family cycle instances.""",
+        description='Family cycle instances.',
         args=PROXY_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2264,7 +2269,7 @@ class Delta(Interface):
     )
     jobs = graphene.List(
         Job,
-        description="""Jobs.""",
+        description='Jobs.',
         args=JOB_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2273,7 +2278,7 @@ class Delta(Interface):
     )
     tasks = graphene.List(
         Task,
-        description="""Task definitions.""",
+        description='Task definitions.',
         args=DEF_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2282,7 +2287,7 @@ class Delta(Interface):
     )
     task_proxies = graphene.List(
         TaskProxy,
-        description="""Task cycle instances.""",
+        description='Task cycle instances.',
         args=PROXY_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2291,7 +2296,7 @@ class Delta(Interface):
     )
     edges = graphene.List(
         Edge,
-        description="""Graph edges""",
+        description='Graph edges',
         args=EDGE_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2310,18 +2315,18 @@ class Delta(Interface):
 
 class Added(ObjectType):
     class Meta:
-        description = """Added node/edge deltas."""
+        description = 'Added node/edge deltas.'
         interfaces = (Delta,)
 
 
 class Updated(ObjectType):
     class Meta:
-        description = """Updated node/edge deltas."""
+        description = 'Updated node/edge deltas.'
         interfaces = (Delta,)
 
     families = graphene.List(
         Family,
-        description="""Family definitions.""",
+        description='Family definitions.',
         args=DEF_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2330,7 +2335,7 @@ class Meta:
     )
     family_proxies = graphene.List(
         FamilyProxy,
-        description="""Family cycle instances.""",
+        description='Family cycle instances.',
         args=PROXY_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2339,7 +2344,7 @@ class Meta:
     )
     jobs = graphene.List(
         Job,
-        description="""Jobs.""",
+        description='Jobs.',
         args=JOB_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2348,7 +2353,7 @@ class Meta:
     )
     tasks = graphene.List(
         Task,
-        description="""Task definitions.""",
+        description='Task definitions.',
         args=DEF_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2357,7 +2362,7 @@ class Meta:
     )
     task_proxies = graphene.List(
         TaskProxy,
-        description="""Task cycle instances.""",
+        description='Task cycle instances.',
         args=PROXY_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2366,7 +2371,7 @@ class Meta:
     )
     edges = graphene.List(
         Edge,
-        description="""Graph edges""",
+        description='Graph edges.',
         args=EDGE_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2385,7 +2390,7 @@ class Meta:
 
 class Deltas(ObjectType):
     class Meta:
-        description = """Grouped deltas of the WFS publish"""
+        description = 'Grouped deltas of the WFS publish.'
     id = ID()  # noqa: A003 (required for schema definition)
     shutdown = Boolean(default_value=False)
     added = Field(
@@ -2408,13 +2413,13 @@ class Meta:
 class Subscriptions(ObjectType):
     """Defines the subscriptions available in the schema."""
     class Meta:
-        description = """Multi-Workflow root level subscriptions."""
+        description = 'Multi-Workflow root level subscriptions.'
 
     deltas = Field(
         Deltas,
         description=Deltas._meta.description,
         workflows=graphene.List(
-            ID, description="List of full ID, i.e. `~user/workflow_id`"
+            ID, description="List of full ID, i.e. `~user/workflow_id`."
         ),
         strip_null=Boolean(default_value=False),
         initial_burst=Boolean(default_value=True),

From 9c88c5ccd09b9b9e20d263de7d1e9e03799b646d Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 6 Aug 2024 17:18:23 +0100
Subject: [PATCH 24/53] jinja2: removed excessive context from Jinja2
 raise/assert statements

---
 changes.d/6289.feat.md                        |  1 +
 cylc/flow/parsec/jinja2support.py             | 26 ++++++-
 tests/functional/jinja2/03-bad.t              | 23 +-----
 .../functional/jinja2/10-builtin-functions.t  | 38 ----------
 .../jinja2/10-builtin-functions/flow.cylc     | 13 ----
 .../jinja2/include-exception/flow.cylc        |  4 -
 .../jinja2/include-exception/foo.cylc         |  3 -
 tests/integration/validate/test_jinja2.py     | 76 +++++++++++++++++++
 tests/unit/parsec/test_jinja2support.py       |  9 +--
 9 files changed, 102 insertions(+), 91 deletions(-)
 create mode 100644 changes.d/6289.feat.md
 delete mode 100644 tests/functional/jinja2/10-builtin-functions.t
 delete mode 100644 tests/functional/jinja2/10-builtin-functions/flow.cylc
 delete mode 100644 tests/functional/jinja2/include-exception/flow.cylc
 delete mode 100644 tests/functional/jinja2/include-exception/foo.cylc
 create mode 100644 tests/integration/validate/test_jinja2.py

diff --git a/changes.d/6289.feat.md b/changes.d/6289.feat.md
new file mode 100644
index 00000000000..9284a768f2f
--- /dev/null
+++ b/changes.d/6289.feat.md
@@ -0,0 +1 @@
+Made the errors resulting from Jinja2 `raise` and `assert` statements more straight forward.
diff --git a/cylc/flow/parsec/jinja2support.py b/cylc/flow/parsec/jinja2support.py
index 74335358421..be6b99eb19b 100644
--- a/cylc/flow/parsec/jinja2support.py
+++ b/cylc/flow/parsec/jinja2support.py
@@ -38,6 +38,8 @@
     TemplateSyntaxError)
 
 from cylc.flow import LOG
+from cylc.flow.exceptions import InputError
+import cylc.flow.flags
 from cylc.flow.parsec.exceptions import Jinja2Error
 from cylc.flow.parsec.fileparse import get_cylc_env_vars
 
@@ -91,16 +93,19 @@ def root_render_func(context, *args, **kwargs):
         return templ
 
 
-def raise_helper(message, error_type='Error'):
+class Jinja2AssertionError(Exception):
+    """Exception raised by the Jinja2 "raise" and "assert" functions."""
+
+
+def raise_helper(message):
     """Provides a Jinja2 function for raising exceptions."""
-    # TODO - this more nicely
-    raise Exception('Jinja2 %s: %s' % (error_type, message))
+    raise Jinja2AssertionError(message)
 
 
 def assert_helper(logical, message):
     """Provides a Jinja2 function for asserting logical expressions."""
     if not logical:
-        raise_helper(message, 'Assertion Error')
+        raise_helper(message)
     return ''  # Prevent None return value polluting output.
 
 
@@ -298,6 +303,19 @@ def jinja2process(
             lines=get_error_lines(fpath, flines),
             filename=filename
         ) from None
+    except Jinja2AssertionError as exc:
+        if cylc.flow.flags.verbosity < 1:
+            # raise a user-friently representation of this assertion error
+            raise InputError(
+                f'{str(exc)}'
+                '\n(add --verbose for more context)'
+            ) from None
+
+        # raise the full form of the error in verbose mode
+        raise Jinja2Error(
+            exc,
+            lines=get_error_lines(fpath, flines),
+        ) from None
     except Exception as exc:
         raise Jinja2Error(
             exc,
diff --git a/tests/functional/jinja2/03-bad.t b/tests/functional/jinja2/03-bad.t
index 5741932e8dc..8d6688183c7 100644
--- a/tests/functional/jinja2/03-bad.t
+++ b/tests/functional/jinja2/03-bad.t
@@ -20,7 +20,7 @@
 # an include statement
 
 . "$(dirname "$0")/test_header"
-set_test_number 12
+set_test_number 10
 
 sub_tabs() {
     sed -i 's/\t/ /g' "${TEST_NAME}.stderr"
@@ -51,27 +51,6 @@ __HERE__
 
 purge_workflow
 
-#------------------------------------------------------------------------------
-# Test generic exception in an include file
-install_workflow "${TEST_NAME_BASE}" include-exception
-
-TEST_NAME="${TEST_NAME_BASE}-include-exception"
-run_fail "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}"
-
-sub_tabs
-cmp_ok "${TEST_NAME}.stderr" <<__HERE__
-Jinja2Error: Jinja2 Error: some error
-File ${WORKFLOW_RUN_DIR}/foo.cylc
-  # line before error
-  {{ raise('some error') }} <-- Exception
-File ${WORKFLOW_RUN_DIR}/flow.cylc
-  #!Jinja2
-  # line before
-  {% include "foo.cylc" %} <-- Exception
-__HERE__
-
-purge_workflow
-
 #------------------------------------------------------------------------------
 # Test syntax error in an include file
 install_workflow "${TEST_NAME_BASE}" include-badsyntax
diff --git a/tests/functional/jinja2/10-builtin-functions.t b/tests/functional/jinja2/10-builtin-functions.t
deleted file mode 100644
index c9016f795f6..00000000000
--- a/tests/functional/jinja2/10-builtin-functions.t
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-# 
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#-------------------------------------------------------------------------------
-# jinja2 test cylc-provided functions.
-. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
-set_test_number 5
-#-------------------------------------------------------------------------------
-install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}"-pass
-run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" \
-    -s 'FOO="True"' \
-    -s 'ANSWER="42"'
-TEST_NAME="${TEST_NAME_BASE}"-fail-assert
-run_fail "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" \
-    -s 'FOO="True"' \
-    -s 'ANSWER="43"'
-grep_ok 'Jinja2 Assertion Error: Universal' "${TEST_NAME}.stderr"
-TEST_NAME="${TEST_NAME_BASE}"-fail-raise
-run_fail "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" -s 'ANSWER="42"'
-grep_ok 'Jinja2 Error: FOO must be defined' "${TEST_NAME}.stderr"
-#-------------------------------------------------------------------------------
-purge
diff --git a/tests/functional/jinja2/10-builtin-functions/flow.cylc b/tests/functional/jinja2/10-builtin-functions/flow.cylc
deleted file mode 100644
index 704704f438c..00000000000
--- a/tests/functional/jinja2/10-builtin-functions/flow.cylc
+++ /dev/null
@@ -1,13 +0,0 @@
-#!Jinja2
-
-{% if not FOO is defined %}
-    {{ raise('FOO must be defined.') }}
-{% endif %}
-
-{{ assert(ANSWER | int == 42, 'Universal constant incorrectly set.') }}
-
-[scheduling]
-    [[graph]]
-        R1 = foo
-[runtime]
-    [[foo]]
diff --git a/tests/functional/jinja2/include-exception/flow.cylc b/tests/functional/jinja2/include-exception/flow.cylc
deleted file mode 100644
index 8486a2ebb9c..00000000000
--- a/tests/functional/jinja2/include-exception/flow.cylc
+++ /dev/null
@@ -1,4 +0,0 @@
-#!Jinja2
-# line before
-{% include "foo.cylc" %}
-# line after
diff --git a/tests/functional/jinja2/include-exception/foo.cylc b/tests/functional/jinja2/include-exception/foo.cylc
deleted file mode 100644
index 77d507554b6..00000000000
--- a/tests/functional/jinja2/include-exception/foo.cylc
+++ /dev/null
@@ -1,3 +0,0 @@
-# line before error
-{{ raise('some error') }}
-# line after error
diff --git a/tests/integration/validate/test_jinja2.py b/tests/integration/validate/test_jinja2.py
new file mode 100644
index 00000000000..8b2eae2c797
--- /dev/null
+++ b/tests/integration/validate/test_jinja2.py
@@ -0,0 +1,76 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import re
+from textwrap import dedent
+
+from cylc.flow.exceptions import InputError
+from cylc.flow.parsec.exceptions import Jinja2Error
+
+import pytest
+
+
+@pytest.fixture
+def flow_cylc(tmp_path):
+    """Write a flow.cylc file containing the provided text."""
+    def _inner(text):
+        nonlocal tmp_path
+        (tmp_path / 'flow.cylc').write_text(dedent(text).strip())
+        return tmp_path
+
+    return _inner
+
+
+@pytest.mark.parametrize(
+    'line',
+    [
+        pytest.param("raise('some error message')", id='raise'),
+        pytest.param("assert(False, 'some error message')", id='assert'),
+    ],
+)
+def test_raise_helper(flow_cylc, validate, line, monkeypatch):
+    """It should raise an error from within Jinja2."""
+    # it should raise a minimal InputError
+    # (because assert/raise are used to validate inputs)
+    # - intended for users of the workflow
+    src_dir = flow_cylc(f'''
+        #!Jinja2
+        {{{{ {line} }}}}
+    ''')
+    with pytest.raises(
+        InputError,
+        match=(
+            r'^some error message'
+            r'\n\(add --verbose for more context\)$'
+        ),
+    ):
+        validate(src_dir)
+
+    # in verbose mode, it should raise the full error
+    # (this includes the Jinja2 context including file/line info)
+    # - intended for developers of the workflow
+    monkeypatch.setattr('cylc.flow.flags.verbosity', 1)
+    with pytest.raises(
+        Jinja2Error,
+        match=(
+            r'^some error message'
+            r'\nFile /.*/pytest-.*/test_.*/flow.cylc'
+            r'\n  #!Jinja2'
+            r'\n  \{\{ ' + re.escape(line) + r' \}\}'
+            r'\t<-- Jinja2AssertionError$'
+        )
+    ):
+        validate(src_dir)
diff --git a/tests/unit/parsec/test_jinja2support.py b/tests/unit/parsec/test_jinja2support.py
index 68fb450d4cc..385769463dc 100644
--- a/tests/unit/parsec/test_jinja2support.py
+++ b/tests/unit/parsec/test_jinja2support.py
@@ -24,15 +24,10 @@
 def test_raise_helper():
     message = 'Ops'
     error_type = 'CRITICAL'
-    with pytest.raises(Exception) as cm:
+    with pytest.raises(Jinja2AssertionError) as cm:
         raise_helper(message=message)
 
-    assert str(cm.value) == "Jinja2 Error: Ops"
-
-    with pytest.raises(Exception) as cm:
-        raise_helper(message=message, error_type=error_type)
-
-    assert str(cm.value) == "Jinja2 CRITICAL: Ops"
+    assert str(cm.value) == "Ops"
 
 
 def test_assert_helper():

From 0cf10674c78499379a834ced9ed6feda4cba7495 Mon Sep 17 00:00:00 2001
From: Hilary James Oliver <hilary.j.oliver@gmail.com>
Date: Fri, 16 Aug 2024 10:46:44 +1200
Subject: [PATCH 25/53] Update cylc/flow/network/schema.py

Co-authored-by: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
---
 cylc/flow/network/schema.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/cylc/flow/network/schema.py b/cylc/flow/network/schema.py
index 8e20225d4b6..788ec539dcb 100644
--- a/cylc/flow/network/schema.py
+++ b/cylc/flow/network/schema.py
@@ -2229,7 +2229,10 @@ def delta_subs(root, info: 'ResolveInfo', **args) -> AsyncGenerator[Any, None]:
 
 class Pruned(ObjectType):
     class Meta:
-        description = 'WFS Nodes/Edges that have been removed.'
+        description = sstrip('''
+            Objects (e.g. workflows, tasks, jobs) which have been removed from
+            the store.       
+        ''')   
     workflow = String()
     families = graphene.List(String, default_value=[])
     family_proxies = graphene.List(String, default_value=[])

From 4b58a6a7f53f69789466ac88b57832b1433b1679 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Fri, 16 Aug 2024 11:36:43 +0100
Subject: [PATCH 26/53] fixeth lint failure (#6306)

---
 cylc/flow/network/schema.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cylc/flow/network/schema.py b/cylc/flow/network/schema.py
index 788ec539dcb..355f12c4981 100644
--- a/cylc/flow/network/schema.py
+++ b/cylc/flow/network/schema.py
@@ -2231,8 +2231,8 @@ class Pruned(ObjectType):
     class Meta:
         description = sstrip('''
             Objects (e.g. workflows, tasks, jobs) which have been removed from
-            the store.       
-        ''')   
+            the store.
+        ''')
     workflow = String()
     families = graphene.List(String, default_value=[])
     family_proxies = graphene.List(String, default_value=[])

From dab785b395541bd66c8bcced5507d750d8f9fbd2 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Tue, 20 Aug 2024 16:34:28 +0100
Subject: [PATCH 27/53] flake8-bugbear: opt-in to a couple of opinionated
 warnings

---
 cylc/flow/config.py                 |  2 +-
 cylc/flow/scripts/lint.py           | 50 +++++++++++++++++------------
 cylc/flow/scripts/report_timings.py |  6 ++--
 tests/unit/scripts/test_lint.py     |  3 +-
 tox.ini                             |  6 ++++
 5 files changed, 41 insertions(+), 26 deletions(-)

diff --git a/cylc/flow/config.py b/cylc/flow/config.py
index cb987ac6d4c..b91e21afc7e 100644
--- a/cylc/flow/config.py
+++ b/cylc/flow/config.py
@@ -2463,7 +2463,7 @@ def get_taskdef(
                             'Duplicate task message in'
                             f' "[runtime][{name}][outputs]'
                             f'{output} = {message}" - messages must be unique'
-                        )
+                        ) from None
                     valid, msg = TaskOutputValidator.validate(output)
                     if not valid:
                         raise WorkflowConfigError(
diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py
index c8aa7279590..3389143976d 100755
--- a/cylc/flow/scripts/lint.py
+++ b/cylc/flow/scripts/lint.py
@@ -51,53 +51,62 @@
 """
 import functools
 import pkgutil
-from pathlib import Path
 import re
-import sys
 import shutil
+import sys
+from collections import Counter
+from pathlib import Path
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Counter as CounterType,
+    Dict,
+    Iterator,
+    List,
+    Optional,
+    Union,
+)
+
 try:
     # BACK COMPAT: tomli
     #   Support for Python versions before tomllib was added to the
     #   standard library.
     # FROM: Python 3.7
     # TO: Python: 3.10
-    from tomli import (
-        loads as toml_loads,
-        TOMLDecodeError,
-    )
+    from tomli import TOMLDecodeError, loads as toml_loads
 except ImportError:
     from tomllib import (  # type: ignore[no-redef]
         loads as toml_loads,
         TOMLDecodeError,
     )
-from typing import (
-    TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Optional, Union
-)
 
 from ansimarkup import parse as cparse
 
-from cylc.flow import LOG
-from cylc.flow.exceptions import CylcError
 import cylc.flow.flags
-from cylc.flow import job_runner_handlers
+from cylc.flow import LOG, job_runner_handlers
+from cylc.flow.cfgspec.workflow import SPEC, upg
+from cylc.flow.exceptions import CylcError
+from cylc.flow.id_cli import parse_id
 from cylc.flow.job_runner_mgr import JobRunnerManager
 from cylc.flow.loggingutil import set_timestamps
 from cylc.flow.option_parsers import (
+    WORKFLOW_ID_OR_PATH_ARG_DOC,
     CylcOptionParser as COP,
-    WORKFLOW_ID_OR_PATH_ARG_DOC
 )
-from cylc.flow.cfgspec.workflow import upg, SPEC
-from cylc.flow.id_cli import parse_id
 from cylc.flow.parsec.config import ParsecConfig
 from cylc.flow.scripts.cylc import DEAD_ENDS
 from cylc.flow.terminal import cli_function
 
+
 if TYPE_CHECKING:
+    from optparse import Values
+
     # BACK COMPAT: typing_extensions.Literal
     # FROM: Python 3.7
     # TO: Python 3.8
     from typing_extensions import Literal
-    from optparse import Values
+
 
 LINT_TABLE = ['tool', 'cylc', 'lint']
 LINT_SECTION = '.'.join(LINT_TABLE)
@@ -1114,7 +1123,7 @@ def check_cylc_file(
     file: Path,
     file_rel: Path,
     checks: Dict[str, dict],
-    counter: Dict[str, int],
+    counter: CounterType[str],
     modify: bool = False,
 ):
     """Check A Cylc File for Cylc 7 Config"""
@@ -1172,7 +1181,7 @@ def lint(
     file_rel: Path,
     lines: Iterator[str],
     checks: Dict[str, dict],
-    counter: Dict[str, int],
+    counter: CounterType[str],
     modify: bool = False,
     write: Callable = print
 ) -> Iterator[str]:
@@ -1186,7 +1195,7 @@ def lint(
             Iterator which produces one line of text at a time
             e.g. open(file) or iter(['foo\n', 'bar\n', 'baz\n'].
         counter:
-            Dictionary for counting lint hits per category.
+            Counter for counting lint hits per category.
         modify:
             If True, this generator will yield the file one line at a time
             with comments inserted to help users fix their lint.
@@ -1238,7 +1247,6 @@ def lint(
                     msg = check_meta['short'].format(**check)
                 else:
                     msg = check_meta['short']
-                counter.setdefault(check_meta['purpose'], 0)
                 counter[check_meta['purpose']] += 1
                 if modify:
                     # insert a command to help the user
@@ -1473,7 +1481,7 @@ def main(parser: COP, options: 'Values', target=None) -> None:
     )
 
     # Check each file matching a pattern:
-    counter: Dict[str, int] = {}
+    counter: CounterType[str] = Counter()
     for file in get_cylc_files(target, mergedopts[EXCLUDE]):
         LOG.debug(f'Checking {file}')
         check_cylc_file(
diff --git a/cylc/flow/scripts/report_timings.py b/cylc/flow/scripts/report_timings.py
index 362f40953c2..21a4df06edc 100755
--- a/cylc/flow/scripts/report_timings.py
+++ b/cylc/flow/scripts/report_timings.py
@@ -48,13 +48,12 @@
 the database query to obtain the timing information may take some time.
 """
 
-import collections
 import contextlib
 import io as StringIO
 import sys
+from collections import Counter
 from typing import TYPE_CHECKING
 
-
 from cylc.flow import LOG
 from cylc.flow.exceptions import CylcError
 from cylc.flow.id_cli import parse_id
@@ -66,6 +65,7 @@
 from cylc.flow.rundb import CylcWorkflowDAO
 from cylc.flow.terminal import cli_function
 
+
 if TYPE_CHECKING:
     from optparse import Values
 
@@ -278,7 +278,7 @@ def _reshape_timings(timings):
             # there are duplicate entries in the index (see #2509).  The
             # best way around this seems to be to add an intermediate index
             # level (in this case a retry counter) to de-duplicate indices.
-            counts = collections.defaultdict(int)
+            counts = Counter()
             retry = []
             for t in timings.index:
                 counts[t] += 1
diff --git a/tests/unit/scripts/test_lint.py b/tests/unit/scripts/test_lint.py
index 527a33bdfb5..20ff738a1ac 100644
--- a/tests/unit/scripts/test_lint.py
+++ b/tests/unit/scripts/test_lint.py
@@ -16,6 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Tests `cylc lint` CLI Utility."""
 
+from collections import Counter
 import logging
 from pathlib import Path
 from pprint import pformat
@@ -190,7 +191,7 @@
 
 def lint_text(text, checks, ignores=None, modify=False):
     checks = parse_checks(checks, ignores)
-    counter = {}
+    counter = Counter()
     messages = []
     outlines = [
         line
diff --git a/tox.ini b/tox.ini
index d9954dbb7e8..5e87dc7443c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,4 +1,10 @@
 [flake8]
+extend-select=
+    ; Use `raise from` inside `except` for error handling
+    ; https://docs.python.org/3/tutorial/errors.html#exception-chaining
+    B904
+    ; Use Counter() instead of defaultdict(int) to avoid excessive memory use:
+    B910
 ignore=
     ; module level import not at top of file
     E402,

From 55e8b581b0746a3d52a66785b5a6373af2b96a99 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Tue, 20 Aug 2024 18:20:59 +0100
Subject: [PATCH 28/53] Partially revert
 c24ff05546e049a365d48e1e244bef75c02012eb

These raises were incorrectly altered
---
 cylc/flow/config.py       | 4 ++--
 cylc/flow/dbstatecheck.py | 4 ++--
 cylc/flow/xtrigger_mgr.py | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/cylc/flow/config.py b/cylc/flow/config.py
index b91e21afc7e..df27078ee44 100644
--- a/cylc/flow/config.py
+++ b/cylc/flow/config.py
@@ -2470,14 +2470,14 @@ def get_taskdef(
                             f'Invalid task output "'
                             f'[runtime][{name}][outputs]'
                             f'{output} = {message}" - {msg}'
-                        ) from None
+                        )
                     valid, msg = TaskMessageValidator.validate(message)
                     if not valid:
                         raise WorkflowConfigError(
                             f'Invalid task message "'
                             f'[runtime][{name}][outputs]'
                             f'{output} = {message}" - {msg}'
-                        ) from None
+                        )
                     self.taskdefs[name].add_output(output, message)
 
         return self.taskdefs[name]
diff --git a/cylc/flow/dbstatecheck.py b/cylc/flow/dbstatecheck.py
index 1890c697c71..88dd5de3be1 100644
--- a/cylc/flow/dbstatecheck.py
+++ b/cylc/flow/dbstatecheck.py
@@ -78,7 +78,7 @@ def __init__(self, rund, workflow, db_path=None):
         try:
             self.db_point_fmt = self._get_db_point_format()
             self.c7_back_compat_mode = False
-        except sqlite3.OperationalError:
+        except sqlite3.OperationalError as exc:
             # BACK COMPAT: Cylc 7 DB (see method below).
             try:
                 self.db_point_fmt = self._get_db_point_format_compat()
@@ -86,7 +86,7 @@ def __init__(self, rund, workflow, db_path=None):
             except sqlite3.OperationalError:
                 with suppress(Exception):
                     self.conn.close()
-                raise
+                raise exc from None  # original error
 
     def __enter__(self):
         return self
diff --git a/cylc/flow/xtrigger_mgr.py b/cylc/flow/xtrigger_mgr.py
index 525a19cf2d3..ff7c8743846 100644
--- a/cylc/flow/xtrigger_mgr.py
+++ b/cylc/flow/xtrigger_mgr.py
@@ -390,7 +390,7 @@ def _try_workflow_state_backcompat(
             bound_args = sig.bind(*fctx.func_args, **fctx.func_kwargs)
         except TypeError:
             # failed signature check for backcompat function
-            raise
+            raise err from None  # original signature check error
 
         old_sig_str = fctx.get_signature()
         upg_sig_str = "workflow_state({})".format(

From a91e0ef0355a882de71d5c7a0f012e0b56c7310b Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Tue, 20 Aug 2024 18:23:45 +0100
Subject: [PATCH 29/53] Exception chaining: do not swallow potentially useful
 context of caught exceptions

---
 cylc/flow/main_loop/health_check.py |  4 ++--
 cylc/flow/network/__init__.py       | 25 ++++++++++++++-----------
 cylc/flow/param_expand.py           |  4 ++--
 cylc/flow/scripts/cylc.py           |  4 ++--
 cylc/flow/workflow_db_mgr.py        |  4 ++--
 cylc/flow/workflow_files.py         |  4 ++--
 6 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/cylc/flow/main_loop/health_check.py b/cylc/flow/main_loop/health_check.py
index a470fd3cb88..6e42724ba91 100644
--- a/cylc/flow/main_loop/health_check.py
+++ b/cylc/flow/main_loop/health_check.py
@@ -52,8 +52,8 @@ def _check_contact_file(scheduler):
             scheduler.workflow)
         if contact_data != scheduler.contact_data:
             raise CylcError('contact file modified')
-    except (AssertionError, IOError, ValueError, ServiceFileError):
+    except (AssertionError, IOError, ValueError, ServiceFileError) as exc:
         raise CylcError(
             '%s: contact file corrupted/modified and may be left'
             % workflow_files.get_contact_file_path(scheduler.workflow)
-        ) from None
+        ) from exc
diff --git a/cylc/flow/network/__init__.py b/cylc/flow/network/__init__.py
index 315a57a23ba..42b79475ca5 100644
--- a/cylc/flow/network/__init__.py
+++ b/cylc/flow/network/__init__.py
@@ -176,16 +176,14 @@ def _socket_bind(self, min_port, max_port, srv_prv_key_loc=None):
                 srv_prv_key_info.full_key_path)
         except ValueError:
             raise ServiceFileError(
-                f"Failed to find server's public "
-                f"key in "
+                "Failed to find server's public key in "
                 f"{srv_prv_key_info.full_key_path}."
             ) from None
-        except OSError:
+        except OSError as exc:
             raise ServiceFileError(
-                f"IO error opening server's private "
-                f"key from "
+                "IO error opening server's private key from "
                 f"{srv_prv_key_info.full_key_path}."
-            ) from None
+            ) from exc
         if server_private_key is None:  # this can't be caught by exception
             raise ServiceFileError(
                 f"Failed to find server's private "
@@ -238,8 +236,10 @@ def _socket_connect(self, host, port, srv_public_key_loc=None):
         try:
             client_public_key, client_priv_key = zmq.auth.load_certificate(
                 client_priv_key_info.full_key_path)
-        except (OSError, ValueError):
+        except ValueError:
             raise ClientError(error_msg) from None
+        except OSError as exc:
+            raise ClientError(error_msg) from exc
         if client_priv_key is None:  # this can't be caught by exception
             raise ClientError(error_msg)
         self.socket.curve_publickey = client_public_key
@@ -247,6 +247,9 @@ def _socket_connect(self, host, port, srv_public_key_loc=None):
 
         # A client can only connect to the server if it knows its public key,
         # so we grab this from the location it was created on the filesystem:
+        error_msg = (
+            "Failed to load the workflow's public key, so cannot connect."
+        )
         try:
             # 'load_certificate' will try to load both public & private keys
             # from a provided file but will return None, not throw an error,
@@ -256,10 +259,10 @@ def _socket_connect(self, host, port, srv_public_key_loc=None):
             server_public_key = zmq.auth.load_certificate(
                 srv_pub_key_info.full_key_path)[0]
             self.socket.curve_serverkey = server_public_key
-        except (OSError, ValueError):  # ValueError raised w/ no public key
-            raise ClientError(
-                "Failed to load the workflow's public key, so cannot connect."
-            ) from None
+        except ValueError:  # ValueError raised w/ no public key
+            raise ClientError(error_msg) from None
+        except OSError as exc:
+            raise ClientError(error_msg) from exc
 
         self.socket.connect(f'tcp://{host}:{port}')
 
diff --git a/cylc/flow/param_expand.py b/cylc/flow/param_expand.py
index 6ab104404e4..22cf1aa8665 100644
--- a/cylc/flow/param_expand.py
+++ b/cylc/flow/param_expand.py
@@ -196,7 +196,7 @@ def _expand_name(self, results, tmpl, params, spec_vals=None):
                 results.append((tmpl % current_values, current_values))
             except KeyError as exc:
                 raise ParamExpandError(
-                    'parameter %s is not ' 'defined.' % str(exc.args[0])
+                    'parameter %s is not defined.' % str(exc.args[0])
                 ) from None
         else:
             for param_val in params[0][1]:
@@ -427,7 +427,7 @@ def _expand_graph(self, line, all_params,
                     repl = tmpl % param_values
                 except KeyError as exc:
                     raise ParamExpandError(
-                        'parameter %s is not ' 'defined.' % str(exc.args[0])
+                        'parameter %s is not defined.' % str(exc.args[0])
                     ) from None
                 line = line.replace('<' + p_group + '>', repl)
             if line:
diff --git a/cylc/flow/scripts/cylc.py b/cylc/flow/scripts/cylc.py
index 209512264a9..b0a707c5428 100644
--- a/cylc/flow/scripts/cylc.py
+++ b/cylc/flow/scripts/cylc.py
@@ -605,14 +605,14 @@ def pycoverage(cmd_args):  # pragma: no cover
             data_file=str(cylc_wc / '.coverage'),
             source=[str(cylc_wc / 'cylc')]
         )
-    except coverage.misc.CoverageException:
+    except coverage.misc.CoverageException as exc:
         raise Exception(
             # make sure this exception is visible in the traceback
             '\n\n*****************************\n\n'
             'Could not initiate coverage, likely because Cylc was not '
             'installed in editable mode.'
             '\n\n*****************************\n\n'
-        ) from None
+        ) from exc
 
     # start the coverage running
     cov.start()
diff --git a/cylc/flow/workflow_db_mgr.py b/cylc/flow/workflow_db_mgr.py
index d45108ca587..bb7a287e944 100644
--- a/cylc/flow/workflow_db_mgr.py
+++ b/cylc/flow/workflow_db_mgr.py
@@ -700,10 +700,10 @@ def _get_last_run_version(cls, pri_dao: CylcWorkflowDAO) -> Version:
                 ''',  # nosec (table name is a code constant)
                 [cls.KEY_CYLC_VERSION]
             ).fetchone()[0]
-        except (TypeError, OperationalError):
+        except (TypeError, OperationalError) as exc:
             raise ServiceFileError(
                 f"{INCOMPAT_MSG}, or is corrupted."
-            ) from None
+            ) from exc
         return parse_version(last_run_ver)
 
     @classmethod
diff --git a/cylc/flow/workflow_files.py b/cylc/flow/workflow_files.py
index 95626125f89..1550a390f37 100644
--- a/cylc/flow/workflow_files.py
+++ b/cylc/flow/workflow_files.py
@@ -634,8 +634,8 @@ def load_contact_file(id_: str, run_dir=None) -> Dict[str, str]:
     try:
         with open(path) as f:
             file_content = f.read()
-    except IOError:
-        raise ServiceFileError("Couldn't load contact file") from None
+    except IOError as exc:
+        raise ServiceFileError("Couldn't load contact file") from exc
     data: Dict[str, str] = {}
     for line in file_content.splitlines():
         key, value = [item.strip() for item in line.split("=", 1)]

From 3f6cd51245a0aac3f79b78ef8c8fb3b33f4aedea Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 22 Aug 2024 16:20:11 +0100
Subject: [PATCH 30/53] param_expand: str.startswith -> str.index

* `str[0] == x` is ~50% faster than `str.startswith(x)`.
---
 cylc/flow/param_expand.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cylc/flow/param_expand.py b/cylc/flow/param_expand.py
index 22cf1aa8665..06707662d33 100644
--- a/cylc/flow/param_expand.py
+++ b/cylc/flow/param_expand.py
@@ -404,7 +404,7 @@ def _expand_graph(self, line, all_params,
                     pname, offs = REC_P_OFFS.match(item).groups()
                     if offs is None:
                         param_values[pname] = values[pname]
-                    elif offs.startswith('='):
+                    elif offs[0] == '=':
                         # Specific value.
                         try:
                             # Template may require an integer

From 2485aa101048b13db4554e264d562d7d5d03d37b Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 22 Aug 2024 16:23:28 +0100
Subject: [PATCH 31/53] param_expand: OrderedDictWithDefaults -> dict

* We weren't using defaults and native Python dicts are ordered now
  anyway.
* This removes extra init and get/set logic.
---
 cylc/flow/param_expand.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/cylc/flow/param_expand.py b/cylc/flow/param_expand.py
index 06707662d33..38b547d17fe 100644
--- a/cylc/flow/param_expand.py
+++ b/cylc/flow/param_expand.py
@@ -62,7 +62,6 @@ def expand(template, params, results, values=None):
 
 from cylc.flow.exceptions import ParamExpandError
 from cylc.flow.task_id import TaskID
-from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults
 
 # To split runtime heading name lists.
 REC_NAMES = re.compile(r'(?:[^,<]|<[^>]*>)+')
@@ -398,7 +397,7 @@ def _expand_graph(self, line, all_params,
             # Inner loop.
             for p_group in set(REC_P_GROUP.findall(line)):
                 # Parameters must be expanded in the order found.
-                param_values = OrderedDictWithDefaults()
+                param_values = {}
                 tmpl = ''
                 for item in p_group.split(','):
                     pname, offs = REC_P_OFFS.match(item).groups()

From 89eb1d3901d8428143b486a9a51b6495dbf7ea30 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 22 Aug 2024 16:28:01 +0100
Subject: [PATCH 32/53] param_expand: replace str.append with str.join

* str.join(x) is between 5-50% faster than `for i in x: str += i`.
---
 cylc/flow/param_expand.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/cylc/flow/param_expand.py b/cylc/flow/param_expand.py
index 38b547d17fe..1c9dec2598e 100644
--- a/cylc/flow/param_expand.py
+++ b/cylc/flow/param_expand.py
@@ -398,7 +398,6 @@ def _expand_graph(self, line, all_params,
             for p_group in set(REC_P_GROUP.findall(line)):
                 # Parameters must be expanded in the order found.
                 param_values = {}
-                tmpl = ''
                 for item in p_group.split(','):
                     pname, offs = REC_P_OFFS.match(item).groups()
                     if offs is None:
@@ -420,8 +419,10 @@ def _expand_graph(self, line, all_params,
                         else:
                             offval = self._REMOVE
                         param_values[pname] = offval
-                for pname in param_values:
-                    tmpl += self.param_tmpl_cfg[pname]
+                tmpl = ''.join(
+                    self.param_tmpl_cfg[pname]
+                    for pname in param_values
+                )
                 try:
                     repl = tmpl % param_values
                 except KeyError as exc:

From cadf1ef057145873f869092ffcab9784ad7c082e Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Fri, 23 Aug 2024 10:54:19 +0100
Subject: [PATCH 33/53] Ensure that lint S014 has a proper test. (#6324)

---
 cylc/flow/scripts/lint.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py
index 80d5663188e..7509c10a927 100755
--- a/cylc/flow/scripts/lint.py
+++ b/cylc/flow/scripts/lint.py
@@ -195,6 +195,15 @@ def get_wallclock_directives():
 def check_wallclock_directives(line: str) -> Union[Dict[str, str], bool]:
     """Check for job runner specific directives
     equivelent to exection time limit.
+
+    It's recommended that users prefer execution time limit
+    because it gives the Cylc scheduler awareness should communications
+    with a remote job runner be lost.
+
+    Examples:
+        >>> this = check_wallclock_directives
+        >>> this('    -W 42:22')
+        {'directive': '-W 42:22'}
     """
     for directive in set(WALLCLOCK_DIRECTIVES.values()):
         if line.strip().startswith(directive):

From df0336d9ca1a910c6251f64862481fc5687be573 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Mon, 2 Sep 2024 15:46:19 +0100
Subject: [PATCH 34/53] example: cycle on irregular intervals

---
 .../cycle-over-irregular-dates/.validate      |  24 ++++
 .../cycle-over-irregular-dates/index.rst      | 132 ++++++++++++++++++
 .../inter-dependent/flow.cylc                 |  51 +++++++
 .../simple/flow.cylc                          |  40 ++++++
 4 files changed, 247 insertions(+)
 create mode 100755 cylc/flow/etc/examples/cycle-over-irregular-dates/.validate
 create mode 100644 cylc/flow/etc/examples/cycle-over-irregular-dates/index.rst
 create mode 100644 cylc/flow/etc/examples/cycle-over-irregular-dates/inter-dependent/flow.cylc
 create mode 100644 cylc/flow/etc/examples/cycle-over-irregular-dates/simple/flow.cylc

diff --git a/cylc/flow/etc/examples/cycle-over-irregular-dates/.validate b/cylc/flow/etc/examples/cycle-over-irregular-dates/.validate
new file mode 100755
index 00000000000..af06f1c5a7e
--- /dev/null
+++ b/cylc/flow/etc/examples/cycle-over-irregular-dates/.validate
@@ -0,0 +1,24 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -eux
+
+cylc lint ./simple
+cylc validate --check-circular ./simple
+
+cylc lint ./inter-dependent
+cylc validate --check-circular ./inter-dependent
diff --git a/cylc/flow/etc/examples/cycle-over-irregular-dates/index.rst b/cylc/flow/etc/examples/cycle-over-irregular-dates/index.rst
new file mode 100644
index 00000000000..181a57ebc22
--- /dev/null
+++ b/cylc/flow/etc/examples/cycle-over-irregular-dates/index.rst
@@ -0,0 +1,132 @@
+Irregular Cycling
+-----------------
+
+We typically schedule tasks on regular intervals, e.g. ``P1D`` (every day) or
+``PT1H`` (every hour), however, sometimes our intervals are irregular.
+
+:ref:`user_guide.scheduling.exclusions` can be used to "subtract" dates or
+entire recurrences e.g:
+
+``PT1H!PT6H``
+   Every hour, except every six hours.
+``PT1H!(T00, T12)``
+   Every hour, except at 00:00 and 12:00.
+
+However, sometimes we want to schedule tasks on completely irregular intervals
+or at arbitrary dates. E.g, when working on case studies, you might have a list
+or range of arbitrary dates to work with.
+
+
+.. rubric:: Simple Example
+
+.. admonition:: Get a copy of this example
+   :class: hint
+
+   .. code-block:: console
+
+      $ cylc get-resources examples/cycle-over-irregular-dates/simple
+
+This example uses :ref:`Jinja` to define the list of dates and write out a
+scheduling section for each.
+
+.. literalinclude:: simple/flow.cylc
+   :language: cylc
+
+.. tip::
+
+   You can see the result of this Jinja2 code by running the ``cylc view -p``
+   command.
+
+
+.. rubric:: Example with inter-cycle dependencies
+
+.. admonition:: Get a copy of this example
+   :class: hint
+
+   .. code-block:: console
+
+      $ cylc get-resources examples/cycle-over-irregular-dates/inter-dependent
+
+.. _Jinja2 loop variable: https://jinja.palletsprojects.com/en/3.0.x/templates/#list-of-control-structures
+
+If you have dependencies between the cycles, you can make this work by using the
+`Jinja2 loop variable`_.
+
+For example, the previous iteration of the ``{% for date in DATES %}`` loop is
+available as ``loop.previtem`` and the next as ``loop.nextitem``.
+
+If you need to make the tasks which cycle on *irregular* intervals dependent on
+tasks which cycle on *regular* intervals, then you might find the
+:py:func:`strftime <cylc.flow.jinja.filters.strftime.strftime>` function
+helpful as a way of determining the nearest matching cycle.
+
+.. literalinclude:: inter-dependent/flow.cylc
+   :language: cylc
+
+You can see how the cycles are linked together using the ``cylc graph``
+command:
+
+.. NOTE: use "cylc graph . -o foo.dot --cycles 2000 2001" to generate this code
+
+.. digraph:: Example
+   :align: center
+
+   size = "7,15"
+
+   graph [fontname="sans" fontsize="25"]
+   node [fontname="sans"]
+ 
+   subgraph "cluster_20000101T0000Z" { 
+     label="20000101T0000Z"
+     style="dashed"
+     "20000101T0000Z/install" [label="install\n20000101T0000Z"]
+     "20000101T0000Z/prep" [label="prep\n20000101T0000Z"]
+   }
+ 
+   subgraph "cluster_20000105T0600Z" { 
+     label="20000105T0600Z"
+     style="dashed"
+     "20000105T0600Z/plot" [label="plot\n20000105T0600Z"]
+     "20000105T0600Z/run_model" [label="run_model\n20000105T0600Z"]
+   }
+ 
+   subgraph "cluster_20000305T1200Z" { 
+     label="20000305T1200Z"
+     style="dashed"
+     "20000305T1200Z/plot" [label="plot\n20000305T1200Z"]
+     "20000305T1200Z/run_model" [label="run_model\n20000305T1200Z"]
+   }
+ 
+   subgraph "cluster_20000528T1336Z" { 
+     label="20000528T1336Z"
+     style="dashed"
+     "20000528T1336Z/plot" [label="plot\n20000528T1336Z"]
+     "20000528T1336Z/run_model" [label="run_model\n20000528T1336Z"]
+   }
+ 
+   subgraph "cluster_20010101T0000Z" { 
+     label="20010101T0000Z"
+     style="dashed"
+     "20010101T0000Z/prep" [label="prep\n20010101T0000Z"]
+   }
+ 
+   subgraph "cluster_20010105T2324Z" { 
+     label="20010105T2324Z"
+     style="dashed"
+     "20010105T2324Z/plot" [label="plot\n20010105T2324Z"]
+     "20010105T2324Z/run_model" [label="run_model\n20010105T2324Z"]
+   }
+ 
+   "20000101T0000Z/install" -> "20000101T0000Z/prep"
+   "20000101T0000Z/install" -> "20010101T0000Z/prep"
+   "20000101T0000Z/prep" -> "20000105T0600Z/run_model"
+   "20000101T0000Z/prep" -> "20000305T1200Z/run_model"
+   "20000101T0000Z/prep" -> "20000528T1336Z/run_model"
+   "20000105T0600Z/run_model" -> "20000105T0600Z/plot"
+   "20000105T0600Z/run_model" -> "20000305T1200Z/run_model"
+   "20000305T1200Z/run_model" -> "20000305T1200Z/plot"
+   "20000305T1200Z/run_model" -> "20000528T1336Z/run_model"
+   "20000528T1336Z/run_model" -> "20000528T1336Z/plot"
+   "20000528T1336Z/run_model" -> "20010105T2324Z/run_model"
+   "20010101T0000Z/prep" -> "20010105T2324Z/run_model"
+   "20010105T2324Z/run_model" -> "20010105T2324Z/plot"
diff --git a/cylc/flow/etc/examples/cycle-over-irregular-dates/inter-dependent/flow.cylc b/cylc/flow/etc/examples/cycle-over-irregular-dates/inter-dependent/flow.cylc
new file mode 100644
index 00000000000..f0c02a7d709
--- /dev/null
+++ b/cylc/flow/etc/examples/cycle-over-irregular-dates/inter-dependent/flow.cylc
@@ -0,0 +1,51 @@
+#!Jinja2
+
+{%
+    set DATES = [
+        '2000-01-01T00:00Z',
+        '2000-01-01T06:00Z',
+        '2000-03-05T12:00Z',
+        '2000-05-28T13:36Z',
+        '2001-01-05T23:24',
+        '2002-04-30T04:20',
+    ]
+%}
+
+[meta]
+    title = Irregular cycling example
+    description = """
+        A workflow that runs a group of tasks on arbitrary dates
+        with inter-cycle dependencies between those dates.
+    """
+
+[scheduling]
+    initial cycle point = 2000
+    [[graph]]
+        # define the tasks you want to run on startup
+        R1 = install
+
+        # run this graph every year
+        P1Y = """
+            install[^] => prep
+        """
+
+{# loop over the list of dates #}
+{% for date in DATES %}
+        # schedule the tasks to run at each date
+        R1/{{ date }} = """
+            # make "run_model" depend on the "prep" task from the same year
+            prep[{{ date | strftime('%Y') }}] => run_model => plot
+
+    {# include this for all but the first date #}
+    {% if not loop.first %}
+            # make the run_model task depend on its previous instance
+            run_model[ {{ loop.previtem }} ] => run_model
+    {% endif %}
+        """
+{% endfor %}
+
+[runtime]
+    [[install]]
+    [[prep]]
+    [[run_model]]
+    [[plot]]
diff --git a/cylc/flow/etc/examples/cycle-over-irregular-dates/simple/flow.cylc b/cylc/flow/etc/examples/cycle-over-irregular-dates/simple/flow.cylc
new file mode 100644
index 00000000000..be39f88f0c3
--- /dev/null
+++ b/cylc/flow/etc/examples/cycle-over-irregular-dates/simple/flow.cylc
@@ -0,0 +1,40 @@
+#!Jinja2
+
+{%
+    set DATES = [
+        '2000-01-01T00:00Z',
+        '2000-01-01T06:00Z',
+        '2000-03-05T12:00Z',
+        '2000-05-28T13:36Z',
+        '2001-01-05T23:24',
+        '2002-04-30T04:20',
+    ]
+%}
+
+[meta]
+    title = Irregular cycling example
+    description = """
+        A workflow that runs a group of tasks on arbitrary dates.
+    """
+
+[scheduling]
+    # start cycling at the first date
+    initial cycle point = {{ DATES[0] }}
+    [[graph]]
+        # define the tasks you want to run on startup
+        R1 = install
+
+{# loop over the list of dates #}
+{% for date in DATES %}
+        # schedule the tasks to run at each date
+        R1/{{ date }} = """
+            # NOTE: install[^] references the task "install" in the first cycle
+            install[^] => prep => run_model => plot
+        """
+{% endfor %}
+
+[runtime]
+    [[install]]
+    [[prep]]
+    [[run_model]]
+    [[plot]]

From 17e5a6d65508df761b23d830571a280271edfd2e Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 3 Sep 2024 15:56:35 +0100
Subject: [PATCH 35/53] task job mgr: use broadcast_mgr.get_updated_rtconfig
 interface

* Spotted whilst reviewing https://github.com/cylc/cylc-flow/pull/6330
* Replace duplicated logic with a centralised implementation.
---
 cylc/flow/task_job_mgr.py | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/cylc/flow/task_job_mgr.py b/cylc/flow/task_job_mgr.py
index 185966ff12d..debb82ee524 100644
--- a/cylc/flow/task_job_mgr.py
+++ b/cylc/flow/task_job_mgr.py
@@ -51,10 +51,6 @@
     is_remote_platform
 )
 from cylc.flow.job_file import JobFileWriter
-from cylc.flow.parsec.util import (
-    pdeepcopy,
-    poverride
-)
 from cylc.flow.pathutil import get_remote_workflow_run_job_dir
 from cylc.flow.platforms import (
     get_host_from_platform,
@@ -1116,14 +1112,9 @@ def _prep_submit_task_job(
             return itask
 
         # Handle broadcasts
-        overrides = self.task_events_mgr.broadcast_mgr.get_broadcast(
-            itask.tokens
+        rtconfig = self.task_events_mgr.broadcast_mgr.get_updated_rtconfig(
+            itask
         )
-        if overrides:
-            rtconfig = pdeepcopy(itask.tdef.rtconfig)
-            poverride(rtconfig, overrides, prepend=True)
-        else:
-            rtconfig = itask.tdef.rtconfig
 
         # BACK COMPAT: host logic
         # Determine task host or platform now, just before job submission,

From 293c3ccba1877adc1061064b410ea54b62beb4ad Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Thu, 5 Sep 2024 10:53:45 +0100
Subject: [PATCH 36/53] Flake8 B904 (#6356)

---
 cylc/flow/dbstatecheck.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cylc/flow/dbstatecheck.py b/cylc/flow/dbstatecheck.py
index 5039cf0d7ba..fc2d9cf0da3 100644
--- a/cylc/flow/dbstatecheck.py
+++ b/cylc/flow/dbstatecheck.py
@@ -378,7 +378,7 @@ def check_polling_config(selector, is_trigger, is_message):
         try:
             trigger = TASK_STATE_MAP[selector]
         except KeyError:
-            raise InputError(f'No such task state "{selector}"')
+            raise InputError(f'No such task state "{selector}"') from None
         else:
             if trigger is None:
                 raise InputError(

From bac46656ab7ff88c46f500d6515c1bab61f11308 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Tue, 10 Sep 2024 18:15:56 +0100
Subject: [PATCH 37/53] Tidy & add unit test

---
 cylc/flow/clean.py          |   5 +-
 cylc/flow/pathutil.py       |   3 +-
 tests/unit/filetree.py      | 222 ++++++++++++++++++++++++------------
 tests/unit/test_clean.py    |  41 ++++---
 tests/unit/test_pathutil.py |  16 +++
 5 files changed, 195 insertions(+), 92 deletions(-)

diff --git a/cylc/flow/clean.py b/cylc/flow/clean.py
index b38f01b12fc..f73b0364766 100644
--- a/cylc/flow/clean.py
+++ b/cylc/flow/clean.py
@@ -259,7 +259,7 @@ def glob_in_run_dir(
     """Execute a (recursive) glob search in the given run directory.
 
     Returns list of any absolute paths that match the pattern. However:
-    * Does not follow symlinks (apart from the spcedified symlink dirs).
+    * Does not follow symlinks (apart from the specified symlink dirs).
     * Also does not return matching subpaths of matching directories (because
         that would be redundant).
 
@@ -281,6 +281,9 @@ def glob_in_run_dir(
     results: List[Path] = []
     subpath_excludes: Set[Path] = set()
     for path in matches:
+        # Iterate down through ancestors (starting at the run dir) to
+        # weed out redundant subpaths of matched directories and subpaths of
+        # non-standard symlinks
         for rel_ancestor in reversed(path.relative_to(run_dir).parents):
             ancestor = run_dir / rel_ancestor
             if ancestor in subpath_excludes:
diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py
index aa097a2e680..99f4c4e518f 100644
--- a/cylc/flow/pathutil.py
+++ b/cylc/flow/pathutil.py
@@ -481,7 +481,8 @@ def parse_rm_dirs(rm_dirs: Iterable[str]) -> Set[str]:
 
 
 def is_relative_to(path1: Union[Path, str], path2: Union[Path, str]) -> bool:
-    """Return whether or not path1 is relative to path2.
+    """Return whether or not path1 is relative to path2 (including if they are
+    the same path).
 
     Normalizes both paths to avoid trickery with paths containing `..`
     somewhere in them.
diff --git a/tests/unit/filetree.py b/tests/unit/filetree.py
index bef09e4e556..d1830b51cd3 100644
--- a/tests/unit/filetree.py
+++ b/tests/unit/filetree.py
@@ -27,16 +27,21 @@
                 'file.txt': None
             }
         },
-        # Symlinks are represented by pathlib.Path, with the target represented
-        # by the relative path from the tmp_path directory:
-        'symlink': Path('dir/another-dir')
+        # Symlinks are represented by the Symlink class, with the target
+        # represented by the relative path from the tmp_path directory:
+        'symlink': Symlink('dir/another-dir')
     }
 """
 
-from pathlib import Path
+from pathlib import Path, PosixPath
 from typing import Any, Dict, List
 
 
+class Symlink(PosixPath):
+    """A class to represent a symlink target."""
+    ...
+
+
 def create_filetree(
     filetree: Dict[str, Any], location: Path, root: Path
 ) -> None:
@@ -53,10 +58,9 @@ def create_filetree(
         if isinstance(entry, dict):
             path.mkdir(exist_ok=True)
             create_filetree(entry, path, root)
-        elif isinstance(entry, Path):
+        elif isinstance(entry, Symlink):
             path.symlink_to(root / entry)
         else:
-
             path.touch()
 
 
@@ -79,89 +83,155 @@ def get_filetree_as_list(
 
 
 FILETREE_1 = {
-    'cylc-run': {'foo': {'bar': {
-        '.service': {'db': None},
-        'flow.cylc': None,
-        'log': Path('sym/cylc-run/foo/bar/log'),
-        'mirkwood': Path('you-shall-not-pass/mirkwood'),
-        'rincewind.txt': Path('you-shall-not-pass/rincewind.txt')
-    }}},
-    'sym': {'cylc-run': {'foo': {'bar': {
-        'log': {
-            'darmok': Path('you-shall-not-pass/darmok'),
-            'temba.txt': Path('you-shall-not-pass/temba.txt'),
-            'bib': {
-                'fortuna.txt': None
-            }
-        }
-    }}}},
+    'cylc-run': {
+        'foo': {
+            'bar': {
+                '.service': {
+                    'db': None,
+                },
+                'flow.cylc': None,
+                'log': Symlink('sym/cylc-run/foo/bar/log'),
+                'mirkwood': Symlink('you-shall-not-pass/mirkwood'),
+                'rincewind.txt': Symlink('you-shall-not-pass/rincewind.txt'),
+            },
+        },
+    },
+    'sym': {
+        'cylc-run': {
+            'foo': {
+                'bar': {
+                    'log': {
+                        'darmok': Symlink('you-shall-not-pass/darmok'),
+                        'temba.txt': Symlink('you-shall-not-pass/temba.txt'),
+                        'bib': {
+                            'fortuna.txt': None,
+                        },
+                    },
+                },
+            },
+        },
+    },
     'you-shall-not-pass': {  # Nothing in here should get deleted
         'darmok': {
-            'jalad.txt': None
+            'jalad.txt': None,
         },
         'mirkwood': {
-            'spiders.txt': None
+            'spiders.txt': None,
         },
         'rincewind.txt': None,
-        'temba.txt': None
-    }
+        'temba.txt': None,
+    },
 }
 
 FILETREE_2 = {
-    'cylc-run': {'foo': {'bar': Path('sym-run/cylc-run/foo/bar')}},
-    'sym-run': {'cylc-run': {'foo': {'bar': {
-        '.service': {'db': None},
-        'flow.cylc': None,
-        'share': Path('sym-share/cylc-run/foo/bar/share')
-    }}}},
-    'sym-share': {'cylc-run': {'foo': {'bar': {
-        'share': {
-            'cycle': Path('sym-cycle/cylc-run/foo/bar/share/cycle')
-        }
-    }}}},
-    'sym-cycle': {'cylc-run': {'foo': {'bar': {
-        'share': {
-            'cycle': {
-                'macklunkey.txt': None
-            }
-        }
-    }}}},
-    'you-shall-not-pass': {}
+    'cylc-run': {'foo': {'bar': Symlink('sym-run/cylc-run/foo/bar')}},
+    'sym-run': {
+        'cylc-run': {
+            'foo': {
+                'bar': {
+                    '.service': {
+                        'db': None,
+                    },
+                    'flow.cylc': None,
+                    'share': Symlink('sym-share/cylc-run/foo/bar/share'),
+                },
+            },
+        },
+    },
+    'sym-share': {
+        'cylc-run': {
+            'foo': {
+                'bar': {
+                    'share': {
+                        'cycle': Symlink(
+                            'sym-cycle/cylc-run/foo/bar/share/cycle'
+                        ),
+                    },
+                },
+            },
+        },
+    },
+    'sym-cycle': {
+        'cylc-run': {
+            'foo': {
+                'bar': {
+                    'share': {
+                        'cycle': {
+                            'macklunkey.txt': None,
+                        },
+                    },
+                },
+            },
+        },
+    },
+    'you-shall-not-pass': {},
 }
 
 FILETREE_3 = {
-    'cylc-run': {'foo': {'bar': Path('sym-run/cylc-run/foo/bar')}},
-    'sym-run': {'cylc-run': {'foo': {'bar': {
-        '.service': {'db': None},
-        'flow.cylc': None,
-        'share': {
-            'cycle': Path('sym-cycle/cylc-run/foo/bar/share/cycle')
-        }
-    }}}},
-    'sym-cycle': {'cylc-run': {'foo': {'bar': {
-        'share': {
-            'cycle': {
-                'sokath.txt': None
-            }
-        }
-    }}}},
-    'you-shall-not-pass': {}
+    'cylc-run': {
+        'foo': {
+            'bar': Symlink('sym-run/cylc-run/foo/bar'),
+        },
+    },
+    'sym-run': {
+        'cylc-run': {
+            'foo': {
+                'bar': {
+                    '.service': {
+                        'db': None,
+                    },
+                    'flow.cylc': None,
+                    'share': {
+                        'cycle': Symlink(
+                            'sym-cycle/cylc-run/foo/bar/share/cycle'
+                        ),
+                    },
+                },
+            },
+        },
+    },
+    'sym-cycle': {
+        'cylc-run': {
+            'foo': {
+                'bar': {
+                    'share': {
+                        'cycle': {
+                            'sokath.txt': None,
+                        },
+                    },
+                },
+            },
+        },
+    },
+    'you-shall-not-pass': {},
 }
 
 FILETREE_4 = {
-    'cylc-run': {'foo': {'bar': {
-        '.service': {'db': None},
-        'flow.cylc': None,
-        'share': {
-            'cycle': Path('sym-cycle/cylc-run/foo/bar/share/cycle')
-        }
-    }}},
-    'sym-cycle': {'cylc-run': {'foo': {'bar': {
-        'share': {
-            'cycle': {
-                'kiazi.txt': None
-            }
-        }
-    }}}},
-    'you-shall-not-pass': {}
+    'cylc-run': {
+        'foo': {
+            'bar': {
+                '.service': {
+                    'db': None,
+                },
+                'flow.cylc': None,
+                'share': {
+                    'cycle': Symlink('sym-cycle/cylc-run/foo/bar/share/cycle'),
+                },
+            },
+        },
+    },
+    'sym-cycle': {
+        'cylc-run': {
+            'foo': {
+                'bar': {
+                    'share': {
+                        'cycle': {
+                            'kiazi.txt': None,
+                        },
+                    },
+                },
+            },
+        },
+    },
+    'you-shall-not-pass': {},
 }
diff --git a/tests/unit/test_clean.py b/tests/unit/test_clean.py
index 2a67daf4bf4..5227e9d3876 100644
--- a/tests/unit/test_clean.py
+++ b/tests/unit/test_clean.py
@@ -34,8 +34,7 @@
 
 import pytest
 
-from cylc.flow import CYLC_LOG
-from cylc.flow import clean as cylc_clean
+from cylc.flow import CYLC_LOG, clean as cylc_clean
 from cylc.flow.clean import (
     _clean_using_glob,
     _remote_clean_cmd,
@@ -64,10 +63,12 @@
     FILETREE_2,
     FILETREE_3,
     FILETREE_4,
+    Symlink,
     create_filetree,
     get_filetree_as_list,
 )
 
+
 NonCallableFixture = Any
 
 
@@ -536,7 +537,7 @@ def _filetree_for_testing_cylc_clean(
                 'cylc-run': {'foo': {'bar': {
                     '.service': {'db': None},
                     'flow.cylc': None,
-                    'rincewind.txt': Path('whatever')
+                    'rincewind.txt': Symlink('whatever')
                 }}},
                 'sym': {'cylc-run': {'foo': {'bar': {}}}}
             }
@@ -548,12 +549,12 @@ def _filetree_for_testing_cylc_clean(
                 'cylc-run': {'foo': {'bar': {
                     '.service': {'db': None},
                     'flow.cylc': None,
-                    'log': Path('whatever'),
-                    'mirkwood': Path('whatever')
+                    'log': Symlink('whatever'),
+                    'mirkwood': Symlink('whatever')
                 }}},
                 'sym': {'cylc-run': {'foo': {'bar': {
                     'log': {
-                        'darmok': Path('whatever'),
+                        'darmok': Symlink('whatever'),
                         'bib': {}
                     }
                 }}}}
@@ -612,7 +613,7 @@ def test__clean_using_glob(
                 'cylc-run': {'foo': {'bar': {
                     '.service': {'db': None},
                     'flow.cylc': None,
-                    'rincewind.txt': Path('whatever')
+                    'rincewind.txt': Symlink('whatever')
                 }}},
                 'sym': {'cylc-run': {}}
             },
@@ -625,12 +626,12 @@ def test__clean_using_glob(
                 'cylc-run': {'foo': {'bar': {
                     '.service': {'db': None},
                     'flow.cylc': None,
-                    'log': Path('whatever'),
-                    'mirkwood': Path('whatever')
+                    'log': Symlink('whatever'),
+                    'mirkwood': Symlink('whatever')
                 }}},
                 'sym': {'cylc-run': {'foo': {'bar': {
                     'log': {
-                        'darmok': Path('whatever'),
+                        'darmok': Symlink('whatever'),
                         'bib': {}
                     }
                 }}}}
@@ -641,11 +642,13 @@ def test__clean_using_glob(
             {'**/cycle'},
             FILETREE_2,
             {
-                'cylc-run': {'foo': {'bar': Path('sym-run/cylc-run/foo/bar')}},
+                'cylc-run': {'foo': {
+                    'bar': Symlink('sym-run/cylc-run/foo/bar')
+                }},
                 'sym-run': {'cylc-run': {'foo': {'bar': {
                     '.service': {'db': None},
                     'flow.cylc': None,
-                    'share': Path('sym-share/cylc-run/foo/bar/share')
+                    'share': Symlink('sym-share/cylc-run/foo/bar/share')
                 }}}},
                 'sym-share': {'cylc-run': {'foo': {'bar': {
                     'share': {}
@@ -658,7 +661,9 @@ def test__clean_using_glob(
             {'share'},
             FILETREE_2,
             {
-                'cylc-run': {'foo': {'bar': Path('sym-run/cylc-run/foo/bar')}},
+                'cylc-run': {'foo': {
+                    'bar': Symlink('sym-run/cylc-run/foo/bar')
+                }},
                 'sym-run': {'cylc-run': {'foo': {'bar': {
                     '.service': {'db': None},
                     'flow.cylc': None,
@@ -689,7 +694,9 @@ def test__clean_using_glob(
             {'*'},
             FILETREE_2,
             {
-                'cylc-run': {'foo': {'bar': Path('sym-run/cylc-run/foo/bar')}},
+                'cylc-run': {'foo': {
+                    'bar': Symlink('sym-run/cylc-run/foo/bar')
+                }},
                 'sym-run': {'cylc-run': {'foo': {'bar': {
                     '.service': {'db': None},
                 }}}},
@@ -1088,6 +1095,12 @@ def test_clean_top_level(tmp_run_dir: Callable):
              'cylc-run/foo/bar/share/cycle'],
             id="filetree2 **"
         ),
+        pytest.param(
+            'share',
+            FILETREE_2,
+            ['cylc-run/foo/bar/share'],
+            id="filetree2 share"
+        ),
         pytest.param(
             '**',
             FILETREE_3,
diff --git a/tests/unit/test_pathutil.py b/tests/unit/test_pathutil.py
index 6ed9e9ec120..41a47a71409 100644
--- a/tests/unit/test_pathutil.py
+++ b/tests/unit/test_pathutil.py
@@ -40,6 +40,7 @@
     get_workflow_run_share_dir,
     get_workflow_run_work_dir,
     get_workflow_test_log_path,
+    is_relative_to,
     make_localhost_symlinks,
     make_workflow_run_tree,
     parse_rm_dirs,
@@ -576,3 +577,18 @@ def test_get_workflow_name_from_id(
 
     result = get_workflow_name_from_id(id_)
     assert result == name
+
+
+@pytest.mark.parametrize('abs_path', [True, False])
+@pytest.mark.parametrize('path1, path2, expected', [
+    param('/foo/bar/baz', '/foo/bar', True, id="child"),
+    param('/foo/bar', '/foo/bar/baz', False, id="parent"),
+    param('/foo/bar', '/foo/bar', True, id="same-path"),
+    param('/cat/dog', '/hat/bog', False, id="different"),
+    param('/a/b/c/../x/y', '/a/b/x', True, id="trickery"),
+])
+def test_is_relative_to(abs_path, path1, path2, expected):
+    if not abs_path:  # absolute & relative versions of same test
+        path1.lstrip('/')
+        path2.lstrip('/')
+    assert is_relative_to(path1, path2) == expected

From 615a430aff4e17d9c57fd3b547883fd8e5b9b91f Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Tue, 10 Sep 2024 18:43:51 +0100
Subject: [PATCH 38/53] `cylc clean`: fix bug where `--rm share` would not
 clean `share/cycle` symlink first

---
 changes.d/6364.fix.md    |  1 +
 cylc/flow/clean.py       | 17 ++++++++++++-----
 tests/unit/test_clean.py | 16 ++--------------
 3 files changed, 15 insertions(+), 19 deletions(-)
 create mode 100644 changes.d/6364.fix.md

diff --git a/changes.d/6364.fix.md b/changes.d/6364.fix.md
new file mode 100644
index 00000000000..9ff613e2c18
--- /dev/null
+++ b/changes.d/6364.fix.md
@@ -0,0 +1 @@
+Fixed bug where `cylc clean <workflow> --rm share` would not take care of removing the target of the `share/cycle` symlink directory.
diff --git a/cylc/flow/clean.py b/cylc/flow/clean.py
index f73b0364766..fda1d4f1dd8 100644
--- a/cylc/flow/clean.py
+++ b/cylc/flow/clean.py
@@ -57,6 +57,7 @@
 )
 from cylc.flow.pathutil import (
     get_workflow_run_dir,
+    is_relative_to,
     parse_rm_dirs,
     remove_dir_and_target,
     remove_dir_or_file,
@@ -329,13 +330,19 @@ def _clean_using_glob(
         LOG.info(f"No files matching '{pattern}' in {run_dir}")
         return
     # First clean any matching symlink dirs
-    for path in abs_symlink_dirs:
-        if path in matches:
-            remove_dir_and_target(path)
-            if path == run_dir:
+    for symlink_dir in abs_symlink_dirs:
+        # Note: must clean e.g. share/cycle/ before share/ if the former
+        # is a symlink even if only the latter was specified.
+        if (
+            any(is_relative_to(symlink_dir, path) for path in matches)
+            and symlink_dir.is_symlink()
+        ):
+            remove_dir_and_target(symlink_dir)
+            if symlink_dir == run_dir:
                 # We have deleted the run dir
                 return
-            matches.remove(path)
+            if symlink_dir in matches:
+                matches.remove(symlink_dir)
     # Now clean the rest
     for path in matches:
         remove_dir_or_file(path)
diff --git a/tests/unit/test_clean.py b/tests/unit/test_clean.py
index 5227e9d3876..285bfa6a23f 100644
--- a/tests/unit/test_clean.py
+++ b/tests/unit/test_clean.py
@@ -669,13 +669,7 @@ def test__clean_using_glob(
                     'flow.cylc': None,
                 }}}},
                 'sym-share': {'cylc-run': {}},
-                'sym-cycle': {'cylc-run': {'foo': {'bar': {
-                    'share': {
-                        'cycle': {
-                            'macklunkey.txt': None
-                        }
-                    }
-                }}}}
+                'sym-cycle': {'cylc-run': {}},
             },
             id="filetree2 share"
         ),
@@ -701,13 +695,7 @@ def test__clean_using_glob(
                     '.service': {'db': None},
                 }}}},
                 'sym-share': {'cylc-run': {}},
-                'sym-cycle': {'cylc-run': {'foo': {'bar': {
-                    'share': {
-                        'cycle': {
-                            'macklunkey.txt': None
-                        }
-                    }
-                }}}}
+                'sym-cycle': {'cylc-run': {}},
             },
             id="filetree2 *"
         ),

From 128e5848dbedff813ea656f920ab9bd300e83938 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Tue, 10 Sep 2024 18:57:00 +0100
Subject: [PATCH 39/53] Simplify `cylc clean` reinvocation check

---
 cylc/flow/clean.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/cylc/flow/clean.py b/cylc/flow/clean.py
index fda1d4f1dd8..72c23e1e3e2 100644
--- a/cylc/flow/clean.py
+++ b/cylc/flow/clean.py
@@ -117,12 +117,8 @@ def _clean_check(opts: 'Values', id_: str, run_dir: Path) -> None:
     # Thing to clean must be a dir or broken symlink:
     if not run_dir.is_dir() and not run_dir.is_symlink():
         raise FileNotFoundError(f"No directory to clean at {run_dir}")
-    db_path = (
-        run_dir / WorkflowFiles.Service.DIRNAME / WorkflowFiles.Service.DB
-    )
-    if opts.local_only and not db_path.is_file():
-        # Will reach here if this is cylc clean re-invoked on remote host
-        # (workflow DB only exists on scheduler host); don't need to worry
+    if opts.no_scan:
+        # This is cylc clean re-invoked on remote host; don't need to worry
         # about contact file.
         return
     try:

From 8c53251566b2c239ec7072d21345f8bc85912bc2 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Wed, 18 Sep 2024 12:53:50 +0100
Subject: [PATCH 40/53] subprocpool: populate missing 255 callback arguments

* In some situations, these missing arguments could cause tasks to
  submit fail as the result of a host outage rather than moving onto the
  next host for the platform.
---
 changes.d/6376.fix.md    |  1 +
 cylc/flow/subprocpool.py | 13 +++++++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)
 create mode 100644 changes.d/6376.fix.md

diff --git a/changes.d/6376.fix.md b/changes.d/6376.fix.md
new file mode 100644
index 00000000000..686f7eef945
--- /dev/null
+++ b/changes.d/6376.fix.md
@@ -0,0 +1 @@
+Fixes an issue that could cause Cylc to ignore the remaining hosts in a platform in response to an `ssh` error in some niche circumstances.
diff --git a/cylc/flow/subprocpool.py b/cylc/flow/subprocpool.py
index 864071332b8..5af8b897ac1 100644
--- a/cylc/flow/subprocpool.py
+++ b/cylc/flow/subprocpool.py
@@ -291,8 +291,17 @@ def process(self):
                 )
                 continue
             # Command still running, see if STDOUT/STDERR are readable or not
-            runnings.append([
-                proc, ctx, bad_hosts, callback, callback_args, None, None])
+            runnings.append(
+                [
+                    proc,
+                    ctx,
+                    bad_hosts,
+                    callback,
+                    callback_args,
+                    callback_255,
+                    callback_255_args,
+                ]
+            )
             # Unblock proc's STDOUT/STDERR if necessary. Otherwise, a full
             # STDOUT or STDERR may stop command from proceeding.
             self._poll_proc_pipes(proc, ctx)

From 87bb2d49a53c7807d432b3dbb47b679a0b275796 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 23 Sep 2024 17:38:22 +0100
Subject: [PATCH 41/53] Bump pypa/gh-action-pypi-publish from 1.10.1 to 1.10.2
 (#6378) [skip ci]

Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.10.1 to 1.10.2.
- [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases)
- [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.10.1...v1.10.2)

---
updated-dependencies:
- dependency-name: pypa/gh-action-pypi-publish
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/2_auto_publish_release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/2_auto_publish_release.yml b/.github/workflows/2_auto_publish_release.yml
index 15b4a590a69..3dcf628d0bc 100644
--- a/.github/workflows/2_auto_publish_release.yml
+++ b/.github/workflows/2_auto_publish_release.yml
@@ -38,7 +38,7 @@ jobs:
       uses: cylc/release-actions/build-python-package@v1
 
     - name: Publish distribution to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.10.1
+      uses: pypa/gh-action-pypi-publish@v1.10.2
       with:
         user: __token__ # uses the API token feature of PyPI - least permissions possible
         password: ${{ secrets.PYPI_TOKEN }}

From 2f1e2dfa412a142eb91f4857895a6873ee9b2887 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Wed, 25 Sep 2024 19:32:12 +0100
Subject: [PATCH 42/53] Add ruff, black & isort configs

Not mandated, but for use if wanted (e.g. using "format selection" in IDEs).
Chosen defaults that best match our current style.
---
 pyproject.toml | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/pyproject.toml b/pyproject.toml
index 0f8f095e557..176c2c30256 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -22,3 +22,28 @@ showcontent = true
 directory = "fix"
 name = "🔧 Fixes"
 showcontent = true
+
+
+# Not mandated to use these tools, but if you do:
+
+[tool.ruff]
+line-length = 79
+target-version = "py37"
+
+[tool.ruff.format]
+quote-style = "preserve"
+
+
+[tool.black]
+line-length = 79
+target-version = ['py37']
+skip-string-normalization = true
+
+
+[tool.isort]
+profile = "black"
+line_length = 79
+force_grid_wrap = 2
+lines_after_imports = 2
+combine_as_imports = true
+force_sort_within_sections = true

From 24f444f605555b629071b263980349e95bf3e3e1 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Wed, 25 Sep 2024 19:44:58 +0100
Subject: [PATCH 43/53] Blacken & isort

---
 cylc/flow/pathutil.py | 59 +++++++++++++++++++++++++------------------
 1 file changed, 35 insertions(+), 24 deletions(-)

diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py
index 5311c0ea6ac..22c73694423 100644
--- a/cylc/flow/pathutil.py
+++ b/cylc/flow/pathutil.py
@@ -21,12 +21,21 @@
 import re
 from shutil import rmtree
 from time import sleep
-from typing import Dict, Iterable, Set, Union, Optional, Any
+from typing import (
+    Any,
+    Dict,
+    Iterable,
+    Optional,
+    Set,
+    Union,
+)
 
 from cylc.flow import LOG
 from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
 from cylc.flow.exceptions import (
-    FileRemovalError, InputError, WorkflowFilesError
+    FileRemovalError,
+    InputError,
+    WorkflowFilesError,
 )
 from cylc.flow.platforms import get_localhost_install_target
 
@@ -40,7 +49,7 @@
     ^({re.escape(os.curdir)}|{re.escape(os.pardir)})
     ({re.escape(os.sep)}|$)
     ''',
-    re.VERBOSE
+    re.VERBOSE,
 )
 """Matches relative paths that are explicit (starts with ./)"""
 
@@ -58,9 +67,9 @@
 def expand_path(*args: Union[Path, str]) -> str:
     """Expand both vars and user in path and normalise it, joining any
     extra args."""
-    return os.path.normpath(os.path.expanduser(os.path.expandvars(
-        os.path.join(*args)
-    )))
+    return os.path.normpath(
+        os.path.expanduser(os.path.expandvars(os.path.join(*args)))
+    )
 
 
 def get_remote_workflow_run_dir(
@@ -87,7 +96,7 @@ def get_cylc_run_dir(alt_run_dir: Optional[str] = None) -> str:
 def get_alt_workflow_run_dir(
     alt_run_dir: Union[Path, str],
     workflow_id: Union[Path, str],
-    *args: Union[Path, str]
+    *args: Union[Path, str],
 ) -> str:
     """Return alternate workflow run directory.
 
@@ -125,9 +134,7 @@ def get_workflow_run_scheduler_log_path(workflow):
 
 def get_workflow_file_install_log_dir(workflow, *args):
     """Return workflow file install log file dir, join any extra args."""
-    return get_workflow_run_dir(
-        workflow, 'log', 'remote-install', *args
-    )
+    return get_workflow_run_dir(workflow, 'log', 'remote-install', *args)
 
 
 def get_workflow_run_config_log_dir(workflow, *args):
@@ -173,7 +180,7 @@ def make_workflow_run_tree(workflow):
 def make_localhost_symlinks(
     rund: Union[Path, str],
     named_sub_dir: str,
-    symlink_conf: Optional[Dict[str, Dict[str, str]]] = None
+    symlink_conf: Optional[Dict[str, Dict[str, str]]] = None,
 ) -> Dict[str, Union[Path, str]]:
     """Creates symlinks for any configured symlink dirs from glbl_cfg.
     Args:
@@ -189,7 +196,8 @@ def make_localhost_symlinks(
     symlinks_created = {}
     dirs_to_symlink = get_dirs_to_symlink(
         get_localhost_install_target(),
-        named_sub_dir, symlink_conf=symlink_conf
+        named_sub_dir,
+        symlink_conf=symlink_conf,
     )
     for key, value in dirs_to_symlink.items():
         if value is None:
@@ -204,7 +212,8 @@ def make_localhost_symlinks(
             raise WorkflowFilesError(
                 f"Can't symlink to {target}\n"
                 "Undefined variables, check "
-                f"global config: {', '.join(env_vars)}")
+                f"global config: {', '.join(env_vars)}"
+            )
 
         symlink_success = make_symlink_dir(symlink_path, target)
         # Symlink info returned for logging purposes. Symlinks should be
@@ -217,7 +226,7 @@ def make_localhost_symlinks(
 def get_dirs_to_symlink(
     install_target: str,
     workflow_id: str,
-    symlink_conf: Optional[Dict[str, Dict[str, Any]]] = None
+    symlink_conf: Optional[Dict[str, Dict[str, Any]]] = None,
 ) -> Dict[str, str]:
     """Returns dictionary of directories to symlink.
 
@@ -241,13 +250,15 @@ def get_dirs_to_symlink(
     base_dir = symlink_conf[install_target]['run']
     if base_dir:
         dirs_to_symlink['run'] = os.path.join(
-            base_dir, 'cylc-run', workflow_id)
+            base_dir, 'cylc-run', workflow_id
+        )
     for dir_ in SYMLINKABLE_LOCATIONS:
         link = symlink_conf[install_target].get(dir_, None)
         if (not link) or link == base_dir:
             continue
         dirs_to_symlink[dir_] = os.path.join(
-            link, 'cylc-run', workflow_id, dir_)
+            link, 'cylc-run', workflow_id, dir_
+        )
     return dirs_to_symlink
 
 
@@ -440,16 +451,17 @@ def get_next_rundir_number(run_path: Union[str, Path]) -> int:
         # This function unlikely to be called in circumstances where this will
         # be a problem.
         last_run_num = re_runX.search(  # type: ignore
-            old_run_path).group(1)
+            old_run_path
+        ).group(1)
         last_run_num = int(last_run_num)
     else:
         # If the ``runN`` symlink has been removed, get next numbered run from
         # file names:
         paths = Path(run_path).glob('run[0-9]*')
         run_numbers = (
-            int(m.group(1)) for m in (
-                re_runX.search(i.name) for i in paths
-            ) if m
+            int(m.group(1))
+            for m in (re_runX.search(i.name) for i in paths)
+            if m
         )
         last_run_num = max(run_numbers, default=0)
 
@@ -473,8 +485,8 @@ def parse_rm_dirs(rm_dirs: Iterable[str]) -> Set[str]:
             if os.path.isabs(part):
                 raise InputError("--rm option cannot take absolute paths")
             if (
-                part in {os.curdir, os.pardir} or
-                part.startswith(f"{os.pardir}{os.sep}")  # '../'
+                part in {os.curdir, os.pardir}
+                or part.startswith(f"{os.pardir}{os.sep}")  # '../'
             ):
                 raise InputError(
                     "--rm option cannot take paths that point to the "
@@ -504,8 +516,7 @@ def is_relative_to(path1: Union[Path, str], path2: Union[Path, str]) -> bool:
 
 
 def get_workflow_name_from_id(workflow_id: str) -> str:
-    """Workflow name is the ID shorn of the runN directory name.
-    """
+    """Workflow name is the ID shorn of the runN directory name."""
     cylc_run_dir = Path(get_cylc_run_dir())
     if Path(workflow_id).is_absolute():
         # this is a source directory, not an install dir:

From 70134809a395f04a283a23c95a779efbba9da948 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 26 Sep 2024 16:42:49 +0100
Subject: [PATCH 44/53] fix typo

---
 cylc/flow/etc/examples/extending-workflow/index.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cylc/flow/etc/examples/extending-workflow/index.rst b/cylc/flow/etc/examples/extending-workflow/index.rst
index 036438ff466..35c5b086aec 100644
--- a/cylc/flow/etc/examples/extending-workflow/index.rst
+++ b/cylc/flow/etc/examples/extending-workflow/index.rst
@@ -79,7 +79,7 @@ Running Tasks At The `stop after cycle point`
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 If you have tasks that you want to run before the workflow shuts down at the
-`stop after cycle point`, use the recurrence ``R1/<cycle-point>`` to schedudule
+`stop after cycle point`, use the recurrence ``R1/<cycle-point>`` to schedule
 them, e.g:
 
 .. code-block:: cylc

From 5906a635a172d9909be4a4eaf9ffa28f761d2009 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 1 Oct 2024 10:40:59 +0100
Subject: [PATCH 45/53] tests/i: add test for #6114

---
 tests/integration/test_data_store_mgr.py | 34 ++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/tests/integration/test_data_store_mgr.py b/tests/integration/test_data_store_mgr.py
index 906b1ac052d..fde34696cf3 100644
--- a/tests/integration/test_data_store_mgr.py
+++ b/tests/integration/test_data_store_mgr.py
@@ -350,3 +350,37 @@ async def test_absolute_graph_edges(flow, scheduler, start):
             # +2 for Cylc's runahead
             for cycle in range(1, runahead_cycles + 3)
         }
+
+
+async def test_flow_numbers(flow, scheduler, start):
+    """It should update flow numbers when a task is triggered.
+
+    See https://github.com/cylc/cylc-flow/issues/6114
+    """
+    id_ = flow({
+        'scheduling': {
+            'graph': {
+                'R1': 'a => b'
+            }
+        }
+    })
+    schd = scheduler(id_)
+    async with start(schd):
+        # initialise the data store
+        await schd.update_data_structure()
+
+        # the task should exist in the original flow
+        ds_task = schd.data_store_mgr.get_data_elements(TASK_PROXIES).added[1]
+        assert ds_task.name == 'b'
+        assert ds_task.flow_nums == '[1]'
+
+        # force trigger the task in a new flow
+        schd.pool.force_trigger_tasks(['1/b'], ['2'])
+
+        # update the data store
+        await schd.update_data_structure()
+
+        # the task should now exist in the new flow
+        ds_task = schd.data_store_mgr.get_data_elements(TASK_PROXIES).added[1]
+        assert ds_task.name == 'b'
+        assert ds_task.flow_nums == '[2]'

From 48e296ad5fb1e5afc472169bfe9616fe1f721e9d Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 1 Oct 2024 13:15:03 +0100
Subject: [PATCH 46/53] fix typo

---
 cylc/flow/scripts/lint.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py
index 7509c10a927..39ecb5ab85e 100755
--- a/cylc/flow/scripts/lint.py
+++ b/cylc/flow/scripts/lint.py
@@ -194,7 +194,7 @@ def get_wallclock_directives():
 
 def check_wallclock_directives(line: str) -> Union[Dict[str, str], bool]:
     """Check for job runner specific directives
-    equivelent to exection time limit.
+    equivalent to exection time limit.
 
     It's recommended that users prefer execution time limit
     because it gives the Cylc scheduler awareness should communications

From 9ebba03cfa9e58f5446859b94eb520baa1d9b368 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 4 Oct 2024 10:00:35 +0000
Subject: [PATCH 47/53] Bump pypa/gh-action-pypi-publish from 1.10.2 to 1.10.3

Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.10.2 to 1.10.3.
- [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases)
- [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.10.2...v1.10.3)

---
updated-dependencies:
- dependency-name: pypa/gh-action-pypi-publish
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/2_auto_publish_release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/2_auto_publish_release.yml b/.github/workflows/2_auto_publish_release.yml
index 3dcf628d0bc..1ab1e1cce42 100644
--- a/.github/workflows/2_auto_publish_release.yml
+++ b/.github/workflows/2_auto_publish_release.yml
@@ -38,7 +38,7 @@ jobs:
       uses: cylc/release-actions/build-python-package@v1
 
     - name: Publish distribution to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.10.2
+      uses: pypa/gh-action-pypi-publish@v1.10.3
       with:
         user: __token__ # uses the API token feature of PyPI - least permissions possible
         password: ${{ secrets.PYPI_TOKEN }}

From b2accfb885b4a8ea7dce1ad2bfdff4d089ac31b1 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 10 Oct 2024 14:37:25 +0100
Subject: [PATCH 48/53] examples: using data to define your workflow

---
 .../examples/external-data-files/.validate    |  32 +++++
 .../examples/external-data-files/flow.cylc    |  66 ++++++++++
 .../examples/external-data-files/index.rst    | 122 ++++++++++++++++++
 .../lib/python/load_data.py                   |  12 ++
 .../examples/external-data-files/stations.csv |   7 +
 .../external-data-files/stations.json         |  44 +++++++
 6 files changed, 283 insertions(+)
 create mode 100755 cylc/flow/etc/examples/external-data-files/.validate
 create mode 100644 cylc/flow/etc/examples/external-data-files/flow.cylc
 create mode 100644 cylc/flow/etc/examples/external-data-files/index.rst
 create mode 100644 cylc/flow/etc/examples/external-data-files/lib/python/load_data.py
 create mode 100644 cylc/flow/etc/examples/external-data-files/stations.csv
 create mode 100644 cylc/flow/etc/examples/external-data-files/stations.json

diff --git a/cylc/flow/etc/examples/external-data-files/.validate b/cylc/flow/etc/examples/external-data-files/.validate
new file mode 100755
index 00000000000..2ea8bbb7bd9
--- /dev/null
+++ b/cylc/flow/etc/examples/external-data-files/.validate
@@ -0,0 +1,32 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -eux
+
+ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)"
+
+# run the workflow
+cylc vip --check-circular --no-run-name --no-detach --workflow-name "$ID"
+
+# check the station:ID mapping came out as expected
+grep 'fetch data for heathrow, WMO ID: 03772' "${HOME}/cylc-run/${ID}/log/job/20000101T0000Z/fetch_heathrow/NN/job.out"
+
+# lint
+cylc lint "$ID"
+
+# clean up
+cylc clean "$ID"
diff --git a/cylc/flow/etc/examples/external-data-files/flow.cylc b/cylc/flow/etc/examples/external-data-files/flow.cylc
new file mode 100644
index 00000000000..232448935d4
--- /dev/null
+++ b/cylc/flow/etc/examples/external-data-files/flow.cylc
@@ -0,0 +1,66 @@
+#!Jinja2
+
+[meta]
+    title = Weather Station Workflow
+    description = """
+        This workflow demonstrates how to read in a data file for use in
+        defining your workflow.
+
+        We have a file called "stations.json" which contains a list of weather
+        stations with some data for each. This workflow reads the
+        "stations.json" file and creates a family for each weather station
+        with an environment variable for each data field.
+
+        You can load data in other formats too. Try changing "load_json" to
+        "load_csv" and "stations.json" to "stations.csv" for a CSV example.
+    """
+
+
+{# Import a Python function to load our data. #}
+{% from "load_data" import load_json %}
+
+{# Load data from the specified file. #}
+{% set stations = load_json('stations.json') %}
+
+{# Extract a list of station names from the data file. #}
+{% set station_names = stations | map(attribute="name") | list %}
+
+
+{# Provide Cylc with a list of weather stations. #}
+[task parameters]
+    station = {{ station_names | join(', ') }}
+
+
+[scheduling]
+    initial cycle point = 2000-01-01
+    final cycle point = 2000-01-02
+    [[graph]]
+        P1D = fetch<station> => process<station> => collate
+
+
+[runtime]
+{# Define a family for each weather station #}
+{% for station in stations %}
+    [[STATION<station={{ station["name"] }}>]]
+        [[[environment]]]
+            {# Turn the <station> parameter into an environment variable #}
+            {# NB: Just to show how, we could also have used `station["name"]`. #}
+            name = %(station)s
+            {# Turn the data for this station into environment variables. #}
+            wmo = {{ station["wmo"] }}
+            alt = {{ station["alt"] }}
+            lat = {{ station["lat"] }}
+            lon = {{ station["lon"] }}
+{% endfor %}
+
+    # a task that gets data
+    [[fetch<station>]]
+        inherit = STATION<station>
+        script = echo "fetch data for $name, WMO ID: $wmo"
+
+    [[process<station>]]
+        inherit = STATION<station>
+        script = echo "process data for $name, location: $lat,$lon"
+
+    [[collate]]
+        script = "echo collate data for stations: {{ station_names }}"
diff --git a/cylc/flow/etc/examples/external-data-files/index.rst b/cylc/flow/etc/examples/external-data-files/index.rst
new file mode 100644
index 00000000000..e68b34cfc50
--- /dev/null
+++ b/cylc/flow/etc/examples/external-data-files/index.rst
@@ -0,0 +1,122 @@
+Using Data To Define Your Workflow
+==================================
+
+.. admonition:: Get a copy of this example
+   :class: hint
+
+   .. code-block:: console
+
+      $ cylc get-resources examples/external-data-files
+
+We often want to read in a dataset for use in defining our workflow.
+
+The :ref:`Cylc tutorial <tutorial-cylc-consolidating-configuration>` is an
+example of this where we want one ``get_observations`` task for each of a list
+of weather stations. Each weather station has a name (e.g. "heathrow") and an
+ID (e.g. 3772).
+
+.. code-block:: cylc
+
+   [runtime]
+       [[get_observations_heathrow]]
+           script = get-observations
+           [[[environment]]]
+               SITE_ID = 3772
+       [[get_observations_camborne]]
+           script = get-observations
+           [[[environment]]]
+               SITE_ID = 3808
+       [[get_observations_shetland]]
+           script = get-observations
+           [[[environment]]]
+               SITE_ID = 3005
+       [[get_observations_aldergrove]]
+           script = get-observations
+           [[[environment]]]
+               SITE_ID = 3917
+
+It can be inconvenient to write out the name and ID of each station in your
+workflow like this, however, you may already have this information in a more
+convenient format (i.e. a data file of some form).
+
+With Cylc, we can use :ref:`Jinja2 <Jinja>` to read in a data file and use that data to
+define your workflow.
+
+
+The Approach
+------------
+
+This example has three components:
+
+1. A JSON file containing a list of weather stations along with all the data
+   associated with them.
+
+   .. literalinclude:: stations.json
+      :language: json
+      :caption: stations.json
+
+2. A Python function that reads the JSON file.
+
+   .. code-block:: python
+      :caption: lib/python/load_data.py
+   
+      import json
+      
+      
+      def load_json(filename):
+          with open(filename, 'r') as json_file:
+              return json.load(json_file)
+
+   We put this Python code in the workflow's ``lib/python`` directory which
+   allows us to import it from within our workflow.
+
+3. A ``flow.cylc`` file that uses the Python function to load the
+   data file.
+
+   We can import Python functions with Jinja2 using the following syntax:
+
+   .. code-block::
+
+      {% from "load_data" import load_json %}
+
+   For more information, see :ref:`jinja2.importing_python_modules`.
+
+
+
+The Workflow
+------------
+
+The three files are arranged like so:
+
+.. code-block:: none
+   :caption: File Structure
+
+   |-- flow.cylc
+   |-- lib
+   |   `-- python
+   |       `-- load_data.py
+   `-- stations.json
+
+The ``flow.cylc`` file:
+
+* Imports the Python function.
+* Uses it to load the data.
+* Then uses the data to define the workflow.
+
+.. literalinclude:: flow.cylc
+   :language: ini
+   :caption: flow.cylc
+
+
+Data Types
+----------
+
+We can load other types of data file too. This example also includes the same
+data in CSV format along with a Python function to load CSV data. To try it
+out, open the ``flow.cylc`` file and replace ``stations.json`` with
+``stations.csv`` and ``load_json`` with ``load_csv``.
+
+Any Python code that you import using Jinja2 will be executed using the Python
+environment that Cylc is running in. So if you want to import Python code that
+isn't in the standard library, you may need to get your system administrator to
+install this dependency into the Cylc environment for you.
diff --git a/cylc/flow/etc/examples/external-data-files/lib/python/load_data.py b/cylc/flow/etc/examples/external-data-files/lib/python/load_data.py
new file mode 100644
index 00000000000..7f9dd822884
--- /dev/null
+++ b/cylc/flow/etc/examples/external-data-files/lib/python/load_data.py
@@ -0,0 +1,12 @@
+import csv
+import json
+
+
+def load_csv(filename):
+    with open(filename, 'r') as csv_file:
+        return list(csv.DictReader(csv_file))
+
+
+def load_json(filename):
+    with open(filename, 'r') as json_file:
+        return json.load(json_file)
diff --git a/cylc/flow/etc/examples/external-data-files/stations.csv b/cylc/flow/etc/examples/external-data-files/stations.csv
new file mode 100644
index 00000000000..81c13a53ad2
--- /dev/null
+++ b/cylc/flow/etc/examples/external-data-files/stations.csv
@@ -0,0 +1,7 @@
+name,wmo,alt,lat,lon
+camborne,03808,87,50.21841,-5.32753
+heathrow,03772,25,51.47922.-0.45061
+lerwick,03005,82,60.13893,-1.18491,
+aldergrove,03917,63,54.66365,-6.22534,
+exeter,03844,27,50.73717,-3.40579,
+middle_wallop,03749,90,51.14987,-1.56994
diff --git a/cylc/flow/etc/examples/external-data-files/stations.json b/cylc/flow/etc/examples/external-data-files/stations.json
new file mode 100644
index 00000000000..213d4282eb4
--- /dev/null
+++ b/cylc/flow/etc/examples/external-data-files/stations.json
@@ -0,0 +1,44 @@
+[
+    {
+        "name": "camborne",
+        "wmo": "03808",
+        "alt": 87,
+        "lat": 50.21841,
+        "lon": -5.32753
+    },
+    {
+        "name": "heathrow",
+        "wmo": "03772",
+        "alt": 25,
+        "lat": 51.47922,
+        "lon": -0.45061
+    },
+    {
+        "name": "lerwick",
+        "wmo": "03005",
+        "alt": 82,
+        "lat": 60.13893,
+        "lon": -1.18491
+    },
+    {
+        "name": "aldergrove",
+        "wmo": "03917",
+        "alt": 63,
+        "lat": 54.66365,
+        "lon": -6.22534
+    },
+    {
+        "name": "exeter",
+        "wmo": "03844",
+        "alt": 27,
+        "lat": 50.73717,
+        "lon": -3.40579
+    },
+    {
+        "name": "middle_wallop",
+        "wmo": "03749",
+        "alt": 90,
+        "lat": 51.14987,
+        "lon": -1.56994
+    }
+]

From afeece620e589e85722283cdad7f054d8648ca36 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Fri, 18 Oct 2024 15:41:03 +0100
Subject: [PATCH 49/53] Spelling/formatting

---
 cylc/flow/cfgspec/globalcfg.py |  2 +-
 cylc/flow/scripts/lint.py      | 75 +++++++++++++++++-----------------
 2 files changed, 38 insertions(+), 39 deletions(-)

diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
index 4988da885e2..987d2dc29ac 100644
--- a/cylc/flow/cfgspec/globalcfg.py
+++ b/cylc/flow/cfgspec/globalcfg.py
@@ -1151,7 +1151,7 @@ def default_for(
                 """)
                 for folder, versionadded in SYMLINKABLE_LOCATIONS.items():
                     Conf(folder, VDR.V_STRING, None, desc=f"""
-                        Alternative location for the log dir.
+                        Alternative location for the {folder} dir.
 
                         If specified the workflow {folder} directory will
                         be created in
diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py
index 39ecb5ab85e..0dd991b8fa9 100755
--- a/cylc/flow/scripts/lint.py
+++ b/cylc/flow/scripts/lint.py
@@ -150,15 +150,16 @@
 
 
 DEPRECATED_STRING_TEMPLATES = {
-    'suite': 'workflow',
-    'suite_uuid': 'uuid',
-    'batch_sys_name': 'job_runner_name',
-    'batch_sys_job_id': 'job_id',
-    'user@host': 'platform_name',
-    'task_url': '``URL`` (if set in :cylc:conf:`[meta]URL`)',
-    'workflow_url': (
-        '``workflow_URL`` (if set in '
-        ':cylc:conf:`[runtime][<namespace>][meta]URL`)'),
+    'suite': ['workflow'],
+    'suite_uuid': ['uuid'],
+    'batch_sys_name': ['job_runner_name'],
+    'batch_sys_job_id': ['job_id'],
+    'user@host': ['platform_name'],
+    'task_url': ['URL', '(if set in :cylc:conf:`[meta]URL`)'],
+    'workflow_url': [
+        'workflow_URL',
+        '(if set in :cylc:conf:`[runtime][<namespace>][meta]URL`)',
+    ],
 }
 
 
@@ -359,16 +360,19 @@ def check_for_deprecated_task_event_template_vars(
         >>> this('hello = "My name is %(suite)s"')
         {'suggest': '\\n    * %(suite)s ⇒ %(workflow)s'}
 
-        >>> this('hello = "My name is %(suite)s, %(task_url)s"')
-        {'suggest': '\\n    * %(suite)s ⇒ %(workf...cylc:conf:`[meta]URL`)'}
+        >>> this('x = %(user@host)s, %(suite)')
+        {'suggest': '\\n    * %(user@host)s ⇒ %(platform_name)s'}
+
+        >>> this('x = %(task_url)s')
+        {'suggest': '\\n    * %(task_url)s ⇒ %(URL)s (if set in ...)'}
     """
-    result = []
-    for key, (regex, replacement) in deprecated_string_templates.items():
-        search_outcome = regex.findall(line)
-        if search_outcome and ' ' in replacement:
-            result.append(f'%({key})s - get {replacement}')
-        elif search_outcome:
-            result.append(f'%({key})s ⇒ %({replacement})s')
+    result = [
+        f"%({key})s ⇒ %({replacement})s {' '.join(extra)}".rstrip()
+        for key, (
+            regex, (replacement, *extra),
+        ) in deprecated_string_templates.items()
+        if regex.findall(line)
+    ]
 
     if result:
         return {'suggest': LIST_ITEM + LIST_ITEM.join(result)}
@@ -603,22 +607,17 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]:
             ' rather than job runner directive: ``{directive}``.'
         ),
         'rst': (
-            'Using ``[runtime][TASK]execution time limit`` is'
-            ' recommended in preference to using job runner'
-            ' directives because it allows Cylc to retain awareness'
-            ' of whether the job should have finished, even if contact'
-            ' with the target job runner\'s platform has been lost.'
-            ' \n\nThe following directives are considered equivelent to'
-            ' execution time limit:\n * '
+            "Use :cylc:conf:`flow.cylc[runtime][<namespace>]execution "
+            "time limit` rather than directly specifying a timeout "
+            "directive, otherwise Cylc has no way of knowing when the job "
+            "should have finished. Cylc automatically translates the "
+            "execution time limit to the correct timeout directive for the "
+            "particular job runner:\n"
         )
-        + '\n * '.join((
-            f'``{directive}`` ({job_runner})'
+        + ''.join((
+            f'\n * ``{directive}`` ({job_runner})'
             for job_runner, directive in WALLCLOCK_DIRECTIVES.items()
-        )) + (
-            '\n\n.. note:: Using ``execution time limit`` which'
-            ' is automatically translated to the job runner\'s timeout'
-            ' directive can make your workflow more portable.'
-        ),
+        )),
         FUNCTION: check_wallclock_directives,
     }
 }
@@ -762,12 +761,12 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]:
         'short': (
             'Deprecated template variables: {suggest}'),
         'rst': (
-            'The following template variables, mostly used in event handlers,'
-            'are deprecated, and should be replaced:'
-            + ''.join([
-                f'\n * ``{old}`` ⇒ {new}'
-                for old, new in DEPRECATED_STRING_TEMPLATES.items()
-            ])
+            "The following deprecated template variables, mostly used in "
+            "event handlers, should be replaced:\n"
+            + ''.join(
+                f"\n * ``{old}`` ⇒ ``{new}`` {' '.join(extra)}".rstrip()
+                for old, (new, *extra) in DEPRECATED_STRING_TEMPLATES.items()
+            )
         ),
         'url': (
             'https://cylc.github.io/cylc-doc/stable/html/user-guide/'

From cbc859483975e8454f0387b9e2d36fb6af9c0f40 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Wed, 23 Oct 2024 13:34:25 +0100
Subject: [PATCH 50/53] gracefully handle SIGINT, SIGTERM and SIGHUP

* Closes #4914
* Closes #6438
* If any of these signals are received, set the workflow to stop NOW.
* If the workflow is already stopping NOW, escalate that to NOW NOW
  (e.g. to handle stuck subprocpool processes)
* SIGINT is no longer handled as an exception (panic shutdown).
---
 changes.d/6444.feat.md              |  1 +
 cylc/flow/scheduler.py              | 45 ++++++++++++++++++++--
 cylc/flow/scheduler_cli.py          |  2 -
 tests/functional/rnd/01-signals.t   | 60 +++++++++++++++++++++++++++++
 tests/integration/test_scheduler.py | 28 +++++++++++++-
 5 files changed, 129 insertions(+), 7 deletions(-)
 create mode 100644 changes.d/6444.feat.md
 create mode 100644 tests/functional/rnd/01-signals.t

diff --git a/changes.d/6444.feat.md b/changes.d/6444.feat.md
new file mode 100644
index 00000000000..efa310dfa0b
--- /dev/null
+++ b/changes.d/6444.feat.md
@@ -0,0 +1 @@
+The scheduler now traps the SIGINT, SIGTERM and SIGHUP signals and will respond by shutting down in --now mode. If the workflow is already shutting down in --now mode, it will escalate the shutdown to --now --now mode.
diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py
index 92702b0b55e..d3f7fd18a36 100644
--- a/cylc/flow/scheduler.py
+++ b/cylc/flow/scheduler.py
@@ -22,6 +22,7 @@
 from pathlib import Path
 from queue import Empty, Queue
 from shlex import quote
+import signal
 from socket import gaierror
 from subprocess import DEVNULL, PIPE, Popen
 import sys
@@ -671,7 +672,7 @@ async def run_scheduler(self) -> None:
                 LOG.exception(exc)
                 raise
 
-        except (KeyboardInterrupt, asyncio.CancelledError) as exc:
+        except asyncio.CancelledError as exc:
             await self.handle_exception(exc)
 
         except CylcError as exc:  # Includes SchedulerError
@@ -747,12 +748,37 @@ async def run(self):
         """Run the startup sequence and set the main loop running.
 
         Lightweight wrapper for testing convenience.
-
         """
+        # register signal handlers
+        for sig in (
+            # ctrl+c
+            signal.SIGINT,
+            # the "stop please" signal
+            signal.SIGTERM,
+            # the controlling process bailed
+            # (e.g. terminal quit in no detach mode)
+            signal.SIGHUP,
+        ):
+            signal.signal(sig, self._handle_signal)
+
         await self.start()
         # note run_scheduler handles its own shutdown logic
         await self.run_scheduler()
 
+    def _handle_signal(self, sig: int, frame) -> None:
+        """Handle a signal."""
+        LOG.critical(f'Signal {signal.Signals(sig).name} received')
+        if sig in {signal.SIGINT, signal.SIGTERM, signal.SIGHUP}:
+            if self.stop_mode == StopMode.REQUEST_NOW:
+                # already shutting down NOW -> escalate to NOW_NOW
+                # (i.e. don't run/wait for event handlers)
+                stop_mode = StopMode.REQUEST_NOW_NOW
+            else:
+                # stop NOW (orphan running tasks)
+                stop_mode = StopMode.REQUEST_NOW
+
+            self._set_stop(stop_mode)
+
     def _load_pool_from_tasks(self):
         """Load task pool with specified tasks, for a new run."""
         LOG.info(f"Start task: {self.options.starttask}")
@@ -1688,9 +1714,21 @@ async def shutdown(self, reason: BaseException) -> None:
         # At the moment this method must be called from the main_loop.
         # In the future it should shutdown the main_loop itself but
         # we're not quite there yet.
+
+        # cancel signal handlers
+        def _handle_signal(sig, frame):
+            LOG.warning(
+                f'Signal {signal.Signals(sig).name} received,'
+                ' already shutting down'
+            )
+
+        for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP):
+            signal.signal(sig, _handle_signal)
+
         try:
+            # action shutdown
             await self._shutdown(reason)
-        except (KeyboardInterrupt, asyncio.CancelledError, Exception) as exc:
+        except (asyncio.CancelledError, Exception) as exc:
             # In case of exception in the shutdown method itself.
             LOG.error("Error during shutdown")
             # Suppress the reason for shutdown, which is logged separately
@@ -1712,7 +1750,6 @@ async def _shutdown(self, reason: BaseException) -> None:
             try:
                 self.proc_pool.close()
                 if self.proc_pool.is_not_done():
-                    # e.g. KeyboardInterrupt
                     self.proc_pool.terminate()
                 self.proc_pool.process()
             except Exception as exc:
diff --git a/cylc/flow/scheduler_cli.py b/cylc/flow/scheduler_cli.py
index 390eba52338..601dd3112b6 100644
--- a/cylc/flow/scheduler_cli.py
+++ b/cylc/flow/scheduler_cli.py
@@ -655,8 +655,6 @@ async def _run(scheduler: Scheduler) -> int:
     # stop cylc stop
     except SchedulerError:
         ret = 1
-    except (KeyboardInterrupt, asyncio.CancelledError):
-        ret = 2
     except Exception:
         ret = 3
 
diff --git a/tests/functional/rnd/01-signals.t b/tests/functional/rnd/01-signals.t
new file mode 100644
index 00000000000..125727c8a21
--- /dev/null
+++ b/tests/functional/rnd/01-signals.t
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#-------------------------------------------------------------------------------
+# Test scheduler signal handling
+# # See https://github.com/cylc/cylc-flow/issues/6438
+
+. "$(dirname "$0")/test_header"
+set_test_number 6
+
+
+init_workflow "${TEST_NAME_BASE}" <<__FLOW__
+[scheduling]
+    [[graph]]
+        R1 = foo
+[runtime]
+    [[foo]]
+__FLOW__
+
+
+# test signals on a detached scheduler
+TEST_NAME="${TEST_NAME_BASE}-detatch"
+cylc play "${WORKFLOW_NAME}" --pause
+poll_workflow_running
+PID="$(sed -n 's/CYLC_WORKFLOW_PID=//p' "$HOME/cylc-run/$WORKFLOW_NAME/.service/contact")"
+kill -s SIGINT "$PID"
+poll_workflow_stopped
+log_scan "${TEST_NAME}" "$(cylc cat-log -m p "${WORKFLOW_NAME}")" 10 1 \
+    'Signal SIGINT received' \
+    'Workflow shutting down - REQUEST(NOW)' \
+    'DONE'
+
+
+# test signals on a non-detached scheduler
+TEST_NAME="${TEST_NAME_BASE}-no-detach"
+cylc play "${WORKFLOW_NAME}" --pause --no-detach 2>/dev/null &
+poll_workflow_running
+PID="$(sed -n 's/CYLC_WORKFLOW_PID=//p' "$HOME/cylc-run/$WORKFLOW_NAME/.service/contact")"
+kill -s SIGTERM "$PID"
+poll_workflow_stopped
+log_scan "${TEST_NAME}" "$(cylc cat-log -m p "${WORKFLOW_NAME}")" 10 1 \
+    'Signal SIGTERM received' \
+    'Workflow shutting down - REQUEST(NOW)' \
+    'DONE'
+
+
+purge
diff --git a/tests/integration/test_scheduler.py b/tests/integration/test_scheduler.py
index 502c13af8f6..79b7327206a 100644
--- a/tests/integration/test_scheduler.py
+++ b/tests/integration/test_scheduler.py
@@ -19,6 +19,7 @@
 from pathlib import Path
 import pytest
 import re
+from signal import SIGHUP, SIGINT, SIGTERM
 from typing import Any, Callable
 
 from cylc.flow import commands
@@ -33,7 +34,7 @@
     TASK_STATUS_FAILED
 )
 
-from cylc.flow.workflow_status import AutoRestartMode
+from cylc.flow.workflow_status import AutoRestartMode, StopMode
 
 
 Fixture = Any
@@ -374,3 +375,28 @@ async def test_restart_timeout(
         schd.pool.force_trigger_tasks(['1/one'], {1})
         await asyncio.sleep(0)  # yield control to the main loop
         assert log_filter(log, contains='restart timer stopped')
+
+
+@pytest.mark.parametrize("signal", ((SIGHUP), (SIGINT), (SIGTERM)))
+async def test_signal_escallation(one, start, signal, log_filter):
+    """Double signal should escalate shutdown.
+
+    If a term-like signal is received whilst the workflow is already stopping
+    in NOW mode, then the shutdown should be escalated to NOW_NOW
+    mode.
+
+    See https://github.com/cylc/cylc-flow/pull/6444
+    """
+    async with start(one) as log:
+        # put the workflow in the stopping state
+        one._set_stop(StopMode.REQUEST_CLEAN)
+        assert one.stop_mode.name == 'REQUEST_CLEAN'
+
+        # one signal should escalate this from CLEAN to NOW
+        one._handle_signal(signal, None)
+        assert log_filter(log, contains=signal.name)
+        assert one.stop_mode.name == 'REQUEST_NOW'
+
+        # two signals should escalate this from NOW to NOW_NOW
+        one._handle_signal(signal, None)
+        assert one.stop_mode.name == 'REQUEST_NOW_NOW'

From 5bfb487c657251b9e8ef4a13fec07a8cbb5c1884 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 29 Oct 2024 13:21:16 +0000
Subject: [PATCH 51/53] data store: don't update outputs incrementally

* We don't have a protocol for updating arbitrary lists incrementally.
* See https://github.com/cylc/cylc-flow/issues/6307
* This is a temporary patch to disable incremental output updates until
  we have a suitable protocol for doing so.
---
 cylc/flow/data_store_mgr.py              |  38 ++++----
 tests/integration/test_data_store_mgr.py | 116 ++++++++++++++++++++++-
 2 files changed, 134 insertions(+), 20 deletions(-)

diff --git a/cylc/flow/data_store_mgr.py b/cylc/flow/data_store_mgr.py
index 8a2d23018a4..a4b28d44fdc 100644
--- a/cylc/flow/data_store_mgr.py
+++ b/cylc/flow/data_store_mgr.py
@@ -2420,23 +2420,27 @@ def delta_task_output(
                 objects from the workflow task pool.
 
         """
-        tproxy: Optional[PbTaskProxy]
-        tp_id, tproxy = self.store_node_fetcher(itask.tokens)
-        if not tproxy:
-            return
-        outputs = itask.state.outputs
-        label = outputs.get_trigger(message)
-        # update task instance
-        update_time = time()
-        tp_delta = self.updated[TASK_PROXIES].setdefault(
-            tp_id, PbTaskProxy(id=tp_id))
-        tp_delta.stamp = f'{tp_id}@{update_time}'
-        output = tp_delta.outputs[label]
-        output.label = label
-        output.message = message
-        output.satisfied = outputs.is_message_complete(message)
-        output.time = update_time
-        self.updates_pending = True
+        # TODO: Restore incremental update when we have a protocol to do so
+        # https://github.com/cylc/cylc-flow/issues/6307
+        return self.delta_task_outputs(itask)
+
+        # tproxy: Optional[PbTaskProxy]
+        # tp_id, tproxy = self.store_node_fetcher(itask.tokens)
+        # if not tproxy:
+        #     return
+        # outputs = itask.state.outputs
+        # label = outputs.get_trigger(message)
+        # # update task instance
+        # update_time = time()
+        # tp_delta = self.updated[TASK_PROXIES].setdefault(
+        #     tp_id, PbTaskProxy(id=tp_id))
+        # tp_delta.stamp = f'{tp_id}@{update_time}'
+        # output = tp_delta.outputs[label]
+        # output.label = label
+        # output.message = message
+        # output.satisfied = outputs.is_message_complete(message)
+        # output.time = update_time
+        # self.updates_pending = True
 
     def delta_task_outputs(self, itask: TaskProxy) -> None:
         """Create delta for change in all task proxy outputs.
diff --git a/tests/integration/test_data_store_mgr.py b/tests/integration/test_data_store_mgr.py
index cb19fb1fe23..4f45e3c0b0e 100644
--- a/tests/integration/test_data_store_mgr.py
+++ b/tests/integration/test_data_store_mgr.py
@@ -14,7 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from typing import Iterable, List, cast
+from typing import Iterable, List, TYPE_CHECKING, cast
 
 import pytest
 
@@ -29,6 +29,12 @@
 )
 from cylc.flow.id import Tokens
 from cylc.flow.scheduler import Scheduler
+from cylc.flow.task_events_mgr import TaskEventsManager
+from cylc.flow.task_outputs import (
+    TASK_OUTPUT_SUBMITTED,
+    TASK_OUTPUT_STARTED,
+    TASK_OUTPUT_SUCCEEDED,
+)
 from cylc.flow.task_state import (
     TASK_STATUS_FAILED,
     TASK_STATUS_SUCCEEDED,
@@ -37,8 +43,13 @@
 from cylc.flow.wallclock import get_current_time_string
 
 
-# NOTE: These tests mutate the data store, so running them in isolation may
-# see failures when they actually pass if you run the whole file
+if TYPE_CHECKING:
+    from cylc.flow.scheduler import Scheduler
+
+
+# NOTE: Some of these tests mutate the data store, so running them in
+# isolation may see failures when they actually pass if you run the
+# whole file
 
 
 def job_config(schd):
@@ -392,3 +403,102 @@ async def test_flow_numbers(flow, scheduler, start):
         ds_task = schd.data_store_mgr.get_data_elements(TASK_PROXIES).added[1]
         assert ds_task.name == 'b'
         assert ds_task.flow_nums == '[2]'
+
+
+async def test_delta_task_outputs(one: 'Scheduler', start):
+    """Ensure task outputs are inserted into the store.
+
+    Note: Task outputs should *not* be updated incrementally until we have
+    a protocol for doing so, see https://github.com/cylc/cylc-flow/pull/6403
+    """
+
+    def get_data_outputs():
+        """Return satisfied outputs from the *data* store."""
+        nonlocal one, itask
+        return {
+            output.label
+            for output in one.data_store_mgr.data[one.id][TASK_PROXIES][
+                itask.tokens.id
+            ].outputs.values()
+            if output.satisfied
+        }
+
+    def get_delta_outputs():
+        """Return satisfied outputs from the *delta* store.
+
+        Or return None if there's nothing there.
+        """
+        nonlocal one, itask
+        try:
+            return {
+                output.label
+                for output in one.data_store_mgr.updated[TASK_PROXIES][
+                    itask.tokens.id
+                ].outputs.values()
+                if output.satisfied
+            }
+        except KeyError:
+            return None
+
+    def _patch_remove(*args, **kwargs):
+        """Prevent the task/workflow from completing."""
+        pass
+
+    async with start(one):
+        one.pool.remove = _patch_remove
+
+        # create a job submission
+        itask = one.pool.get_tasks()[0]
+        assert itask
+        itask.submit_num += 1
+        one.data_store_mgr.insert_job(
+            itask.tdef.name, itask.point, itask.state.status, {'submit_num': 1}
+        )
+        await one.update_data_structure()
+
+        # satisfy the submitted & started outputs
+        # (note started implies submitted)
+        one.task_events_mgr.process_message(
+            itask, 'INFO', TaskEventsManager.EVENT_STARTED
+        )
+
+        # the delta should be populated with the newly satisfied outputs
+        assert get_data_outputs() == set()
+        assert get_delta_outputs() == {
+            TASK_OUTPUT_SUBMITTED,
+            TASK_OUTPUT_STARTED,
+        }
+
+        # the delta should be applied to the store
+        await one.update_data_structure()
+        assert get_data_outputs() == {
+            TASK_OUTPUT_SUBMITTED,
+            TASK_OUTPUT_STARTED,
+        }
+        assert get_delta_outputs() is None
+
+        # satisfy the succeeded output
+        one.task_events_mgr.process_message(
+            itask, 'INFO', TaskEventsManager.EVENT_SUCCEEDED
+        )
+
+        # the delta should be populated with ALL satisfied outputs
+        # (not just the newly satisfied output)
+        assert get_data_outputs() == {
+            TASK_OUTPUT_SUBMITTED,
+            TASK_OUTPUT_STARTED,
+        }
+        assert get_delta_outputs() == {
+            TASK_OUTPUT_SUBMITTED,
+            TASK_OUTPUT_STARTED,
+            TASK_OUTPUT_SUCCEEDED,
+        }
+
+        # the delta should be applied to the store
+        await one.update_data_structure()
+        assert get_data_outputs() == {
+            TASK_OUTPUT_SUBMITTED,
+            TASK_OUTPUT_STARTED,
+            TASK_OUTPUT_SUCCEEDED,
+        }
+        assert get_delta_outputs() is None

From 478b0d0a77efff25d27fa9ac621281df25921d22 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 1 Nov 2024 14:42:04 +0000
Subject: [PATCH 52/53] Bump pypa/gh-action-pypi-publish from 1.10.3 to 1.11.0
 (#6455) [skip ci]

---
 .github/workflows/2_auto_publish_release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/2_auto_publish_release.yml b/.github/workflows/2_auto_publish_release.yml
index 1ab1e1cce42..5110cb632c3 100644
--- a/.github/workflows/2_auto_publish_release.yml
+++ b/.github/workflows/2_auto_publish_release.yml
@@ -38,7 +38,7 @@ jobs:
       uses: cylc/release-actions/build-python-package@v1
 
     - name: Publish distribution to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.10.3
+      uses: pypa/gh-action-pypi-publish@v1.11.0
       with:
         user: __token__ # uses the API token feature of PyPI - least permissions possible
         password: ${{ secrets.PYPI_TOKEN }}

From 7fe91e8626d238095a06c790144b1e081ff4c9d6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 7 Nov 2024 09:14:34 +0000
Subject: [PATCH 53/53] Bump pypa/gh-action-pypi-publish from 1.11.0 to 1.12.2

Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.11.0 to 1.12.2.
- [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases)
- [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.11.0...v1.12.2)

---
updated-dependencies:
- dependency-name: pypa/gh-action-pypi-publish
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/2_auto_publish_release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/2_auto_publish_release.yml b/.github/workflows/2_auto_publish_release.yml
index 5110cb632c3..011eb090202 100644
--- a/.github/workflows/2_auto_publish_release.yml
+++ b/.github/workflows/2_auto_publish_release.yml
@@ -38,7 +38,7 @@ jobs:
       uses: cylc/release-actions/build-python-package@v1
 
     - name: Publish distribution to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.11.0
+      uses: pypa/gh-action-pypi-publish@v1.12.2
       with:
         user: __token__ # uses the API token feature of PyPI - least permissions possible
         password: ${{ secrets.PYPI_TOKEN }}