Improve handling of missing responses.json

src/ert/dark_storage/endpoints/experiments.py

@@ -22,11 +22,14 @@ def get_experiments(
         js.ExperimentOut(
             id=experiment.id,
             name=experiment.name,
-            ensemble_ids=[ens.id for ens in experiment.ensembles],
+            ensemble_ids=[
+                ens.id for ens in experiment.ensembles if experiment.is_valid()
+            ],
             priors=create_priors(experiment),
             userdata={},
         )
         for experiment in storage.experiments
+        if experiment.is_valid()
     ]
 
 

src/ert/dark_storage/endpoints/records.py

@@ -124,8 +124,8 @@ def get_ensemble_responses(
 
     response_names_with_observations = set()
     observations = ensemble.experiment.observations
-
-    if len(ensemble.has_data()) == 0:
+    print(f"TRYING TO GET_ENSEMBLE_RESPONSES for {ensemble.name}")
+    if not ensemble.experiment.is_valid() or len(ensemble.has_data()) == 0:
         return {}
 
     for (

src/ert/gui/ertnotifier.py

@@ -54,6 +54,7 @@ def emitErtChange(self) -> None:
     def set_storage(self, storage: Storage) -> None:
         self._storage = storage
         self.storage_changed.emit(storage)
+        print("EMITTED STORAGE CHANGED")
 
     @Slot(object)
     def set_current_ensemble(self, ensemble: Ensemble | None = None) -> None:
@@ -63,3 +64,10 @@ def set_current_ensemble(self, ensemble: Ensemble | None = None) -> None:
     @Slot(bool)
     def set_is_simulation_running(self, is_running: bool) -> None:
         self._is_simulation_running = is_running
+
+    def refresh(self) -> None:
+        print("FORCED REFRESH")
+        if self._storage is None:
+            return
+        self._storage.refresh()
+        self.storage_changed.emit(self._storage)

src/ert/gui/ertwidgets/ensembleselector.py

@@ -4,9 +4,11 @@
 from typing import TYPE_CHECKING
 
 from qtpy.QtCore import Qt, Signal
+from qtpy.QtGui import QStandardItemModel
 from qtpy.QtWidgets import QComboBox
 
 from ert.gui.ertnotifier import ErtNotifier
+from ert.storage.local_ensemble import LocalEnsemble
 from ert.storage.realization_storage_state import RealizationStorageState
 
 if TYPE_CHECKING:
@@ -22,6 +24,7 @@
         update_ert: bool = True,
         show_only_undefined: bool = False,
         show_only_no_children: bool = False,
+        show_only_with_valid_experiment: bool = False,
     ):
         super().__init__()
         self.notifier = notifier
@@ -36,6 +39,7 @@
         # if the ensemble has not been used in an update, as that would
         # invalidate the result
         self._show_only_no_children = show_only_no_children
+        self._show_only_with_valid_experiment = show_only_with_valid_experiment
         self.setSizeAdjustPolicy(QComboBox.AdjustToContents)
 
         self.setEnabled(False)
@@ -57,7 +61,9 @@
 
     @property
     def selected_ensemble(self) -> Ensemble:
-        return self.itemData(self.currentIndex())
+        itemData: LocalEnsemble | None = self.itemData(self.currentIndex())
+        assert itemData is not None
+        return itemData
 
     def populate(self) -> None:
         block = self.blockSignals(True)
@@ -68,9 +74,27 @@
             self.setEnabled(True)
 
         for ensemble in self._ensemble_list():
+            model = self.model()
+            assert isinstance(model, QStandardItemModel)
+            assert model is not None
             self.addItem(
                 f"{ensemble.experiment.name} : {ensemble.name}", userData=ensemble
             )
+            if (
+                self._show_only_with_valid_experiment
+                and not ensemble.experiment.is_valid()
+            ):
+                index = self.count() - 1
+                model_item = model.item(index)
+                assert model_item is not None
+                model_item.setFlags(
+                    model_item.flags()
+                    & ~Qt.ItemFlag.ItemIsEnabled
+                    & ~Qt.ItemFlag.ItemIsSelectable
+                )
+                self.setItemData(
+                    index, "This ensemble is invalid", Qt.ItemDataRole.ToolTipRole
+                )
 
         current_index = self.findData(
             self.notifier.current_ensemble, Qt.ItemDataRole.UserRole
@@ -79,7 +103,6 @@
         self.setCurrentIndex(max(current_index, 0))
 
         self.blockSignals(block)
-
         self.ensemble_populated.emit()
 
     def _ensemble_list(self) -> Iterable[Ensemble]:
@@ -95,6 +118,8 @@
         else:
             ensembles = self.notifier.storage.ensembles
         ensemble_list = list(ensembles)
+        if self._show_only_with_valid_experiment:
+            ensemble_list = [ens for ens in ensemble_list if ens.experiment.is_valid()]
         if self._show_only_no_children:
             parents = [
                 ens.parent for ens in self.notifier.storage.ensembles if ens.parent

src/ert/gui/main_window.py

@@ -153,6 +153,7 @@ def right_clicked(self) -> None:
     def select_central_widget(self) -> None:
         actor = self.sender()
         if actor:
+            self.notifier.refresh()
             index_name = actor.property("index")
 
             for widget in self.central_panels_map.values():

src/ert/gui/simulation/evaluate_ensemble_panel.py

@@ -35,7 +35,9 @@ def __init__(self, ensemble_size: int, run_path: str, notifier: ErtNotifier):
         lab.setWordWrap(True)
         lab.setAlignment(QtCore.Qt.AlignmentFlag.AlignLeft)
         layout.addRow(lab)
-        self._ensemble_selector = EnsembleSelector(notifier, show_only_no_children=True)
+        self._ensemble_selector = EnsembleSelector(
+            notifier, show_only_no_children=True, show_only_with_valid_experiment=True
+        )
         layout.addRow("Ensemble:", self._ensemble_selector)
         runpath_label = CopyableLabel(text=run_path)
         layout.addRow("Runpath:", runpath_label)

src/ert/gui/tools/manage_experiments/manage_experiments_panel.py

@@ -47,7 +47,7 @@ def __init__(self, config: ErtConfig, notifier: ErtNotifier, ensemble_size: int)
     def _add_create_new_ensemble_tab(self) -> None:
         panel = QWidget()
         panel.setObjectName("create_new_ensemble_tab")
-
+        print(f"{self.ensemble_size=}")
         layout = QHBoxLayout()
         storage_widget = StorageWidget(
             self.notifier, self.ert_config, self.ensemble_size
@@ -75,7 +75,11 @@ def _add_initialize_from_scratch_tab(self) -> None:
 
         ensemble_layout = QHBoxLayout()
         ensemble_label = QLabel("Target ensemble:")
-        ensemble_selector = EnsembleSelector(self.notifier, show_only_undefined=True)
+        ensemble_selector = EnsembleSelector(
+            self.notifier,
+            show_only_undefined=True,
+            show_only_with_valid_experiment=True,
+        )
         ensemble_selector.setMinimumWidth(300)
         ensemble_layout.addWidget(ensemble_label)
         ensemble_layout.addWidget(ensemble_selector)

src/ert/gui/tools/manage_experiments/storage_model.py

@@ -63,6 +63,7 @@
         self._id = ensemble.id
         self._start_time = ensemble.started_at
         self._children: list[RealizationModel] = []
+        self._error = ensemble.experiment.error_message
 
     def add_realization(self, realization: RealizationModel) -> None:
         self._children.append(realization)
@@ -75,25 +76,29 @@
     def data(self, index: QModelIndex, role: Qt.ItemDataRole) -> Any:
         if not index.isValid():
             return None
-
         col = index.column()
         if role == Qt.ItemDataRole.DisplayRole:
             if col == _Column.NAME:
                 return self._name
             if col == _Column.TIME:
                 return humanize.naturaltime(self._start_time)
         elif role == Qt.ItemDataRole.ToolTipRole:
+            if self._error:
+                print("FOUND ERROR TOOLTIP")
+                return self._error
             if col == _Column.TIME:
                 return str(self._start_time)
-
         return None
 
 
-class ExperimentModel:
-    def __init__(self, experiment: Experiment, parent: Any):
+class ExperimentModel(QAbstractItemModel):
+    def __init__(self, experiment: Experiment, parent: "StorageModel"):
         self._parent = parent
         self._id = experiment.id
         self._name = experiment.name
+        self._is_valid = experiment.is_valid()
+        self._error = experiment.error_message
+        print(f"{self._is_valid=}")
         self._experiment_type = experiment.metadata.get("ensemble_type")
         self._children: list[EnsembleModel] = []
 
@@ -106,7 +111,7 @@
         return 0
 
     def data(
         self, index: QModelIndex, role: Qt.ItemDataRole = Qt.ItemDataRole.DisplayRole
     ) -> Any:
         if not index.isValid():
             return None
@@ -128,7 +133,11 @@
                 qapp = QApplication.instance()
                 assert isinstance(qapp, QApplication)
                 return qapp.palette().mid()
-
+        if role == Qt.ItemDataRole.ToolTipRole:
+            print("TRYING TO GET TOOLTIP")
+            if self._error:
+                print("FOUND ERROR TOOLTIP")
+                return self._error
         return None
 
 
@@ -140,6 +149,7 @@
 
     @Slot(Storage)
     def reloadStorage(self, storage: Storage) -> None:
+        print("RELOADED STORAGE")
         self.beginResetModel()
         self._load_storage(storage)
         self.endResetModel()
@@ -211,9 +221,31 @@
     def data(self, index: QModelIndex, role: int = Qt.ItemDataRole.DisplayRole) -> Any:
         if not index.isValid():
             return None
-
         return index.internalPointer().data(index, role)
 
+    @override
+    def flags(self, index: QModelIndex) -> Qt.ItemFlag:
+        default_flags = super().flags(index)
+
+        if not index.isValid():
+            return default_flags
+        item = index.internalPointer()
+        if isinstance(item, ExperimentModel) and not item._is_valid:
+            return default_flags & ~Qt.ItemFlag.ItemIsEnabled
+        return default_flags
+
+    @override
+    def hasChildren(self, parent: QModelIndex | None = None) -> bool:
+        if parent is None or not parent.isValid():
+            return True
+
+        flags = self.flags(parent)
+        # hide children if disabled
+        if not (flags & Qt.ItemFlag.ItemIsEnabled):
+            return False
+
+        return super().hasChildren(parent)
+
     @override
     def index(
         self, row: int, column: int, parent: QModelIndex | None = None

src/ert/gui/tools/manage_experiments/storage_widget.py

@@ -100,6 +100,9 @@ def __init__(
         self._tree_view = QTreeView(self)
         storage_model = StorageModel(self._notifier.storage)
         notifier.storage_changed.connect(storage_model.reloadStorage)
+        notifier.storage_changed.connect(
+            lambda *args, **kwargs: print("Storage changed")
+        )
         notifier.ertChanged.connect(
             lambda: storage_model.reloadStorage(self._notifier.storage)
         )
@@ -108,7 +111,7 @@ def __init__(
         search_bar.setPlaceholderText("Filter")
         proxy_model = _SortingProxyModel(storage_model)
         proxy_model.setFilterKeyColumn(-1)  # Search all columns.
-        proxy_model.setSourceModel(storage_model)
+        proxy_model.setSourceModel(storage_model)  # JONAK - CAN THIS BE REMOVED?
         proxy_model.sort(0, Qt.SortOrder.AscendingOrder)
 
         self._tree_view.setModel(proxy_model)

src/ert/gui/tools/plot/plot_api.py

@@ -112,7 +112,7 @@ def all_data_type_keys(self) -> list[PlotApiKeyDefinition]:
                     f"/experiments/{experiment['id']}/ensembles", timeout=self._timeout
                 )
                 self._check_response(response)
-
+                print(f"{experiment=}")
                 for ensemble in response.json():
                     response = client.get(
                         f"/ensembles/{ensemble['id']}/responses", timeout=self._timeout

src/ert/storage/local_ensemble.py

@@ -92,6 +92,7 @@ def create_realization_dir(realization: int) -> Path:
             return self._path / f"realization-{realization}"
 
         self._realization_dir = create_realization_dir
+        self.has_valid_experiment: bool | None = None
 
     @classmethod
     def create(
@@ -406,7 +407,11 @@ def get_ensemble_state(self) -> list[set[RealizationStorageState]]:
         states : list of RealizationStorageState
             list of realization states.
         """
-
+        if not self.experiment.is_valid():
+            logger.warning(
+                f"Could not get ensemble state for ensemble ({self.id}) due to invalid experiment ({self.experiment_id}): {self.experiment.error_message}"
+            )
+            return []
         response_configs = self.experiment.response_configuration
 
         def _parameters_exist_for_realization(realization: int) -> bool:

src/ert/storage/local_experiment.py

@@ -75,6 +75,26 @@ def __init__(
         self._index = _Index.model_validate_json(
             (path / "index.json").read_text(encoding="utf-8")
         )
+        self._validate_files()
+
+    def _validate_files(self) -> None:
+        self.valid_parameters = (self._path / self._parameter_file).exists()
+        self.valid_responses = (self._path / self._responses_file).exists()
+        self.valid_metadata = (self._path / self._metadata_file).exists()
+
+    def is_valid(self) -> bool:
+        return self.valid_parameters and self.valid_responses and self.valid_metadata
+
+    @property
+    def error_message(self) -> str:
+        errors = []
+        if not self.valid_parameters:
+            errors.append("Parameter file is missing")
+        if not self.valid_responses:
+            errors.append("Responses file is missing")
+        if not self.valid_metadata:
+            errors.append("Metadata file is missing")
+        return "\n".join(errors)
 
     @classmethod
     def create(

src/ert/storage/local_storage.py

@@ -226,11 +226,15 @@ def _load_ensembles(self) -> dict[UUID, LocalEnsemble]:
         }
 
     def _load_experiments(self) -> dict[UUID, LocalExperiment]:
-        experiment_ids = {ens.experiment_id for ens in self._ensembles.values()}
-        return {
-            exp_id: LocalExperiment(self, self._experiment_path(exp_id), self.mode)
-            for exp_id in experiment_ids
-        }
+        experiments = {}
+        for ens in self._ensembles.values():
+            experiment = LocalExperiment(
+                self, self._experiment_path(ens.experiment_id), self.mode
+            )
+            ens.has_valid_experiment = experiment.is_valid()
+            experiments[ens.experiment_id] = experiment
+
+        return experiments
 
     def _ensemble_path(self, ensemble_id: UUID) -> Path:
         return self.path / self.ENSEMBLES_PATH / str(ensemble_id)

tests/ert/ui_tests/gui/test_main_window.py

@@ -798,3 +798,33 @@ def wait_for_simulation_completed():
     assert len(button_simulation_status.menu().actions()) == 3
     for choice in button_simulation_status.menu().actions():
         assert "Single realization test-run" in choice.text()
+
+
+def test_that_invalid_experiments_are_disabled(opened_main_window_poly, qtbot):
+    gui = opened_main_window_poly
+
+    def find_and_click_button(button_name: str):
+        button = gui.findChild(QToolButton, button_name)
+        qtbot.mouseClick(button, Qt.LeftButton)
+
+    def run_experiment():
+        run_experiment_panel = wait_for_child(gui, qtbot, ExperimentPanel)
+        qtbot.wait_until(lambda: not run_experiment_panel.isHidden(), timeout=5000)
+        assert run_experiment_panel.run_button.isEnabled()
+        qtbot.mouseClick(run_experiment_panel.run_button, Qt.LeftButton)
+
+    def wait_for_simulation_completed():
+        run_dialogs = get_children(gui, RunDialog)
+        dialog = run_dialogs[-1]
+        qtbot.wait_until(lambda: not dialog.isHidden(), timeout=5000)
+        qtbot.wait_until(lambda: dialog.is_simulation_done() == True, timeout=15000)
+
+    find_and_click_button("button_Start_simulation")
+
+    run_experiment()
+    wait_for_simulation_completed()
+
+    # make sure experiment is valid in manage experiments panel
+    button_manage_experiments = gui.findChild(QToolButton, "button_Manage_experiments")
+    assert button_manage_experiments
+    qtbot.mouseClick(button_manage_experiments, Qt.LeftButton)