Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor - Remove import_scenario export_scenario APIs #1432

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions taipy/core/_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@
delete_job,
delete_jobs,
exists,
export_scenario,
get,
get_cycles,
get_cycles_scenarios,
Expand All @@ -52,7 +51,6 @@
get_sequences,
get_submissions,
get_tasks,
import_scenario,
is_deletable,
is_editable,
is_promotable,
Expand Down
12 changes: 0 additions & 12 deletions taipy/core/_manager/_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.

import pathlib
from typing import Dict, Generic, Iterable, List, Optional, TypeVar, Union

from taipy.logger._taipy_logger import _TaipyLogger
Expand Down Expand Up @@ -153,17 +152,6 @@ def _delete_entities_of_multiple_types(cls, _entity_ids: _EntityIds):
_DataManagerFactory._build_manager()._delete_many(_entity_ids.data_node_ids)
_SubmissionManagerFactory._build_manager()._delete_many(_entity_ids.submission_ids)

@classmethod
def _export(cls, id: str, folder_path: Union[str, pathlib.Path], **kwargs):
return cls._repository._export(id, folder_path)

@classmethod
def _import(cls, entity_file: pathlib.Path, version: str, **kwargs) -> EntityType:
imported_entity = cls._repository._import(entity_file)
imported_entity._version = version
cls._set(imported_entity)
return imported_entity

@classmethod
def _is_editable(cls, entity: Union[EntityType, str]) -> bool:
return True
Expand Down
15 changes: 0 additions & 15 deletions taipy/core/_version/_version_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.

import pathlib
import uuid
from typing import List, Optional, Union

Expand Down Expand Up @@ -231,17 +230,3 @@ def __check_production_migration_config(cls):
@classmethod
def _delete_entities_of_multiple_types(cls, _entity_ids):
raise NotImplementedError

@classmethod
def _import(cls, entity_file: pathlib.Path, version: str, **kwargs) -> _Version:
imported_version = cls._repository._import(entity_file)

comparator_result = Config._comparator._find_conflict_config( # type: ignore[attr-defined]
imported_version.config,
Config._applied_config, # type: ignore[attr-defined]
imported_version.id,
)
if comparator_result.get(_ComparatorResult.CONFLICTED_SECTION_KEY):
raise ConflictedConfigurationError()

return imported_version
45 changes: 0 additions & 45 deletions taipy/core/data/_data_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@
# specific language governing permissions and limitations under the License.

import os
import pathlib
import shutil
from typing import Dict, Iterable, List, Optional, Set, Union

from taipy.config._config import _Config
Expand Down Expand Up @@ -178,46 +176,3 @@ def _get_by_config_id(cls, config_id: str, version_number: Optional[str] = None)
for fil in filters:
fil.update({"config_id": config_id})
return cls._repository._load_all(filters)

@classmethod
def _export(cls, id: str, folder_path: Union[str, pathlib.Path], **kwargs) -> None:
cls._repository._export(id, folder_path)

if not kwargs.get("include_data"):
return

data_node = cls._get(id)
if not isinstance(data_node, _FileDataNodeMixin):
cls._logger.warning(f"Data node {id} is not a file-based data node and the data will not be exported.")
return

if isinstance(folder_path, str):
folder: pathlib.Path = pathlib.Path(folder_path)
else:
folder = folder_path

data_export_dir = folder / Config.core.storage_folder / os.path.dirname(data_node.path)
if not data_export_dir.exists():
data_export_dir.mkdir(parents=True)

data_export_path = data_export_dir / os.path.basename(data_node.path)
if os.path.exists(data_node.path):
shutil.copy2(data_node.path, data_export_path)

@classmethod
def _import(cls, entity_file: pathlib.Path, version: str, **kwargs) -> DataNode:
imported_data_node = cls._repository._import(entity_file)
imported_data_node._version = version
cls._set(imported_data_node)

if not (isinstance(imported_data_node, _FileDataNodeMixin) and isinstance(imported_data_node, DataNode)):
return imported_data_node

data_folder: pathlib.Path = pathlib.Path(str(kwargs.get("data_folder")))
if not data_folder.exists():
return imported_data_node

if (data_folder / imported_data_node.path).exists():
shutil.copy2(data_folder / imported_data_node.path, imported_data_node.path)

return imported_data_node
41 changes: 0 additions & 41 deletions taipy/core/exceptions/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -373,47 +373,6 @@ class FileEmpty(Exception):
"""Raised when a file is empty."""


class ExportPathAlreadyExists(Exception):
"""Raised when the export folder already exists."""

def __init__(self, export_path: str, scenario_id: str):
self.message = (
f"The path '{export_path}' already exists and can not be used to export scenario '{scenario_id}'."
" Please use the 'override' parameter to override it."
)


class EntitiesToBeImportAlredyExist(Exception):
"""Raised when entities in the scenario to be imported have already exists"""

def __init__(self, import_path):
self.message = f"The import archive file {import_path} contains entities that have already existed."


class DataToBeImportAlredyExist(Exception):
"""Raised when data files in the scenario to be imported have already exists"""

def __init__(self, import_path):
self.message = (
f"The import archive file {import_path} contains data files that have already existed."
" Please use the 'override' parameter to override those."
)


class ImportArchiveDoesntContainAnyScenario(Exception):
"""Raised when the import archive file doesn't contain any scenario"""

def __init__(self, import_path):
self.message = f"The import archive file {import_path} doesn't contain any scenario."


class ImportScenarioDoesntHaveAVersion(Exception):
"""Raised when the import scenario doesn't have a version"""

def __init__(self, import_path):
self.message = f"The import scenario in the import archive file {import_path} doesn't have a version."


class SQLQueryCannotBeExecuted(Exception):
"""Raised when an SQL Query cannot be executed."""

Expand Down
91 changes: 1 addition & 90 deletions taipy/core/scenario/_scenario_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,14 @@
# specific language governing permissions and limitations under the License.

import datetime
import pathlib
import tempfile
import zipfile
from functools import partial
from typing import Any, Callable, Dict, List, Literal, Optional, Type, Union
from typing import Any, Callable, Dict, List, Literal, Optional, Union

from taipy.config import Config

from .._entity._entity_ids import _EntityIds
from .._manager._manager import _Manager
from .._repository._abstract_repository import _AbstractRepository
from .._version._version_manager_factory import _VersionManagerFactory
from .._version._version_mixin import _VersionMixin
from ..common.warn_if_inputs_not_ready import _warn_if_inputs_not_ready
from ..config.scenario_config import ScenarioConfig
Expand All @@ -32,9 +28,6 @@
DeletingPrimaryScenario,
DifferentScenarioConfigs,
DoesNotBelongToACycle,
EntitiesToBeImportAlredyExist,
ImportArchiveDoesntContainAnyScenario,
ImportScenarioDoesntHaveAVersion,
InsufficientScenarioToCompare,
InvalidScenario,
NonExistingComparator,
Expand Down Expand Up @@ -475,85 +468,3 @@ def _get_by_config_id(cls, config_id: str, version_number: Optional[str] = None)
for fil in filters:
fil.update({"config_id": config_id})
return cls._repository._load_all(filters)

@classmethod
def _import_scenario_and_children_entities(
cls,
zip_file_path: pathlib.Path,
override: bool,
entity_managers: Dict[str, Type[_Manager]],
) -> Optional[Scenario]:
with tempfile.TemporaryDirectory() as tmp_dir:
with zipfile.ZipFile(zip_file_path) as zip_file:
zip_file.extractall(tmp_dir)

tmp_dir_path = pathlib.Path(tmp_dir)

if not ((tmp_dir_path / "scenarios").exists() or (tmp_dir_path / "scenario").exists()):
raise ImportArchiveDoesntContainAnyScenario(zip_file_path)

if not (tmp_dir_path / "version").exists():
raise ImportScenarioDoesntHaveAVersion(zip_file_path)

# Import the version to check for compatibility
entity_managers["version"]._import(next((tmp_dir_path / "version").iterdir()), "")

valid_entity_folders = list(entity_managers.keys())
valid_data_folder = Config.core.storage_folder

imported_scenario = None
imported_entities: Dict[str, List] = {}

for entity_folder in tmp_dir_path.iterdir():
if not entity_folder.is_dir() or entity_folder.name not in valid_entity_folders + [valid_data_folder]:
cls._logger.warning(f"{entity_folder} is not a valid Taipy folder and will not be imported.")
continue

try:
for entity_type in valid_entity_folders:
# Skip the version folder as it is already handled
if entity_type == "version":
continue

entity_folder = tmp_dir_path / entity_type
if not entity_folder.exists():
continue

manager = entity_managers[entity_type]
imported_entities[entity_type] = []

for entity_file in entity_folder.iterdir():
# Check if the to-be-imported entity already exists
entity_id = entity_file.stem
if manager._exists(entity_id):
if override:
cls._logger.warning(f"{entity_id} already exists and will be overridden.")
else:
cls._logger.error(
f"{entity_id} already exists. Please use the 'override' parameter to override it."
)
raise EntitiesToBeImportAlredyExist(zip_file_path)

# Import the entity
imported_entity = manager._import(
entity_file,
version=_VersionManagerFactory._build_manager()._get_latest_version(),
data_folder=tmp_dir_path / valid_data_folder,
)

imported_entities[entity_type].append(imported_entity.id)
if entity_type in ["scenario", "scenarios"]:
imported_scenario = imported_entity
except Exception as err:
cls._logger.error(f"An error occurred during the import: {err}. Rollback the import.")

# Rollback the import
for entity_type, entity_ids in list(imported_entities.items())[::-1]:
manager = entity_managers[entity_type]
for entity_id in entity_ids:
if manager._exists(entity_id):
manager._delete(entity_id)
raise err

cls._logger.info(f"Scenario {imported_scenario.id} has been successfully imported.") # type: ignore[union-attr]
return imported_scenario
25 changes: 0 additions & 25 deletions taipy/core/scenario/scenario.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
# specific language governing permissions and limitations under the License.
from __future__ import annotations

import pathlib
import uuid
from datetime import datetime
from typing import Any, Callable, Dict, List, Optional, Set, Union
Expand Down Expand Up @@ -615,30 +614,6 @@ def submit(

return _ScenarioManagerFactory._build_manager()._submit(self, callbacks, force, wait, timeout, **properties)

def export(
self,
folder_path: Union[str, pathlib.Path],
override: bool = False,
include_data: bool = False,
):
"""Export all related entities of this scenario to a folder.

Parameters:
folder_path (Union[str, pathlib.Path]): The folder path to export the scenario to.
If the path exists and the override parameter is False, an exception is raised.
override (bool): If True, the existing folder will be overridden. Default is False.
include_data (bool): If True, the file-based data nodes are exported as well.
This includes Pickle, CSV, Excel, Parquet, and JSON data nodes.
If the scenario has a data node that is not file-based, a warning will be logged, and the data node
will not be exported. The default value is False.

Raises:
ExportPathAlreadyExists^: If the `folder_path` already exists and the override parameter is False.
"""
from ... import core as tp

return tp.export_scenario(self.id, folder_path, override, include_data)

def set_primary(self):
"""Promote the scenario as the primary scenario of its cycle.

Expand Down
28 changes: 0 additions & 28 deletions taipy/core/sequence/_sequence_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,6 @@
# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.

import json
import pathlib
from functools import partial
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union

Expand Down Expand Up @@ -398,32 +396,6 @@ def _exists(cls, entity_id: str) -> bool:
"""
return True if cls._get(entity_id) else False

@classmethod
def _export(cls, id: str, folder_path: Union[str, pathlib.Path], **kwargs) -> None:
"""
Export a Sequence entity.
"""
if isinstance(folder_path, str):
folder: pathlib.Path = pathlib.Path(folder_path)
else:
folder = folder_path

export_dir = folder / cls._model_name
if not export_dir.exists():
export_dir.mkdir(parents=True)

export_path = export_dir / f"{id}.json"
sequence_name, scenario_id = cls._breakdown_sequence_id(id)
sequence = {"id": id, "owner_id": scenario_id, "parent_ids": [scenario_id], "name": sequence_name}

scenario = _ScenarioManagerFactory._build_manager()._get(scenario_id)
if sequence_data := scenario._sequences.get(sequence_name, None):
sequence.update(sequence_data)
with open(export_path, "w", encoding="utf-8") as export_file:
export_file.write(json.dumps(sequence))
else:
raise ModelNotFound(cls._model_name, id)

@classmethod
def __log_error_entity_not_found(cls, sequence_id: Union[SequenceId, str]):
cls._logger.error(f"{cls._ENTITY_NAME} not found: {str(sequence_id)}")
Expand Down
Loading
Loading