diff --git a/examples/notebooks/direct_job_execution.ipynb b/examples/notebooks/direct_job_execution.ipynb
index 20d4c725..19cfd73f 100644
--- a/examples/notebooks/direct_job_execution.ipynb
+++ b/examples/notebooks/direct_job_execution.ipynb
@@ -37,19 +37,15 @@
    "outputs": [],
    "source": [
     "from lm_buddy import LMBuddy\n",
-    "from lm_buddy.jobs.configs import (\n",
-    "    FinetuningJobConfig,\n",
-    "    FinetuningRayConfig,\n",
-    "    LMHarnessJobConfig,\n",
-    "    LMHarnessEvaluationConfig,\n",
-    ")\n",
-    "from lm_buddy.integrations.huggingface import (\n",
+    "from lm_buddy.configs.jobs.finetuning import FinetuningJobConfig, FinetuningRayConfig\n",
+    "from lm_buddy.configs.jobs.lm_harness import LMHarnessJobConfig, LMHarnessEvaluationConfig\n",
+    "from lm_buddy.configs.huggingface import (\n",
     "    AutoModelConfig,\n",
-    "    TextDatasetConfig,\n",
+    "    DatasetConfig,\n",
     "    TrainerConfig,\n",
     "    AdapterConfig,\n",
     ")\n",
-    "from lm_buddy.integrations.wandb import WandbRunConfig"
+    "from lm_buddy.configs.wandb import WandbRunConfig"
    ]
   },
   {
@@ -69,7 +65,7 @@
     "model_config = AutoModelConfig(path=\"hf://distilgpt2\")\n",
     "\n",
     "# Text dataset for finetuning\n",
-    "dataset_config = TextDatasetConfig(\n",
+    "dataset_config = DatasetConfig(\n",
     "    path=\"hf://imdb\",\n",
     "    split=\"train[:100]\",\n",
     "    text_field=\"text\",\n",
diff --git a/pyproject.toml b/pyproject.toml
index 76a42cde..73d90f17 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "lm-buddy"
-version = "0.8.0"
+version = "0.9.0"
 authors = [
     { name = "Sean Friedowitz", email = "sean@mozilla.ai" },
     { name = "Aaron Gonzales", email = "aaron@mozilla.ai" },
diff --git a/src/lm_buddy/buddy.py b/src/lm_buddy/buddy.py
index 956ca789..4e6be5d3 100644
--- a/src/lm_buddy/buddy.py
+++ b/src/lm_buddy/buddy.py
@@ -1,17 +1,20 @@
 import wandb
 
-from lm_buddy.integrations.wandb import WandbResumeMode
-from lm_buddy.jobs._entrypoints import run_finetuning, run_lm_harness, run_prometheus, run_ragas
-from lm_buddy.jobs.common import EvaluationResult, FinetuningResult, LMBuddyJobType
-from lm_buddy.jobs.configs import (
+from lm_buddy.configs.jobs import (
     EvaluationJobConfig,
     FinetuningJobConfig,
-    LMBuddyJobConfig,
+    JobConfig,
     LMHarnessJobConfig,
     PrometheusJobConfig,
     RagasJobConfig,
 )
+from lm_buddy.jobs.common import EvaluationResult, FinetuningResult, JobType
+from lm_buddy.jobs.evaluation.lm_harness import run_lm_harness
+from lm_buddy.jobs.evaluation.prometheus import run_prometheus
+from lm_buddy.jobs.evaluation.ragas import run_ragas
+from lm_buddy.jobs.finetuning import run_finetuning
 from lm_buddy.paths import strip_path_prefix
+from lm_buddy.tracking.run_utils import WandbResumeMode
 
 
 class LMBuddy:
@@ -25,10 +28,7 @@ def __init__(self):
         pass
 
     def _generate_artifact_lineage(
-        self,
-        config: LMBuddyJobConfig,
-        results: list[wandb.Artifact],
-        job_type: LMBuddyJobType,
+        self, config: JobConfig, results: list[wandb.Artifact], job_type: JobType
     ) -> None:
         """Link input artifacts and log output artifacts to a run.
 
@@ -51,7 +51,7 @@ def _generate_artifact_lineage(
     def finetune(self, config: FinetuningJobConfig) -> FinetuningResult:
         """Run a supervised finetuning job with the provided configuration."""
         result = run_finetuning(config)
-        self._generate_artifact_lineage(config, result.artifacts, LMBuddyJobType.FINETUNING)
+        self._generate_artifact_lineage(config, result.artifacts, JobType.FINETUNING)
         return result
 
     def evaluate(self, config: EvaluationJobConfig) -> EvaluationResult:
@@ -68,5 +68,5 @@ def evaluate(self, config: EvaluationJobConfig) -> EvaluationResult:
                 result = run_ragas(ragas_config)
             case _:
                 raise ValueError(f"Invlid configuration for evaluation: {type(config)}")
-        self._generate_artifact_lineage(config, result.artifacts, LMBuddyJobType.EVALUATION)
+        self._generate_artifact_lineage(config, result.artifacts, JobType.EVALUATION)
         return result
diff --git a/src/lm_buddy/cli/evaluate.py b/src/lm_buddy/cli/evaluate.py
index aff3b6ee..73f58a60 100644
--- a/src/lm_buddy/cli/evaluate.py
+++ b/src/lm_buddy/cli/evaluate.py
@@ -2,7 +2,7 @@
 
 from lm_buddy import LMBuddy
 from lm_buddy.cli.utils import parse_config_option
-from lm_buddy.jobs.configs import LMHarnessJobConfig, PrometheusJobConfig, RagasJobConfig
+from lm_buddy.configs.jobs import LMHarnessJobConfig, PrometheusJobConfig, RagasJobConfig
 
 
 @click.group(name="evaluate", help="Run an LM Buddy evaluation job.")
diff --git a/src/lm_buddy/cli/finetune.py b/src/lm_buddy/cli/finetune.py
index d63ab781..6ff720a9 100644
--- a/src/lm_buddy/cli/finetune.py
+++ b/src/lm_buddy/cli/finetune.py
@@ -2,7 +2,7 @@
 
 from lm_buddy import LMBuddy
 from lm_buddy.cli.utils import parse_config_option
-from lm_buddy.jobs.configs import FinetuningJobConfig
+from lm_buddy.configs.jobs import FinetuningJobConfig
 
 
 @click.command(name="finetune", help="Run an LM Buddy finetuning job.")
diff --git a/src/lm_buddy/cli/utils.py b/src/lm_buddy/cli/utils.py
index 66a0831c..10654be8 100644
--- a/src/lm_buddy/cli/utils.py
+++ b/src/lm_buddy/cli/utils.py
@@ -1,9 +1,9 @@
 from pathlib import Path
 from typing import TypeVar
 
-from lm_buddy.jobs.configs.base import LMBuddyJobConfig
+from lm_buddy.configs.jobs.common import JobConfig
 
-ConfigType = TypeVar("ConfigType", bound=LMBuddyJobConfig)
+ConfigType = TypeVar("ConfigType", bound=JobConfig)
 
 
 def parse_config_option(config_cls: type[ConfigType], config: str) -> ConfigType:
diff --git a/src/lm_buddy/integrations/__init__.py b/src/lm_buddy/configs/__init__.py
similarity index 100%
rename from src/lm_buddy/integrations/__init__.py
rename to src/lm_buddy/configs/__init__.py
diff --git a/src/lm_buddy/types.py b/src/lm_buddy/configs/common.py
similarity index 53%
rename from src/lm_buddy/types.py
rename to src/lm_buddy/configs/common.py
index ce42b7de..de15c882 100644
--- a/src/lm_buddy/types.py
+++ b/src/lm_buddy/configs/common.py
@@ -1,7 +1,11 @@
+import contextlib
+import tempfile
+from pathlib import Path
 from typing import Annotated, Any
 
 import torch
 from pydantic import BaseModel, BeforeValidator, PlainSerializer, WithJsonSchema
+from pydantic_yaml import parse_yaml_file_as, to_yaml_file
 
 
 def validate_torch_dtype(x: Any) -> torch.dtype:
@@ -28,7 +32,7 @@ def validate_torch_dtype(x: Any) -> torch.dtype:
 """
 
 
-class BaseLMBuddyConfig(
+class LMBuddyConfig(
     BaseModel,
     extra="forbid",
     arbitrary_types_allowed=True,
@@ -38,3 +42,26 @@ class BaseLMBuddyConfig(
 
     Defines some common settings used by all subclasses.
     """
+
+    @classmethod
+    def from_yaml_file(cls, path: Path | str):
+        return parse_yaml_file_as(cls, path)
+
+    def to_yaml_file(self, path: Path | str):
+        to_yaml_file(path, self, exclude_none=True)
+
+    @contextlib.contextmanager
+    def to_tempfile(self, *, name: str = "config.yaml", dir: str | Path | None = None):
+        """Enter a context manager with the config written to a temporary YAML file.
+
+        Keyword Args:
+            name (str): Name of the config file in the tmp directory. Defaults to "config.yaml".
+            dir (str | Path | None): Root path of the temporary directory. Defaults to None.
+
+        Returns:
+            Path to the temporary config file.
+        """
+        with tempfile.TemporaryDirectory(dir=dir) as tmpdir:
+            config_path = Path(tmpdir) / name
+            self.to_yaml_file(config_path)
+            yield config_path
diff --git a/src/lm_buddy/configs/huggingface.py b/src/lm_buddy/configs/huggingface.py
new file mode 100644
index 00000000..ef7696ca
--- /dev/null
+++ b/src/lm_buddy/configs/huggingface.py
@@ -0,0 +1,158 @@
+import dataclasses
+from typing import Any
+
+from peft import PeftConfig, PeftType, TaskType
+from pydantic import field_validator, model_validator
+from transformers import BitsAndBytesConfig
+
+from lm_buddy.configs.common import LMBuddyConfig, SerializableTorchDtype
+from lm_buddy.paths import AssetPath, PathPrefix
+
+DEFAULT_TEXT_FIELD: str = "text"
+
+
+class AutoModelConfig(LMBuddyConfig):
+    """Settings passed to a HuggingFace AutoModel instantiation.
+
+    The model to load can either be a HuggingFace repo or an artifact reference on W&B.
+    """
+
+    path: AssetPath
+    trust_remote_code: bool = False
+    torch_dtype: SerializableTorchDtype | None = None
+
+
+class AutoTokenizerConfig(LMBuddyConfig):
+    """Settings passed to a HuggingFace AutoTokenizer instantiation."""
+
+    path: AssetPath
+    trust_remote_code: bool | None = None
+    use_fast: bool | None = None
+
+
+class DatasetConfig(LMBuddyConfig):
+    """Settings passed to load a HuggingFace text dataset.
+
+    The dataset can either contain a single text column named by the `text_field` parameter,
+    or a `prompt_template` can be provided to format columns of the dataset as the `text_field`.
+    """
+
+    path: AssetPath
+    text_field: str = DEFAULT_TEXT_FIELD
+    prompt_template: str | None = None
+    split: str | None = None
+    test_size: float | None = None
+    seed: int | None = None
+
+    @model_validator(mode="after")
+    def validate_split_if_huggingface_path(cls, config: "DatasetConfig"):
+        """
+        Ensure a  `split` is provided when loading a HuggingFace dataset directly from HF Hub.
+        This makes it such that the `load_dataset` function returns the type `Dataset`
+        instead of `DatasetDict`, which makes some of the downstream logic easier.
+        """
+        if config.split is None and config.path.startswith(PathPrefix.HUGGINGFACE):
+            raise ValueError(
+                "A `split` must be specified when loading a dataset directly from HuggingFace."
+            )
+        return config
+
+
+class AdapterConfig(LMBuddyConfig, extra="allow"):
+    """Configuration containing PEFT adapter settings.
+
+    The type of adapter is controlled by the required `peft_type` field,
+    which must be one of the allowed values from the PEFT `PeftType` enumeration.
+    Extra arguments are allowed and are passed down to the HuggingFace `PeftConfig`
+    class determined by the `peft_type` argument.
+
+    The `task_type` for the adapter is also required.
+    By default, this is set to `TaskType.CAUSAL_LM`
+    which is appropriate for causal language model finetuning.
+    See the allowed values in the PEFT `TaskType` enumeration.
+    """
+
+    peft_type: PeftType
+    task_type: TaskType = TaskType.CAUSAL_LM
+
+    @staticmethod
+    def _get_peft_config_class(peft_type: PeftType) -> type[PeftConfig]:
+        # Internal import to avoid bringing the global variable from peft into module scope
+        from peft.mapping import PEFT_TYPE_TO_CONFIG_MAPPING
+
+        return PEFT_TYPE_TO_CONFIG_MAPPING[peft_type]
+
+    @field_validator("peft_type", "task_type", mode="before")
+    def sanitize_enum_args(cls, x):
+        if isinstance(x, str):
+            x = x.strip().upper()
+        return x
+
+    @model_validator(mode="after")
+    def validate_adapter_args(cls, config: "AdapterConfig"):
+        peft_type = config.peft_type
+
+        # PeftConfigs are standard dataclasses so can extract their allowed field names
+        adapter_cls = cls._get_peft_config_class(peft_type)
+        allowed_fields = {x.name for x in dataclasses.fields(adapter_cls)}
+
+        # Filter fields to those found on the PeftConfig
+        extra_fields = config.model_fields_set.difference(allowed_fields)
+        if extra_fields:
+            raise ValueError(f"Unknowon arguments for {peft_type} adapter: {extra_fields}")
+
+        return config
+
+    def as_huggingface(self) -> PeftConfig:
+        adapter_cls = self._get_peft_config_class(self.peft_type)
+        return adapter_cls(**self.model_dump())
+
+
+class QuantizationConfig(LMBuddyConfig):
+    """Basic quantization settings to pass to training and evaluation jobs.
+
+    Note that in order to use BitsAndBytes quantization on Ray,
+    you must ensure that the runtime environment is installed with GPU support.
+    This can be configured by setting the `entrypoint_num_gpus > 0` when submitting a job
+    to the cluster.
+    """
+
+    load_in_8bit: bool | None = None
+    load_in_4bit: bool | None = None
+    bnb_4bit_quant_type: str = "fp4"
+    bnb_4bit_compute_dtype: SerializableTorchDtype | None = None
+
+    def as_huggingface(self) -> BitsAndBytesConfig:
+        return BitsAndBytesConfig(
+            load_in_4bit=self.load_in_4bit,
+            load_in_8bit=self.load_in_8bit,
+            bnb_4bit_compute_dtype=self.bnb_4bit_compute_dtype,
+            bnb_4bit_quant_type=self.bnb_4bit_quant_type,
+        )
+
+
+class TrainerConfig(LMBuddyConfig):
+    """Configuration for a HuggingFace trainer/training arguments.
+
+    This mainly encompasses arguments passed to the HuggingFace `TrainingArguments` class,
+    but also contains some additional parameters for the `Trainer` or `SFTTrainer` classes.
+    """
+
+    max_seq_length: int | None = None
+    num_train_epochs: float | None = None
+    per_device_train_batch_size: int | None = None
+    per_device_eval_batch_size: int | None = None
+    learning_rate: float | None = None
+    weight_decay: float | None = None
+    gradient_accumulation_steps: int | None = None
+    gradient_checkpointing: bool | None = None
+    evaluation_strategy: str | None = None
+    eval_steps: float | None = None
+    logging_strategy: str | None = None
+    logging_steps: float | None = None
+    save_strategy: str | None = None
+    save_steps: int | None = None
+
+    def training_args(self) -> dict[str, Any]:
+        """Return the arguments to the HuggingFace `TrainingArguments` class."""
+        return self.model_dump(exclude={"max_seq_length"}, exclude_none=True)
diff --git a/src/lm_buddy/configs/jobs/__init__.py b/src/lm_buddy/configs/jobs/__init__.py
new file mode 100644
index 00000000..b05459b5
--- /dev/null
+++ b/src/lm_buddy/configs/jobs/__init__.py
@@ -0,0 +1,16 @@
+from lm_buddy.configs.jobs.common import JobConfig
+from lm_buddy.configs.jobs.finetuning import FinetuningJobConfig
+from lm_buddy.configs.jobs.lm_harness import LMHarnessJobConfig
+from lm_buddy.configs.jobs.prometheus import PrometheusJobConfig
+from lm_buddy.configs.jobs.ragas import RagasJobConfig
+
+EvaluationJobConfig = LMHarnessJobConfig | PrometheusJobConfig | RagasJobConfig
+
+__all__ = [
+    "JobConfig",
+    "FinetuningJobConfig",
+    "LMHarnessJobConfig",
+    "PrometheusJobConfig",
+    "RagasJobConfig",
+    "EvaluationJobConfig",
+]
diff --git a/src/lm_buddy/jobs/configs/base.py b/src/lm_buddy/configs/jobs/common.py
similarity index 93%
rename from src/lm_buddy/jobs/configs/base.py
rename to src/lm_buddy/configs/jobs/common.py
index 417d5157..e096d358 100644
--- a/src/lm_buddy/jobs/configs/base.py
+++ b/src/lm_buddy/configs/jobs/common.py
@@ -6,12 +6,12 @@
 from pydantic import Field
 from pydantic_yaml import parse_yaml_file_as, to_yaml_file
 
-from lm_buddy.integrations.wandb import WandbRunConfig
+from lm_buddy.configs.common import LMBuddyConfig
+from lm_buddy.configs.wandb import WandbRunConfig
 from lm_buddy.paths import AssetPath, PathPrefix
-from lm_buddy.types import BaseLMBuddyConfig
 
 
-class LMBuddyJobConfig(BaseLMBuddyConfig):
+class JobConfig(LMBuddyConfig):
     """Configuration that comprises the entire input to an LM Buddy job.
 
     This class implements helper methods for de/serializing the configuration from file.
diff --git a/src/lm_buddy/jobs/configs/finetuning.py b/src/lm_buddy/configs/jobs/finetuning.py
similarity index 88%
rename from src/lm_buddy/jobs/configs/finetuning.py
rename to src/lm_buddy/configs/jobs/finetuning.py
index dce71419..a020614d 100644
--- a/src/lm_buddy/jobs/configs/finetuning.py
+++ b/src/lm_buddy/configs/jobs/finetuning.py
@@ -1,19 +1,19 @@
 from pydantic import Field, field_validator, model_validator
 
-from lm_buddy.integrations.huggingface import (
+from lm_buddy.configs.common import LMBuddyConfig
+from lm_buddy.configs.huggingface import (
     AdapterConfig,
     AutoModelConfig,
     AutoTokenizerConfig,
+    DatasetConfig,
     QuantizationConfig,
-    TextDatasetConfig,
     TrainerConfig,
 )
-from lm_buddy.jobs.configs import LMBuddyJobConfig
+from lm_buddy.configs.jobs.common import JobConfig
 from lm_buddy.paths import AssetPath
-from lm_buddy.types import BaseLMBuddyConfig
 
 
-class FinetuningRayConfig(BaseLMBuddyConfig):
+class FinetuningRayConfig(LMBuddyConfig):
     """Misc settings passed to Ray train for finetuning.
 
     Includes information for scaling, checkpointing, and runtime storage.
@@ -24,11 +24,11 @@ class FinetuningRayConfig(BaseLMBuddyConfig):
     storage_path: str | None = None  # TODO: This should be set globally somehow
 
 
-class FinetuningJobConfig(LMBuddyJobConfig):
+class FinetuningJobConfig(JobConfig):
     """Configuration to submit an LLM finetuning job."""
 
     model: AutoModelConfig
-    dataset: TextDatasetConfig
+    dataset: DatasetConfig
     tokenizer: AutoTokenizerConfig
     quantization: QuantizationConfig | None = None
     adapter: AdapterConfig | None = None
diff --git a/src/lm_buddy/jobs/configs/lm_harness.py b/src/lm_buddy/configs/jobs/lm_harness.py
similarity index 81%
rename from src/lm_buddy/jobs/configs/lm_harness.py
rename to src/lm_buddy/configs/jobs/lm_harness.py
index 88af2446..41830b94 100644
--- a/src/lm_buddy/jobs/configs/lm_harness.py
+++ b/src/lm_buddy/configs/jobs/lm_harness.py
@@ -2,17 +2,14 @@
 
 from pydantic import conlist, model_validator
 
-from lm_buddy.integrations.huggingface import (
-    AutoModelConfig,
-    QuantizationConfig,
-)
-from lm_buddy.integrations.vllm import InferenceServerConfig
-from lm_buddy.jobs.configs import LMBuddyJobConfig
+from lm_buddy.configs.common import LMBuddyConfig
+from lm_buddy.configs.huggingface import AutoModelConfig, QuantizationConfig
+from lm_buddy.configs.jobs.common import JobConfig
+from lm_buddy.configs.vllm import InferenceServerConfig
 from lm_buddy.paths import AssetPath
-from lm_buddy.types import BaseLMBuddyConfig
 
 
-class LocalChatCompletionsConfig(BaseLMBuddyConfig):
+class LocalChatCompletionsConfig(LMBuddyConfig):
     """Configuration for a "local-chat-completions" model in lm-harness.
 
     The "local-chat-completions" model is powered by a self-hosted inference server,
@@ -35,7 +32,7 @@ def validate_inference_engine(cls, config: "LocalChatCompletionsConfig"):
         return config
 
 
-class LMHarnessEvaluationConfig(BaseLMBuddyConfig):
+class LMHarnessEvaluationConfig(LMBuddyConfig):
     """Misc settings provided to an lm-harness evaluation job."""
 
     tasks: conlist(str, min_length=1)
@@ -44,7 +41,7 @@ class LMHarnessEvaluationConfig(BaseLMBuddyConfig):
     limit: int | float | None = None
 
 
-class LMHarnessJobConfig(LMBuddyJobConfig):
+class LMHarnessJobConfig(JobConfig):
     """Configuration to run an lm-evaluation-harness evaluation job."""
 
     model: AutoModelConfig | LocalChatCompletionsConfig
diff --git a/src/lm_buddy/jobs/configs/prometheus.py b/src/lm_buddy/configs/jobs/prometheus.py
similarity index 74%
rename from src/lm_buddy/jobs/configs/prometheus.py
rename to src/lm_buddy/configs/jobs/prometheus.py
index 008622d6..dd1b608a 100644
--- a/src/lm_buddy/jobs/configs/prometheus.py
+++ b/src/lm_buddy/configs/jobs/prometheus.py
@@ -1,13 +1,13 @@
 from pydantic import Field
 
-from lm_buddy.integrations.huggingface import TextDatasetConfig
-from lm_buddy.integrations.vllm import VLLMCompletionsConfig
-from lm_buddy.jobs.configs import LMBuddyJobConfig
+from lm_buddy.configs.common import LMBuddyConfig
+from lm_buddy.configs.huggingface import DatasetConfig
+from lm_buddy.configs.jobs.common import JobConfig
+from lm_buddy.configs.vllm import VLLMCompletionsConfig
 from lm_buddy.paths import AssetPath
-from lm_buddy.types import BaseLMBuddyConfig
 
 
-class PrometheusEvaluationConfig(BaseLMBuddyConfig):
+class PrometheusEvaluationConfig(LMBuddyConfig):
     """Parameters specific to Prometheus evaluation."""
 
     num_answers: int = 3
@@ -21,13 +21,13 @@ class PrometheusEvaluationConfig(BaseLMBuddyConfig):
     conversation_system_message: str = "You are a fair evaluator language model."
 
 
-class PrometheusJobConfig(LMBuddyJobConfig):
+class PrometheusJobConfig(JobConfig):
     """Configuration for a Prometheus judge evaluation task."""
 
     prometheus: VLLMCompletionsConfig = Field(
         description="Externally hosted Prometheus judge model."
     )
-    dataset: TextDatasetConfig = Field(
+    dataset: DatasetConfig = Field(
         description="Dataset of text completions to evaluate using the Prometheus judge model."
     )
     evaluation: PrometheusEvaluationConfig = Field(
diff --git a/src/lm_buddy/jobs/configs/ragas.py b/src/lm_buddy/configs/jobs/ragas.py
similarity index 81%
rename from src/lm_buddy/jobs/configs/ragas.py
rename to src/lm_buddy/configs/jobs/ragas.py
index ae200413..b0f22a1c 100644
--- a/src/lm_buddy/jobs/configs/ragas.py
+++ b/src/lm_buddy/configs/jobs/ragas.py
@@ -2,12 +2,11 @@
 
 from pydantic import Field, field_validator
 
-from lm_buddy.integrations.huggingface import AutoModelConfig
-from lm_buddy.integrations.huggingface.dataset_config import TextDatasetConfig
-from lm_buddy.integrations.vllm import VLLMCompletionsConfig
-from lm_buddy.jobs.configs.base import LMBuddyJobConfig
+from lm_buddy.configs.common import LMBuddyConfig
+from lm_buddy.configs.huggingface import AutoModelConfig, DatasetConfig
+from lm_buddy.configs.jobs.common import JobConfig
+from lm_buddy.configs.vllm import VLLMCompletionsConfig
 from lm_buddy.paths import AssetPath
-from lm_buddy.types import BaseLMBuddyConfig
 
 RagasEvaluationMetric = Literal[
     "faithfulness",
@@ -17,7 +16,7 @@
 ]
 
 
-class RagasEvaluationConfig(BaseLMBuddyConfig):
+class RagasEvaluationConfig(LMBuddyConfig):
     """Parameters specifically required for RAGAs Evaluation"""
 
     metrics: list[RagasEvaluationMetric] = Field(
@@ -43,7 +42,7 @@ def validate_embedding_model_arg(cls, x):
         return x
 
 
-class RagasJobConfig(LMBuddyJobConfig):
+class RagasJobConfig(JobConfig):
     """Configuration to run a Ragas evaluation job.
 
     This job loads a dataset from an existing path on our cluster.
@@ -52,7 +51,7 @@ class RagasJobConfig(LMBuddyJobConfig):
     """
 
     judge: VLLMCompletionsConfig = Field(description="Externally hosted Ragas judge model.")
-    dataset: TextDatasetConfig = Field(
+    dataset: DatasetConfig = Field(
         description="Dataset of text completions to evaluate using the Ragas judge model."
     )
     evaluation: RagasEvaluationConfig = Field(
diff --git a/src/lm_buddy/integrations/vllm.py b/src/lm_buddy/configs/vllm.py
similarity index 89%
rename from src/lm_buddy/integrations/vllm.py
rename to src/lm_buddy/configs/vllm.py
index e376366d..431f011d 100644
--- a/src/lm_buddy/integrations/vllm.py
+++ b/src/lm_buddy/configs/vllm.py
@@ -1,8 +1,8 @@
+from lm_buddy.configs.common import LMBuddyConfig
 from lm_buddy.paths import AssetPath
-from lm_buddy.types import BaseLMBuddyConfig
 
 
-class InferenceServerConfig(BaseLMBuddyConfig):
+class InferenceServerConfig(LMBuddyConfig):
     """Generic configuration for an externally hosted inference endpoint.
 
     The inference server is defined by an endpoint with the provided `base_url`.
@@ -16,10 +16,10 @@ class InferenceServerConfig(BaseLMBuddyConfig):
     """
 
     base_url: str
-    engine: AssetPath | None = None
+    engine: AssetPath
 
 
-class VLLMCompletionsConfig(BaseLMBuddyConfig):
+class VLLMCompletionsConfig(LMBuddyConfig):
     """Configuration for a vLLM-based completions service
 
     The "local-chat-completions" model is powered by a self-hosted inference server,
@@ -33,7 +33,6 @@ class VLLMCompletionsConfig(BaseLMBuddyConfig):
     """
 
     inference: InferenceServerConfig
-
     # vLLM-specific params
     best_of: int | None = None
     max_tokens: int | None = None
diff --git a/src/lm_buddy/integrations/wandb/run_config.py b/src/lm_buddy/configs/wandb.py
similarity index 96%
rename from src/lm_buddy/integrations/wandb/run_config.py
rename to src/lm_buddy/configs/wandb.py
index 5fff3773..b88ee969 100644
--- a/src/lm_buddy/integrations/wandb/run_config.py
+++ b/src/lm_buddy/configs/wandb.py
@@ -5,10 +5,10 @@
 from wandb.apis.public import Run
 from wandb.util import random_string
 
-from lm_buddy.types import BaseLMBuddyConfig
+from lm_buddy.configs.common import LMBuddyConfig
 
 
-class WandbRunConfig(BaseLMBuddyConfig):
+class WandbRunConfig(LMBuddyConfig):
     """Configuration required to log to a W&B run.
 
     A W&B Run is uniquely identified by the combination of `<entity>/<project>/<id>`.
diff --git a/src/lm_buddy/integrations/huggingface/__init__.py b/src/lm_buddy/integrations/huggingface/__init__.py
deleted file mode 100644
index aacabf25..00000000
--- a/src/lm_buddy/integrations/huggingface/__init__.py
+++ /dev/null
@@ -1,8 +0,0 @@
-# ruff: noqa: I001
-from lm_buddy.integrations.huggingface.adapter_config import *
-from lm_buddy.integrations.huggingface.dataset_config import *
-from lm_buddy.integrations.huggingface.model_config import *
-from lm_buddy.integrations.huggingface.quantization_config import *
-from lm_buddy.integrations.huggingface.tokenizer_config import *
-from lm_buddy.integrations.huggingface.trainer_config import *
-from lm_buddy.integrations.huggingface.asset_loader import *
diff --git a/src/lm_buddy/integrations/huggingface/adapter_config.py b/src/lm_buddy/integrations/huggingface/adapter_config.py
deleted file mode 100644
index 92ad1f9e..00000000
--- a/src/lm_buddy/integrations/huggingface/adapter_config.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import dataclasses
-
-from peft import PeftConfig, PeftType, TaskType
-from pydantic import field_validator, model_validator
-
-from lm_buddy.types import BaseLMBuddyConfig
-
-
-def _get_peft_config_class(peft_type: PeftType) -> type[PeftConfig]:
-    # Internal import to avoid bringing the global variable from peft into module scope
-    from peft.mapping import PEFT_TYPE_TO_CONFIG_MAPPING
-
-    return PEFT_TYPE_TO_CONFIG_MAPPING[peft_type]
-
-
-class AdapterConfig(BaseLMBuddyConfig, extra="allow"):
-    """Configuration containing PEFT adapter settings.
-
-    The type of adapter is controlled by the required `peft_type` field,
-    which must be one of the allowed values from the PEFT `PeftType` enumeration.
-    Extra arguments are allowed and are passed down to the HuggingFace `PeftConfig`
-    class determined by the `peft_type` argument.
-
-    The `task_type` for the adapter is also required.
-    By default, this is set to `TaskType.CAUSAL_LM`
-    which is appropriate for causal language model finetuning.
-    See the allowed values in the PEFT `TaskType` enumeration.
-    """
-
-    peft_type: PeftType
-    task_type: TaskType = TaskType.CAUSAL_LM
-
-    @field_validator("peft_type", "task_type", mode="before")
-    def sanitize_enum_args(cls, x):
-        if isinstance(x, str):
-            x = x.strip().upper()
-        return x
-
-    @model_validator(mode="after")
-    def validate_adapter_args(cls, config: "AdapterConfig"):
-        peft_type = config.peft_type
-
-        # PeftConfigs are standard dataclasses so can extract their allowed field names
-        adapter_cls = _get_peft_config_class(peft_type)
-        allowed_fields = {x.name for x in dataclasses.fields(adapter_cls)}
-
-        # Filter fields to those found on the PeftConfig
-        extra_fields = config.model_fields_set.difference(allowed_fields)
-        if extra_fields:
-            raise ValueError(f"Unknowon arguments for {peft_type} adapter: {extra_fields}")
-
-        return config
-
-    def as_huggingface(self) -> PeftConfig:
-        adapter_cls = _get_peft_config_class(self.peft_type)
-        return adapter_cls(**self.model_dump())
diff --git a/src/lm_buddy/integrations/huggingface/dataset_config.py b/src/lm_buddy/integrations/huggingface/dataset_config.py
deleted file mode 100644
index c16d0f2f..00000000
--- a/src/lm_buddy/integrations/huggingface/dataset_config.py
+++ /dev/null
@@ -1,42 +0,0 @@
-from pydantic import model_validator
-
-from lm_buddy.paths import AssetPath, PathPrefix
-from lm_buddy.types import BaseLMBuddyConfig
-
-DEFAULT_TEXT_FIELD: str = "text"
-
-
-class DatasetConfig(BaseLMBuddyConfig):
-    """Base configuration to load a HuggingFace dataset."""
-
-    path: AssetPath
-    split: str | None = None
-    test_size: float | None = None
-    seed: int | None = None
-
-    @model_validator(mode="after")
-    def validate_split_if_huggingface_path(cls, config: "DatasetConfig"):
-        """
-        Ensure a  `split` is provided when loading a HuggingFace dataset directly from HF Hub.
-        This makes it such that the `load_dataset` function returns the type `Dataset`
-        instead of `DatasetDict`, which makes some of the downstream logic easier.
-        """
-        if config.split is None and config.path.startswith(PathPrefix.HUGGINGFACE):
-            raise ValueError(
-                "A `split` must be specified when loading a dataset directly from HuggingFace."
-            )
-        return config
-
-
-class TextDatasetConfig(DatasetConfig):
-    """Settings passed to load a HuggingFace text dataset.
-
-    Inherits fields from the the base `DatasetConfig`.
-    The dataset should contain a single text column named by the `text_field` parameter.
-
-    A `prompt_template` can be provided to format columns of the dataset.
-    The formatted prompt is added to the dataset as the `text_field`.
-    """
-
-    text_field: str = DEFAULT_TEXT_FIELD
-    prompt_template: str | None = None
diff --git a/src/lm_buddy/integrations/huggingface/model_config.py b/src/lm_buddy/integrations/huggingface/model_config.py
deleted file mode 100644
index 8e2defbe..00000000
--- a/src/lm_buddy/integrations/huggingface/model_config.py
+++ /dev/null
@@ -1,13 +0,0 @@
-from lm_buddy.paths import AssetPath
-from lm_buddy.types import BaseLMBuddyConfig, SerializableTorchDtype
-
-
-class AutoModelConfig(BaseLMBuddyConfig):
-    """Settings passed to a HuggingFace AutoModel instantiation.
-
-    The model to load can either be a HuggingFace repo or an artifact reference on W&B.
-    """
-
-    path: AssetPath
-    trust_remote_code: bool = False
-    torch_dtype: SerializableTorchDtype | None = None
diff --git a/src/lm_buddy/integrations/huggingface/quantization_config.py b/src/lm_buddy/integrations/huggingface/quantization_config.py
deleted file mode 100644
index b0c75032..00000000
--- a/src/lm_buddy/integrations/huggingface/quantization_config.py
+++ /dev/null
@@ -1,26 +0,0 @@
-from transformers import BitsAndBytesConfig
-
-from lm_buddy.types import BaseLMBuddyConfig, SerializableTorchDtype
-
-
-class QuantizationConfig(BaseLMBuddyConfig):
-    """Basic quantization settings to pass to training and evaluation jobs.
-
-    Note that in order to use BitsAndBytes quantization on Ray,
-    you must ensure that the runtime environment is installed with GPU support.
-    This can be configured by setting the `entrypoint_num_gpus > 0` when submitting a job
-    to the cluster.
-    """
-
-    load_in_8bit: bool | None = None
-    load_in_4bit: bool | None = None
-    bnb_4bit_quant_type: str = "fp4"
-    bnb_4bit_compute_dtype: SerializableTorchDtype | None = None
-
-    def as_huggingface(self) -> BitsAndBytesConfig:
-        return BitsAndBytesConfig(
-            load_in_4bit=self.load_in_4bit,
-            load_in_8bit=self.load_in_8bit,
-            bnb_4bit_compute_dtype=self.bnb_4bit_compute_dtype,
-            bnb_4bit_quant_type=self.bnb_4bit_quant_type,
-        )
diff --git a/src/lm_buddy/integrations/huggingface/tokenizer_config.py b/src/lm_buddy/integrations/huggingface/tokenizer_config.py
deleted file mode 100644
index 2c41e052..00000000
--- a/src/lm_buddy/integrations/huggingface/tokenizer_config.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from lm_buddy.paths import AssetPath
-from lm_buddy.types import BaseLMBuddyConfig
-
-
-class AutoTokenizerConfig(BaseLMBuddyConfig):
-    """Settings passed to a HuggingFace AutoTokenizer instantiation."""
-
-    path: AssetPath
-    trust_remote_code: bool | None = None
-    use_fast: bool | None = None
diff --git a/src/lm_buddy/integrations/huggingface/trainer_config.py b/src/lm_buddy/integrations/huggingface/trainer_config.py
deleted file mode 100644
index 9097b4dd..00000000
--- a/src/lm_buddy/integrations/huggingface/trainer_config.py
+++ /dev/null
@@ -1,30 +0,0 @@
-from typing import Any
-
-from lm_buddy.types import BaseLMBuddyConfig
-
-
-class TrainerConfig(BaseLMBuddyConfig):
-    """Configuration for a HuggingFace trainer/training arguments.
-
-    This mainly encompasses arguments passed to the HuggingFace `TrainingArguments` class,
-    but also contains some additional parameters for the `Trainer` or `SFTTrainer` classes.
-    """
-
-    max_seq_length: int | None = None
-    num_train_epochs: float | None = None
-    per_device_train_batch_size: int | None = None
-    per_device_eval_batch_size: int | None = None
-    learning_rate: float | None = None
-    weight_decay: float | None = None
-    gradient_accumulation_steps: int | None = None
-    gradient_checkpointing: bool | None = None
-    evaluation_strategy: str | None = None
-    eval_steps: float | None = None
-    logging_strategy: str | None = None
-    logging_steps: float | None = None
-    save_strategy: str | None = None
-    save_steps: int | None = None
-
-    def training_args(self) -> dict[str, Any]:
-        """Return the arguments to the HuggingFace `TrainingArguments` class."""
-        return self.model_dump(exclude={"max_seq_length"}, exclude_none=True)
diff --git a/src/lm_buddy/integrations/wandb/__init__.py b/src/lm_buddy/integrations/wandb/__init__.py
deleted file mode 100644
index cbf899c8..00000000
--- a/src/lm_buddy/integrations/wandb/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from lm_buddy.integrations.wandb.artifact_utils import *
-from lm_buddy.integrations.wandb.run_config import *
-from lm_buddy.integrations.wandb.run_utils import *
diff --git a/src/lm_buddy/jobs/_entrypoints/__init__.py b/src/lm_buddy/jobs/_entrypoints/__init__.py
deleted file mode 100644
index 22654163..00000000
--- a/src/lm_buddy/jobs/_entrypoints/__init__.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from lm_buddy.jobs._entrypoints.finetuning import run_finetuning
-from lm_buddy.jobs._entrypoints.lm_harness import run_lm_harness
-from lm_buddy.jobs._entrypoints.prometheus import run_prometheus
-from lm_buddy.jobs._entrypoints.ragas import run_ragas
-
-__all__ = ["run_finetuning", "run_lm_harness", "run_prometheus", "run_ragas"]
diff --git a/src/lm_buddy/integrations/huggingface/asset_loader.py b/src/lm_buddy/jobs/asset_loader.py
similarity index 96%
rename from src/lm_buddy/integrations/huggingface/asset_loader.py
rename to src/lm_buddy/jobs/asset_loader.py
index 88b6dc11..7a4879fe 100644
--- a/src/lm_buddy/integrations/huggingface/asset_loader.py
+++ b/src/lm_buddy/jobs/asset_loader.py
@@ -13,22 +13,19 @@
     PreTrainedTokenizer,
 )
 
-from lm_buddy.integrations.huggingface import (
+from lm_buddy.configs.huggingface import (
     AutoModelConfig,
     AutoTokenizerConfig,
     DatasetConfig,
     QuantizationConfig,
 )
-from lm_buddy.integrations.wandb import get_artifact_directory, get_artifact_from_api
 from lm_buddy.paths import AssetPath, PathPrefix, strip_path_prefix
+from lm_buddy.tracking.artifact_utils import get_artifact_directory, get_artifact_from_api
 
 
 class HuggingFaceAssetLoader:
     """Helper class for loading HuggingFace assets from LM Buddy configurations.
 
-    This class depends on an `ArtifactLoader` in order to resolve actual paths from
-    artifact references.
-
     TODO: We can probably move these to standalone functions now that ArtifactLoader is gone.
           What if we add other deps (e.g, S3 client in the future?)
     """
diff --git a/src/lm_buddy/jobs/common.py b/src/lm_buddy/jobs/common.py
index 59afe823..3d6a6feb 100644
--- a/src/lm_buddy/jobs/common.py
+++ b/src/lm_buddy/jobs/common.py
@@ -5,13 +5,9 @@
 
 import pandas as pd
 import wandb
-from datasets import Dataset
 
-from lm_buddy.integrations.huggingface import TextDatasetConfig
-from lm_buddy.preprocessing import format_dataset_with_prompt
 
-
-class LMBuddyJobType(str, Enum):
+class JobType(str, Enum):
     """Enumeration of logical job types runnable via the LM Buddy."""
 
     PREPROCESSING = "preprocessing"
@@ -39,15 +35,3 @@ class EvaluationResult(JobResult):
 
     tables: dict[str, pd.DataFrame]
     dataset_path: Path | None
-
-
-def preprocess_text_dataset(dataset: Dataset, dataset_config: TextDatasetConfig) -> Dataset:
-    """Prompt format a text dataset if a prompt template is specified on the config."""
-    if dataset_config.prompt_template is not None:
-        return format_dataset_with_prompt(
-            dataset=dataset,
-            template=dataset_config.prompt_template,
-            output_field=dataset_config.text_field,
-        )
-    else:
-        return dataset
diff --git a/src/lm_buddy/jobs/configs/__init__.py b/src/lm_buddy/jobs/configs/__init__.py
deleted file mode 100644
index a87b0b65..00000000
--- a/src/lm_buddy/jobs/configs/__init__.py
+++ /dev/null
@@ -1,26 +0,0 @@
-from lm_buddy.jobs.configs.base import LMBuddyJobConfig
-from lm_buddy.jobs.configs.finetuning import FinetuningJobConfig, FinetuningRayConfig
-from lm_buddy.jobs.configs.lm_harness import (
-    LMHarnessEvaluationConfig,
-    LMHarnessJobConfig,
-    LocalChatCompletionsConfig,
-)
-from lm_buddy.jobs.configs.prometheus import PrometheusEvaluationConfig, PrometheusJobConfig
-from lm_buddy.jobs.configs.ragas import RagasEvaluationConfig, RagasJobConfig
-
-EvaluationJobConfig = LMHarnessJobConfig | PrometheusJobConfig | RagasJobConfig
-
-
-__all__ = [
-    "LMBuddyJobConfig",
-    "FinetuningJobConfig",
-    "FinetuningRayConfig",
-    "LMHarnessEvaluationConfig",
-    "LMHarnessJobConfig",
-    "LocalChatCompletionsConfig",
-    "PrometheusEvaluationConfig",
-    "PrometheusJobConfig",
-    "RagasEvaluationConfig",
-    "RagasJobConfig",
-    "EvaluationJobConfig",
-]
diff --git a/src/lm_buddy/jobs/_entrypoints/lm_harness.py b/src/lm_buddy/jobs/evaluation/lm_harness.py
similarity index 91%
rename from src/lm_buddy/jobs/_entrypoints/lm_harness.py
rename to src/lm_buddy/jobs/evaluation/lm_harness.py
index 2fb31b8a..78f4c4b6 100644
--- a/src/lm_buddy/jobs/_entrypoints/lm_harness.py
+++ b/src/lm_buddy/jobs/evaluation/lm_harness.py
@@ -6,10 +6,15 @@
 from lm_eval.models.huggingface import HFLM
 from lm_eval.models.openai_completions import OpenaiCompletionsLM
 
-from lm_buddy.integrations.huggingface import AutoModelConfig, HuggingFaceAssetLoader
-from lm_buddy.integrations.wandb import ArtifactType, build_table_artifact, default_artifact_name
+from lm_buddy.configs.huggingface import AutoModelConfig
+from lm_buddy.configs.jobs.lm_harness import LMHarnessJobConfig, LocalChatCompletionsConfig
+from lm_buddy.jobs.asset_loader import HuggingFaceAssetLoader
 from lm_buddy.jobs.common import EvaluationResult
-from lm_buddy.jobs.configs import LMHarnessJobConfig, LocalChatCompletionsConfig
+from lm_buddy.tracking.artifact_utils import (
+    ArtifactType,
+    build_table_artifact,
+    default_artifact_name,
+)
 
 
 def get_per_task_dataframes(
diff --git a/src/lm_buddy/jobs/_entrypoints/prometheus.py b/src/lm_buddy/jobs/evaluation/prometheus.py
similarity index 92%
rename from src/lm_buddy/jobs/_entrypoints/prometheus.py
rename to src/lm_buddy/jobs/evaluation/prometheus.py
index 6d883888..64f128a0 100644
--- a/src/lm_buddy/jobs/_entrypoints/prometheus.py
+++ b/src/lm_buddy/jobs/evaluation/prometheus.py
@@ -13,14 +13,16 @@
 from openai import Completion, OpenAI, OpenAIError
 from tqdm import tqdm
 
-from lm_buddy.integrations.huggingface import AutoTokenizerConfig, HuggingFaceAssetLoader
-from lm_buddy.integrations.wandb import (
+from lm_buddy.configs.huggingface import AutoTokenizerConfig
+from lm_buddy.configs.jobs.prometheus import PrometheusJobConfig
+from lm_buddy.jobs.asset_loader import HuggingFaceAssetLoader
+from lm_buddy.jobs.common import EvaluationResult
+from lm_buddy.preprocessing import format_dataset_with_prompt
+from lm_buddy.tracking.artifact_utils import (
     ArtifactType,
     build_directory_artifact,
     default_artifact_name,
 )
-from lm_buddy.jobs.common import EvaluationResult, preprocess_text_dataset
-from lm_buddy.jobs.configs import PrometheusJobConfig
 
 
 @dataclass
@@ -108,7 +110,10 @@ def run_eval(config: PrometheusJobConfig) -> Path:
     # load dataset from W&B artifact
     hf_loader = HuggingFaceAssetLoader()
     dataset = hf_loader.load_dataset(config.dataset)
-    dataset = preprocess_text_dataset(dataset, config.dataset)
+    if config.dataset.prompt_template is not None:
+        dataset = format_dataset_with_prompt(
+            dataset, config.dataset.prompt_template, config.dataset.text_field
+        )
 
     # get the tokenizer
     tokenizer_config = AutoTokenizerConfig(path=config.prometheus.inference.engine)
diff --git a/src/lm_buddy/jobs/_entrypoints/ragas.py b/src/lm_buddy/jobs/evaluation/ragas.py
similarity index 85%
rename from src/lm_buddy/jobs/_entrypoints/ragas.py
rename to src/lm_buddy/jobs/evaluation/ragas.py
index 63268934..291fe696 100644
--- a/src/lm_buddy/jobs/_entrypoints/ragas.py
+++ b/src/lm_buddy/jobs/evaluation/ragas.py
@@ -6,14 +6,15 @@
 from ragas import evaluate as ragas_evaluate
 from ragas.metrics import answer_relevancy, context_precision, context_recall, faithfulness
 
-from lm_buddy.integrations.huggingface import HuggingFaceAssetLoader
-from lm_buddy.integrations.wandb import (
+from lm_buddy.configs.jobs.ragas import RagasJobConfig
+from lm_buddy.jobs.asset_loader import HuggingFaceAssetLoader
+from lm_buddy.jobs.common import EvaluationResult
+from lm_buddy.preprocessing import format_dataset_with_prompt
+from lm_buddy.tracking.artifact_utils import (
     ArtifactType,
     build_directory_artifact,
     default_artifact_name,
 )
-from lm_buddy.jobs.common import EvaluationResult, preprocess_text_dataset
-from lm_buddy.jobs.configs import RagasJobConfig
 
 RAGAS_METRICS_MAP = {
     "faithfulness": faithfulness,
@@ -27,7 +28,10 @@ def run_eval(config: RagasJobConfig) -> Path:
     # load dataset from W&B artifact
     hf_loader = HuggingFaceAssetLoader()
     evaluation_dataset = hf_loader.load_dataset(config.dataset)
-    evaluation_dataset = preprocess_text_dataset(evaluation_dataset, config.dataset)
+    if config.dataset.prompt_template is not None:
+        evaluation_dataset = format_dataset_with_prompt(
+            evaluation_dataset, config.dataset.prompt_template, config.dataset.text_field
+        )
 
     # ragas custom model args
     ragas_args = {}
diff --git a/src/lm_buddy/jobs/_entrypoints/finetuning.py b/src/lm_buddy/jobs/finetuning.py
similarity index 84%
rename from src/lm_buddy/jobs/_entrypoints/finetuning.py
rename to src/lm_buddy/jobs/finetuning.py
index 47a744ed..ff11fcd2 100644
--- a/src/lm_buddy/jobs/_entrypoints/finetuning.py
+++ b/src/lm_buddy/jobs/finetuning.py
@@ -9,15 +9,16 @@
 from transformers import TrainingArguments
 from trl import SFTTrainer
 
-from lm_buddy.integrations.huggingface import HuggingFaceAssetLoader
-from lm_buddy.integrations.wandb import (
+from lm_buddy.configs.jobs.finetuning import FinetuningJobConfig
+from lm_buddy.jobs.asset_loader import HuggingFaceAssetLoader
+from lm_buddy.jobs.common import FinetuningResult, JobType
+from lm_buddy.preprocessing import format_dataset_with_prompt
+from lm_buddy.tracking.artifact_utils import (
     ArtifactType,
-    WandbResumeMode,
     build_directory_artifact,
     default_artifact_name,
 )
-from lm_buddy.jobs.common import FinetuningResult, LMBuddyJobType, preprocess_text_dataset
-from lm_buddy.jobs.configs import FinetuningJobConfig
+from lm_buddy.tracking.run_utils import WandbResumeMode
 
 
 def is_tracking_enabled(config: FinetuningJobConfig):
@@ -33,8 +34,13 @@ def load_and_train(config: FinetuningJobConfig):
     tokenizer = hf_loader.load_pretrained_tokenizer(config.tokenizer)
 
     datasets = hf_loader.load_and_split_dataset(config.dataset)
-    for split, dataset in datasets.items():
-        datasets[split] = preprocess_text_dataset(dataset, config.dataset)
+    if config.dataset.prompt_template is not None:
+        for split, dataset in datasets.items():
+            datasets[split] = format_dataset_with_prompt(
+                dataset=dataset,
+                prompt_template=config.dataset.prompt_template,
+                output_field=config.dataset.text_field,
+            )
 
     training_args = TrainingArguments(
         output_dir="out",  # Local checkpoint path on a worker
@@ -69,7 +75,7 @@ def training_function(config_data: dict[str, Any]):
         with wandb.init(
             name=config.name,
             resume=WandbResumeMode.NEVER,
-            job_type=LMBuddyJobType.FINETUNING,
+            job_type=JobType.FINETUNING,
             **config.tracking.model_dump(),
         ):
             load_and_train(config)
diff --git a/tests/unit/integrations/__init__.py b/src/lm_buddy/tracking/__init__.py
similarity index 100%
rename from tests/unit/integrations/__init__.py
rename to src/lm_buddy/tracking/__init__.py
diff --git a/src/lm_buddy/integrations/wandb/artifact_utils.py b/src/lm_buddy/tracking/artifact_utils.py
similarity index 100%
rename from src/lm_buddy/integrations/wandb/artifact_utils.py
rename to src/lm_buddy/tracking/artifact_utils.py
diff --git a/src/lm_buddy/integrations/wandb/run_utils.py b/src/lm_buddy/tracking/run_utils.py
similarity index 94%
rename from src/lm_buddy/integrations/wandb/run_utils.py
rename to src/lm_buddy/tracking/run_utils.py
index 1f15bd0b..76a6c3d8 100644
--- a/src/lm_buddy/integrations/wandb/run_utils.py
+++ b/src/lm_buddy/tracking/run_utils.py
@@ -4,7 +4,7 @@
 import wandb
 from wandb.apis.public import Run as ApiRun
 
-from lm_buddy.integrations.wandb import WandbRunConfig
+from lm_buddy.configs.wandb import WandbRunConfig
 
 
 class WandbResumeMode(str, Enum):
diff --git a/tests/conftest.py b/tests/conftest.py
index 422cbcbc..1d1b0377 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -5,7 +5,7 @@
 
 import pytest
 
-from lm_buddy.integrations.wandb import ArtifactType, build_directory_artifact
+from lm_buddy.tracking.artifact_utils import ArtifactType, build_directory_artifact
 
 
 @pytest.fixture(scope="session")
diff --git a/tests/integration/test_finetuning.py b/tests/integration/test_finetuning.py
index c41c549c..b43c367c 100644
--- a/tests/integration/test_finetuning.py
+++ b/tests/integration/test_finetuning.py
@@ -1,16 +1,17 @@
 import pytest
 
 from lm_buddy import LMBuddy
-from lm_buddy.integrations.huggingface import AutoModelConfig, TextDatasetConfig, TrainerConfig
-from lm_buddy.integrations.wandb import ArtifactType, WandbRunConfig
-from lm_buddy.jobs.configs import FinetuningJobConfig, FinetuningRayConfig
+from lm_buddy.configs.huggingface import AutoModelConfig, DatasetConfig, TrainerConfig
+from lm_buddy.configs.jobs.finetuning import FinetuningJobConfig, FinetuningRayConfig
+from lm_buddy.configs.wandb import WandbRunConfig
 from lm_buddy.paths import format_file_path
+from lm_buddy.tracking.artifact_utils import ArtifactType
 
 
 @pytest.fixture
 def job_config(llm_model_path, text_dataset_path) -> FinetuningJobConfig:
     model_config = AutoModelConfig(path=format_file_path(llm_model_path))
-    dataset_config = TextDatasetConfig(
+    dataset_config = DatasetConfig(
         path=format_file_path(text_dataset_path),
         text_field="text",
         split="train",
diff --git a/tests/integration/test_lm_harness.py b/tests/integration/test_lm_harness.py
index 29832a00..be7e279a 100644
--- a/tests/integration/test_lm_harness.py
+++ b/tests/integration/test_lm_harness.py
@@ -1,9 +1,9 @@
 import pytest
 
 from lm_buddy import LMBuddy
-from lm_buddy.integrations.huggingface import AutoModelConfig
-from lm_buddy.integrations.wandb import WandbRunConfig
-from lm_buddy.jobs.configs import LMHarnessEvaluationConfig, LMHarnessJobConfig
+from lm_buddy.configs.huggingface import AutoModelConfig
+from lm_buddy.configs.jobs.lm_harness import LMHarnessEvaluationConfig, LMHarnessJobConfig
+from lm_buddy.configs.wandb import WandbRunConfig
 from lm_buddy.paths import format_file_path
 
 
diff --git a/tests/unit/integrations/huggingface/__init__.py b/tests/unit/configs/__init__.py
similarity index 100%
rename from tests/unit/integrations/huggingface/__init__.py
rename to tests/unit/configs/__init__.py
diff --git a/tests/unit/integrations/wandb/__init__.py b/tests/unit/configs/jobs/__init__.py
similarity index 100%
rename from tests/unit/integrations/wandb/__init__.py
rename to tests/unit/configs/jobs/__init__.py
diff --git a/tests/unit/conftest.py b/tests/unit/configs/jobs/conftest.py
similarity index 82%
rename from tests/unit/conftest.py
rename to tests/unit/configs/jobs/conftest.py
index d4362d4b..2c2f116f 100644
--- a/tests/unit/conftest.py
+++ b/tests/unit/configs/jobs/conftest.py
@@ -1,15 +1,15 @@
 import pytest
 from peft import PeftType, TaskType
 
-from lm_buddy.integrations.huggingface import (
+from lm_buddy.configs.huggingface import (
     AdapterConfig,
     AutoModelConfig,
     AutoTokenizerConfig,
+    DatasetConfig,
     QuantizationConfig,
-    TextDatasetConfig,
 )
-from lm_buddy.integrations.vllm import InferenceServerConfig
-from lm_buddy.integrations.wandb import WandbRunConfig
+from lm_buddy.configs.vllm import InferenceServerConfig
+from lm_buddy.configs.wandb import WandbRunConfig
 
 
 @pytest.fixture
@@ -36,13 +36,13 @@ def tokenizer_config_with_artifact():
 
 @pytest.fixture
 def dataset_config_with_repo_id():
-    return TextDatasetConfig(path="hf://databricks/dolly15k", text_field="text", split="train")
+    return DatasetConfig(path="hf://databricks/dolly15k", text_field="text", split="train")
 
 
 @pytest.fixture
 def dataset_config_with_artifact():
     artifact_path = "wandb://project/dataset-artifact:latest"
-    return TextDatasetConfig(path=artifact_path)
+    return DatasetConfig(path=artifact_path)
 
 
 @pytest.fixture
diff --git a/tests/unit/jobs/configs/test_finetuning_config.py b/tests/unit/configs/jobs/test_finetuning_config.py
similarity index 92%
rename from tests/unit/jobs/configs/test_finetuning_config.py
rename to tests/unit/configs/jobs/test_finetuning_config.py
index 120b43b8..f8096f32 100644
--- a/tests/unit/jobs/configs/test_finetuning_config.py
+++ b/tests/unit/configs/jobs/test_finetuning_config.py
@@ -1,8 +1,8 @@
 import pytest
 from pydantic import ValidationError
 
-from lm_buddy.integrations.huggingface import TextDatasetConfig
-from lm_buddy.jobs.configs import FinetuningJobConfig, FinetuningRayConfig
+from lm_buddy.configs.huggingface import DatasetConfig
+from lm_buddy.configs.jobs.finetuning import FinetuningJobConfig, FinetuningRayConfig
 from tests.test_utils import copy_pydantic_json
 
 
@@ -55,7 +55,7 @@ def test_load_example_config(examples_dir):
 def test_argument_validation():
     model_path = "hf://model-repo-id"
     tokenizer_path = "hf://tokenizer-repo-id"
-    dataset_config = TextDatasetConfig(path="hf://dataset-repo-id", split="train")
+    dataset_config = DatasetConfig(path="hf://dataset-repo-id", split="train")
 
     # Strings should be upcast to configs as the path argument
     allowed_config = FinetuningJobConfig(
diff --git a/tests/unit/jobs/configs/test_lm_harness_config.py b/tests/unit/configs/jobs/test_lm_harness_config.py
similarity index 96%
rename from tests/unit/jobs/configs/test_lm_harness_config.py
rename to tests/unit/configs/jobs/test_lm_harness_config.py
index 70fd3d6f..9a0d1c66 100644
--- a/tests/unit/jobs/configs/test_lm_harness_config.py
+++ b/tests/unit/configs/jobs/test_lm_harness_config.py
@@ -1,12 +1,12 @@
 import pytest
 from pydantic import ValidationError
 
-from lm_buddy.integrations.vllm import InferenceServerConfig
-from lm_buddy.jobs.configs import (
+from lm_buddy.configs.jobs.lm_harness import (
     LMHarnessEvaluationConfig,
     LMHarnessJobConfig,
     LocalChatCompletionsConfig,
 )
+from lm_buddy.configs.vllm import InferenceServerConfig
 from tests.test_utils import copy_pydantic_json
 
 
diff --git a/tests/unit/integrations/huggingface/test_adapter_config.py b/tests/unit/configs/test_adapter_config.py
similarity index 79%
rename from tests/unit/integrations/huggingface/test_adapter_config.py
rename to tests/unit/configs/test_adapter_config.py
index cb0d118a..af0f4a6c 100644
--- a/tests/unit/integrations/huggingface/test_adapter_config.py
+++ b/tests/unit/configs/test_adapter_config.py
@@ -2,8 +2,7 @@
 from peft import PeftType, TaskType
 from pydantic import ValidationError
 
-from lm_buddy.integrations.huggingface import AdapterConfig
-from lm_buddy.integrations.huggingface.adapter_config import _get_peft_config_class
+from lm_buddy.configs.huggingface import AdapterConfig
 
 
 def test_enum_sanitzation():
@@ -27,5 +26,5 @@ def test_huggingface_conversion():
         config = AdapterConfig(peft_type=peft_type)
 
         hf_config = config.as_huggingface()
-        expected_cls = _get_peft_config_class(peft_type)
+        expected_cls = AdapterConfig._get_peft_config_class(peft_type)
         assert type(hf_config) is expected_cls
diff --git a/tests/unit/test_types.py b/tests/unit/configs/test_common.py
similarity index 57%
rename from tests/unit/test_types.py
rename to tests/unit/configs/test_common.py
index 47dc0b06..794ca3dc 100644
--- a/tests/unit/test_types.py
+++ b/tests/unit/configs/test_common.py
@@ -2,11 +2,11 @@
 import torch
 from pydantic import ValidationError
 
-from lm_buddy.types import BaseLMBuddyConfig, SerializableTorchDtype
+from lm_buddy.configs.common import LMBuddyConfig, SerializableTorchDtype
 
 
-def test_base_config_settings():
-    class TestConfig(BaseLMBuddyConfig):
+def test_config_settings():
+    class TestConfig(LMBuddyConfig):
         value: int
 
     # Validate assignment
@@ -19,8 +19,19 @@ class TestConfig(BaseLMBuddyConfig):
         TestConfig(value=42, foo="bar")
 
 
+def test_config_as_tempfile():
+    class TestConfig(LMBuddyConfig):
+        name: str
+
+    config = TestConfig(name="test-config")
+    config_name = "my-job-config.yaml"
+    with config.to_tempfile(name=config_name) as path:
+        assert path.name == config_name
+        assert TestConfig.from_yaml_file(path) == config
+
+
 def test_serializable_torch_dtype():
-    class TestConfig(BaseLMBuddyConfig):
+    class TestConfig(LMBuddyConfig):
         torch_dtype: SerializableTorchDtype
 
     config = TestConfig(torch_dtype="bfloat16")
diff --git a/tests/unit/integrations/huggingface/test_dataset_config.py b/tests/unit/configs/test_dataset_config.py
similarity index 82%
rename from tests/unit/integrations/huggingface/test_dataset_config.py
rename to tests/unit/configs/test_dataset_config.py
index fbc513b6..3f092846 100644
--- a/tests/unit/integrations/huggingface/test_dataset_config.py
+++ b/tests/unit/configs/test_dataset_config.py
@@ -1,7 +1,7 @@
 import pytest
 from pydantic import ValidationError
 
-from lm_buddy.integrations.huggingface import DatasetConfig
+from lm_buddy.configs.huggingface import DatasetConfig
 from lm_buddy.paths import format_huggingface_path
 
 
diff --git a/tests/unit/integrations/huggingface/test_quantization_config.py b/tests/unit/configs/test_quantization_config.py
similarity index 86%
rename from tests/unit/integrations/huggingface/test_quantization_config.py
rename to tests/unit/configs/test_quantization_config.py
index 7efaac0c..c823f316 100644
--- a/tests/unit/integrations/huggingface/test_quantization_config.py
+++ b/tests/unit/configs/test_quantization_config.py
@@ -1,7 +1,7 @@
 import pytest
 import torch
 
-from lm_buddy.integrations.huggingface import QuantizationConfig
+from lm_buddy.configs.huggingface import QuantizationConfig
 from tests.test_utils import copy_pydantic_json
 
 
diff --git a/tests/unit/integrations/wandb/test_run_config.py b/tests/unit/configs/test_run_config.py
similarity index 96%
rename from tests/unit/integrations/wandb/test_run_config.py
rename to tests/unit/configs/test_run_config.py
index 8da07230..cb998f14 100644
--- a/tests/unit/integrations/wandb/test_run_config.py
+++ b/tests/unit/configs/test_run_config.py
@@ -4,7 +4,7 @@
 import pytest
 from pydantic import ValidationError
 
-from lm_buddy.integrations.wandb import WandbRunConfig
+from lm_buddy.configs.wandb import WandbRunConfig
 from tests.test_utils import copy_pydantic_json
 
 
diff --git a/tests/unit/jobs/configs/__init__.py b/tests/unit/jobs/configs/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/tests/unit/jobs/configs/test_job_config.py b/tests/unit/jobs/configs/test_job_config.py
deleted file mode 100644
index 47114302..00000000
--- a/tests/unit/jobs/configs/test_job_config.py
+++ /dev/null
@@ -1,14 +0,0 @@
-from lm_buddy.jobs.configs import LMBuddyJobConfig
-from lm_buddy.paths import AssetPath
-
-
-def test_config_as_tempfile():
-    class TestConfig(LMBuddyJobConfig):
-        def asset_paths(self) -> set[AssetPath]:
-            return super().asset_paths()
-
-    config = TestConfig(name="test-config")
-    config_name = "my-job-config.yaml"
-    with config.to_tempfile(name=config_name) as path:
-        assert path.name == config_name
-        assert TestConfig.from_yaml_file(path) == config
diff --git a/tests/unit/integrations/huggingface/test_asset_loader.py b/tests/unit/jobs/test_asset_loader.py
similarity index 88%
rename from tests/unit/integrations/huggingface/test_asset_loader.py
rename to tests/unit/jobs/test_asset_loader.py
index a3293b99..976d8a55 100644
--- a/tests/unit/integrations/huggingface/test_asset_loader.py
+++ b/tests/unit/jobs/test_asset_loader.py
@@ -1,7 +1,8 @@
 import torch
 from datasets import Dataset, DatasetDict
 
-from lm_buddy.integrations.huggingface import AutoModelConfig, DatasetConfig, HuggingFaceAssetLoader
+from lm_buddy.configs.huggingface import AutoModelConfig, DatasetConfig
+from lm_buddy.jobs.asset_loader import HuggingFaceAssetLoader
 from lm_buddy.paths import format_file_path