Merge branch 'main' into feat/remove_template

.github/workflows/lint.yml

@@ -0,0 +1,30 @@
+name: Lint Test
+
+on:
+  push:
+    branches:
+      - main 
+  pull_request:
+    branches:
+      - main 
+
+jobs:
+  run-pre-commit:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install pre-commit
+        run: |
+          python -m pip install --upgrade pip
+          pip install pre-commit
+
+      - name: Run pre-commit
+        run: pre-commit run --all-files

evalscope/perf/arguments.py

@@ -32,6 +32,9 @@ class Arguments:
     wandb_api_key: Optional[str] = None  # WandB API key for logging
     name: Optional[str] = None  # Name for the run
 
+    # Output settings
+    outputs_dir: str = 'outputs'
+
     # Prompt settings
     max_prompt_length: int = sys.maxsize  # Maximum length of the prompt
     min_prompt_length: int = 0  # Minimum length of the prompt
@@ -57,7 +60,6 @@ class Arguments:
 
     @staticmethod
     def from_args(args):
-
         return Arguments(
             model=args.model,
             attn_implementation=args.attn_implementation,
@@ -72,6 +74,7 @@ def from_args(args):
             headers=args.headers,
             wandb_api_key=args.wandb_api_key,
             name=args.name,
+            outputs_dir=args.outputs_dir,
             debug=args.debug,
             tokenizer_path=args.tokenizer_path,
             api=args.api,
@@ -152,6 +155,9 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--prompt', type=str, required=False, default=None, help='Specified the request prompt')
     parser.add_argument('--query-template', type=str, default=None, help='Specify the query template')
 
+    # Output settings
+    parser.add_argument('--outputs-dir', help='Outputs dir.', default='outputs')
+
     # Dataset settings
     parser.add_argument('--dataset', type=str, default='openqa', help='Specify the dataset')
     parser.add_argument('--dataset-path', type=str, required=False, help='Path to the dataset file')
@@ -170,6 +176,7 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--stream', action='store_true', help='Stream output with SSE', default=None)
     parser.add_argument('--temperature', type=float, help='The sample temperature', default=None)
     parser.add_argument('--top-p', type=float, help='Sampling top p', default=None)
+
     # yapf: enable
 
 

evalscope/perf/benchmark.py

@@ -138,7 +138,7 @@ async def statistic_benchmark_metric_worker(benchmark_data_queue: asyncio.Queue,
     api_plugin_class = ApiRegistry(args.api)
     api_plugin = api_plugin_class(args.tokenizer_path)
 
-    result_db_path = get_result_db_path(args.name, args.model)
+    result_db_path = get_result_db_path(args.name, args.model, args.outputs_dir)
     # Initialize wandb
     if args.wandb_api_key:
         import wandb
@@ -192,7 +192,7 @@ async def statistic_benchmark_metric_worker(benchmark_data_queue: asyncio.Queue,
 async def start_server(args: Arguments) -> bool:
     if args.api.startswith('local'):
         #  start local server
-        server = threading.Thread(target=start_app, args=(copy.deepcopy(args), ), daemon=True)
+        server = threading.Thread(target=start_app, args=(copy.deepcopy(args),), daemon=True)
         server.start()
 
         if args.dataset.startswith('speed_benchmark'):

evalscope/perf/utils/db_util.py

@@ -88,9 +88,8 @@ def insert_benchmark_data(cursor: sqlite3.Cursor, benchmark_data: BenchmarkData)
         cursor.execute(query, common_columns)
 
 
-def get_result_db_path(name, model):
+def get_result_db_path(name, model, output_dir):
     current_time = datetime.now().strftime('%Y%m%d_%H%M%S')
-    output_dir = './outputs'
     result_db_path = os.path.join(output_dir, f'{name or model}_perf', current_time, 'benchmark_data.db')
 
     if not os.path.exists(os.path.dirname(result_db_path)):