From 151799d765473be9ffa88201297efcd8192f2f2d Mon Sep 17 00:00:00 2001 From: Shubham Toshniwal Date: Wed, 10 Jul 2024 11:42:49 -0700 Subject: [PATCH 1/3] Nemotron eval map --- .../inference/prompt/nemotron/eval_map.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 nemo_skills/inference/prompt/nemotron/eval_map.py diff --git a/nemo_skills/inference/prompt/nemotron/eval_map.py b/nemo_skills/inference/prompt/nemotron/eval_map.py new file mode 100644 index 000000000..95b434463 --- /dev/null +++ b/nemo_skills/inference/prompt/nemotron/eval_map.py @@ -0,0 +1,25 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# need to contain an EVAL_MAP dictionary mapping model version name to prompts for different benchmarks +# can have a default key that will be used if benchmark name is not in dict + +EVAL_MAP = { + 'base': { + 'default': 'nemotron/fewshot', + }, + 'instruct': { # nemotron-instruct + 'default': 'nemotron/zeroshot', + }, +} From 8744d87cedb78ac24b4a429ee8c2a2137c7baf6e Mon Sep 17 00:00:00 2001 From: Shubham Toshniwal Date: Wed, 10 Jul 2024 11:45:00 -0700 Subject: [PATCH 2/3] shtoshni/nemotron-fix --- nemo_skills/inference/prompt/nemotron/eval_map.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nemo_skills/inference/prompt/nemotron/eval_map.py b/nemo_skills/inference/prompt/nemotron/eval_map.py index 95b434463..51d401aa8 100644 --- a/nemo_skills/inference/prompt/nemotron/eval_map.py +++ b/nemo_skills/inference/prompt/nemotron/eval_map.py @@ -22,4 +22,5 @@ 'instruct': { # nemotron-instruct 'default': 'nemotron/zeroshot', }, + } From fe1cb8f550b8e31f1bddd26479fb1d291c9206d1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 20:07:07 +0000 Subject: [PATCH 3/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nemo_skills/inference/prompt/nemotron/eval_map.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nemo_skills/inference/prompt/nemotron/eval_map.py b/nemo_skills/inference/prompt/nemotron/eval_map.py index 51d401aa8..4f27c8928 100644 --- a/nemo_skills/inference/prompt/nemotron/eval_map.py +++ b/nemo_skills/inference/prompt/nemotron/eval_map.py @@ -16,11 +16,10 @@ # can have a default key that will be used if benchmark name is not in dict EVAL_MAP = { - 'base': { + 'base': { 'default': 'nemotron/fewshot', }, 'instruct': { # nemotron-instruct 'default': 'nemotron/zeroshot', }, - }