diff --git a/master/internal/api_command.go b/master/internal/api_command.go index 4ef783db660..e0dafee6159 100644 --- a/master/internal/api_command.go +++ b/master/internal/api_command.go @@ -22,6 +22,7 @@ import ( "github.com/determined-ai/determined/master/internal/api/apiutils" "github.com/determined-ai/determined/master/internal/authz" "github.com/determined-ai/determined/master/internal/command" + masterConfig "github.com/determined-ai/determined/master/internal/config" "github.com/determined-ai/determined/master/internal/configpolicy" "github.com/determined-ai/determined/master/internal/db" "github.com/determined-ai/determined/master/internal/grpcutil" @@ -144,6 +145,13 @@ func (a *apiServer) getCommandLaunchParams(ctx context.Context, req *protoComman config.Resources.ResourcePool = poolName.String() config.Resources.Slots = resources.Slots + // Apply the scheduler's default priority. + if config.Resources.Priority == nil { + prio := masterConfig.DefaultPriorityForPool(poolName.String()) + config.Resources.Priority = &prio + } + // TODO (CM-493) NTSC invariant config overrides + var contextDirectory []byte config.WorkDir, contextDirectory, err = fillContextDir(config.WorkDir, workDirInDefaults, req.Files) if err != nil { diff --git a/master/internal/api_generic_tasks.go b/master/internal/api_generic_tasks.go index c6a7f59fb7c..10a646f2a7f 100644 --- a/master/internal/api_generic_tasks.go +++ b/master/internal/api_generic_tasks.go @@ -18,6 +18,7 @@ import ( "github.com/determined-ai/determined/master/internal/api" "github.com/determined-ai/determined/master/internal/authz" "github.com/determined-ai/determined/master/internal/command" + masterConfig "github.com/determined-ai/determined/master/internal/config" "github.com/determined-ai/determined/master/internal/db" "github.com/determined-ai/determined/master/internal/grpcutil" "github.com/determined-ai/determined/master/internal/job/jobservice" @@ -131,6 +132,12 @@ func (a *apiServer) getGenericTaskLaunchParameters( taskConfig.Resources.RawResourcePool = &rawResourcePool taskConfig.Resources.RawSlots = &resources.Slots + // Apply the scheduler's default priority. + if taskConfig.Resources.Priority() == nil { + prio := masterConfig.DefaultPriorityForPool(poolName.String()) + taskConfig.Resources.RawPriority = &prio + } + var contextDirectoryBytes []byte taskConfig.WorkDir, contextDirectoryBytes, err = fillContextDir( taskConfig.WorkDir, diff --git a/master/internal/api_user_intg_test.go b/master/internal/api_user_intg_test.go index 21c0668e1c3..c0f1d673abf 100644 --- a/master/internal/api_user_intg_test.go +++ b/master/internal/api_user_intg_test.go @@ -74,6 +74,7 @@ func MockRM() *mocks.ResourceManager { }, nil) mockRM.On("SmallerValueIsHigherPriority").Return(true, nil) + mockRM.On("SetGroupPriority", mock.Anything).Return(nil) return &mockRM } diff --git a/master/internal/config/config.go b/master/internal/config/config.go index d8222952890..306cc1b27bf 100644 --- a/master/internal/config/config.go +++ b/master/internal/config/config.go @@ -616,7 +616,6 @@ func readRMPreemptionStatus(config *ResourceManagerWithPoolsConfig, rpName strin // ReadPriority resolves the priority value for a job. func ReadPriority(rpName string, jobConf interface{}) int { - config := GetMasterConfig() var prio *int // look at the individual job config switch conf := jobConf.(type) { @@ -628,13 +627,18 @@ func ReadPriority(rpName string, jobConf interface{}) int { if prio != nil { return *prio } - // if not found, fall back to the resource pools config + return DefaultPriorityForPool(rpName) +} + +// DefaultPriorityForPool returns the default priority for any jobs (user-defined if provided, otherwise our default). +func DefaultPriorityForPool(rpName string) int { + config := GetMasterConfig() for _, rm := range config.ResourceManagers() { for _, rpConfig := range rm.ResourcePools { if rpConfig.PoolName == rpName { schedulerConf := rpConfig.Scheduler - prio = readPriorityFromScheduler(schedulerConf) + prio := readPriorityFromScheduler(schedulerConf) if prio != nil { return *prio } @@ -647,16 +651,13 @@ func ReadPriority(rpName string, jobConf interface{}) int { return *prio } } - if rm.ResourceManager.KubernetesRM != nil { return KubernetesDefaultPriority } - - break + return DefaultSchedulingPriority } } } - return DefaultSchedulingPriority } diff --git a/master/internal/core_experiment.go b/master/internal/core_experiment.go index c1aef00c503..88161fccc94 100644 --- a/master/internal/core_experiment.go +++ b/master/internal/core_experiment.go @@ -17,6 +17,7 @@ import ( "github.com/determined-ai/determined/master/internal/api" "github.com/determined-ai/determined/master/internal/authz" "github.com/determined-ai/determined/master/internal/checkpoints" + masterConfig "github.com/determined-ai/determined/master/internal/config" "github.com/determined-ai/determined/master/internal/configpolicy" detContext "github.com/determined-ai/determined/master/internal/context" "github.com/determined-ai/determined/master/internal/db" @@ -348,6 +349,12 @@ func (m *Master) parseCreateExperiment(ctx context.Context, req *apiv1.CreateExp config.RawCheckpointStorage, &m.config.CheckpointStorage, ) + // Apply the scheduler's default priority. + if config.Resources().Priority() == nil { + prio := masterConfig.DefaultPriorityForPool(poolName.String()) + config.RawResources.RawPriority = &prio + } + // Lastly, apply any json-schema-defined defaults. config = schemas.WithDefaults(config)