chore: default priority earlier to ensure constraints are satisfied […

…CM-553] (#10043)

Co-authored-by: Amanda Vialva <[email protected]>

master/internal/api_command.go

@@ -22,6 +22,7 @@ import (
 	"github.com/determined-ai/determined/master/internal/api/apiutils"
 	"github.com/determined-ai/determined/master/internal/authz"
 	"github.com/determined-ai/determined/master/internal/command"
+	masterConfig "github.com/determined-ai/determined/master/internal/config"
 	"github.com/determined-ai/determined/master/internal/configpolicy"
 	"github.com/determined-ai/determined/master/internal/db"
 	"github.com/determined-ai/determined/master/internal/grpcutil"
@@ -144,6 +145,13 @@ func (a *apiServer) getCommandLaunchParams(ctx context.Context, req *protoComman
 	config.Resources.ResourcePool = poolName.String()
 	config.Resources.Slots = resources.Slots
 
+	// Apply the scheduler's default priority.
+	if config.Resources.Priority == nil {
+		prio := masterConfig.DefaultPriorityForPool(poolName.String())
+		config.Resources.Priority = &prio
+	}
+	// TODO (CM-493) NTSC invariant config overrides
+
 	var contextDirectory []byte
 	config.WorkDir, contextDirectory, err = fillContextDir(config.WorkDir, workDirInDefaults, req.Files)
 	if err != nil {

master/internal/api_generic_tasks.go

@@ -18,6 +18,7 @@ import (
 	"github.com/determined-ai/determined/master/internal/api"
 	"github.com/determined-ai/determined/master/internal/authz"
 	"github.com/determined-ai/determined/master/internal/command"
+	masterConfig "github.com/determined-ai/determined/master/internal/config"
 	"github.com/determined-ai/determined/master/internal/db"
 	"github.com/determined-ai/determined/master/internal/grpcutil"
 	"github.com/determined-ai/determined/master/internal/job/jobservice"
@@ -131,6 +132,12 @@ func (a *apiServer) getGenericTaskLaunchParameters(
 	taskConfig.Resources.RawResourcePool = &rawResourcePool
 	taskConfig.Resources.RawSlots = &resources.Slots
 
+	// Apply the scheduler's default priority.
+	if taskConfig.Resources.Priority() == nil {
+		prio := masterConfig.DefaultPriorityForPool(poolName.String())
+		taskConfig.Resources.RawPriority = &prio
+	}
+
 	var contextDirectoryBytes []byte
 	taskConfig.WorkDir, contextDirectoryBytes, err = fillContextDir(
 		taskConfig.WorkDir,

master/internal/api_user_intg_test.go

@@ -74,6 +74,7 @@ func MockRM() *mocks.ResourceManager {
 	}, nil)
 
 	mockRM.On("SmallerValueIsHigherPriority").Return(true, nil)
+	mockRM.On("SetGroupPriority", mock.Anything).Return(nil)
 
 	return &mockRM
 }

master/internal/config/config.go

@@ -616,7 +616,6 @@ func readRMPreemptionStatus(config *ResourceManagerWithPoolsConfig, rpName strin
 
 // ReadPriority resolves the priority value for a job.
 func ReadPriority(rpName string, jobConf interface{}) int {
-	config := GetMasterConfig()
 	var prio *int
 	// look at the individual job config
 	switch conf := jobConf.(type) {
@@ -628,13 +627,18 @@ func ReadPriority(rpName string, jobConf interface{}) int {
 	if prio != nil {
 		return *prio
 	}
-
 	// if not found, fall back to the resource pools config
+	return DefaultPriorityForPool(rpName)
+}
+
+// DefaultPriorityForPool returns the default priority for any jobs (user-defined if provided, otherwise our default).
+func DefaultPriorityForPool(rpName string) int {
+	config := GetMasterConfig()
 	for _, rm := range config.ResourceManagers() {
 		for _, rpConfig := range rm.ResourcePools {
 			if rpConfig.PoolName == rpName {
 				schedulerConf := rpConfig.Scheduler
-				prio = readPriorityFromScheduler(schedulerConf)
+				prio := readPriorityFromScheduler(schedulerConf)
 				if prio != nil {
 					return *prio
 				}
@@ -647,16 +651,13 @@ func ReadPriority(rpName string, jobConf interface{}) int {
 						return *prio
 					}
 				}
-
 				if rm.ResourceManager.KubernetesRM != nil {
 					return KubernetesDefaultPriority
 				}
-
-				break
+				return DefaultSchedulingPriority
 			}
 		}
 	}
-
 	return DefaultSchedulingPriority
 }
 

master/internal/core_experiment.go

@@ -17,6 +17,7 @@ import (
 	"github.com/determined-ai/determined/master/internal/api"
 	"github.com/determined-ai/determined/master/internal/authz"
 	"github.com/determined-ai/determined/master/internal/checkpoints"
+	masterConfig "github.com/determined-ai/determined/master/internal/config"
 	"github.com/determined-ai/determined/master/internal/configpolicy"
 	detContext "github.com/determined-ai/determined/master/internal/context"
 	"github.com/determined-ai/determined/master/internal/db"
@@ -348,6 +349,12 @@ func (m *Master) parseCreateExperiment(ctx context.Context, req *apiv1.CreateExp
 		config.RawCheckpointStorage, &m.config.CheckpointStorage,
 	)
 
+	// Apply the scheduler's default priority.
+	if config.Resources().Priority() == nil {
+		prio := masterConfig.DefaultPriorityForPool(poolName.String())
+		config.RawResources.RawPriority = &prio
+	}
+
 	// Lastly, apply any json-schema-defined defaults.
 	config = schemas.WithDefaults(config)