From 5eeffde0b36302a6ab073bd5e3e6a5de2b6f6655 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Fri, 23 Jun 2023 09:42:08 -0400 Subject: [PATCH 01/20] Initial version of the SAS solver interfaces and unit tests. --- pyomo/solvers/plugins/solvers/SAS.py | 700 ++++++++++++++++++++++ pyomo/solvers/plugins/solvers/__init__.py | 1 + pyomo/solvers/tests/checks/test_SAS.py | 462 ++++++++++++++ 3 files changed, 1163 insertions(+) create mode 100644 pyomo/solvers/plugins/solvers/SAS.py create mode 100644 pyomo/solvers/tests/checks/test_SAS.py diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py new file mode 100644 index 00000000000..7f50b7a2970 --- /dev/null +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -0,0 +1,700 @@ +__all__ = ['SAS'] + +import logging +import sys +import os + +from io import StringIO +from abc import ABC, abstractmethod +from contextlib import redirect_stdout + +from pyomo.opt.base import ProblemFormat, ResultsFormat, OptSolver +from pyomo.opt.base.solvers import SolverFactory +from pyomo.common.collections import Bunch +from pyomo.opt.results import ( + SolverResults, + SolverStatus, + TerminationCondition, + SolutionStatus, + ProblemSense, +) +from pyomo.common.tempfiles import TempfileManager +from pyomo.core.base import Var +from pyomo.core.base.block import _BlockData +from pyomo.core.kernel.block import IBlock + + +logger = logging.getLogger('pyomo.solvers') + + +STATUS_TO_SOLVERSTATUS = { + "OK": SolverStatus.ok, + "SYNTAX_ERROR": SolverStatus.error, + "DATA_ERROR": SolverStatus.error, + "OUT_OF_MEMORY": SolverStatus.aborted, + "IO_ERROR": SolverStatus.error, + "ERROR": SolverStatus.error, +} + +# This combines all status codes from OPTLP/solvelp and OPTMILP/solvemilp +SOLSTATUS_TO_TERMINATIONCOND = { + "OPTIMAL": TerminationCondition.optimal, + "OPTIMAL_AGAP": TerminationCondition.optimal, + "OPTIMAL_RGAP": TerminationCondition.optimal, + "OPTIMAL_COND": TerminationCondition.optimal, + "TARGET": TerminationCondition.optimal, + "CONDITIONAL_OPTIMAL": TerminationCondition.optimal, + "FEASIBLE": TerminationCondition.feasible, + "INFEASIBLE": TerminationCondition.infeasible, + "UNBOUNDED": TerminationCondition.unbounded, + "INFEASIBLE_OR_UNBOUNDED": TerminationCondition.infeasibleOrUnbounded, + "SOLUTION_LIM": TerminationCondition.maxEvaluations, + "NODE_LIM_SOL": TerminationCondition.maxEvaluations, + "NODE_LIM_NOSOL": TerminationCondition.maxEvaluations, + "ITERATION_LIMIT_REACHED": TerminationCondition.maxIterations, + "TIME_LIM_SOL": TerminationCondition.maxTimeLimit, + "TIME_LIM_NOSOL": TerminationCondition.maxTimeLimit, + "TIME_LIMIT_REACHED": TerminationCondition.maxTimeLimit, + "ABORTED": TerminationCondition.userInterrupt, + "ABORT_SOL": TerminationCondition.userInterrupt, + "ABORT_NOSOL": TerminationCondition.userInterrupt, + "OUTMEM_SOL": TerminationCondition.solverFailure, + "OUTMEM_NOSOL": TerminationCondition.solverFailure, + "FAILED": TerminationCondition.solverFailure, + "FAIL_SOL": TerminationCondition.solverFailure, + "FAIL_NOSOL": TerminationCondition.solverFailure, +} + + +SOLSTATUS_TO_MESSAGE = { + "OPTIMAL": "The solution is optimal.", + "OPTIMAL_AGAP": "The solution is optimal within the absolute gap specified by the ABSOBJGAP= option.", + "OPTIMAL_RGAP": "The solution is optimal within the relative gap specified by the RELOBJGAP= option.", + "OPTIMAL_COND": "The solution is optimal, but some infeasibilities (primal, bound, or integer) exceed tolerances due to scaling or choice of a small INTTOL= value.", + "TARGET": "The solution is not worse than the target specified by the TARGET= option.", + "CONDITIONAL_OPTIMAL": "The solution is optimal, but some infeasibilities (primal, dual or bound) exceed tolerances due to scaling or preprocessing.", + "FEASIBLE": "The problem is feasible. This status is displayed when the IIS=TRUE option is specified and the problem is feasible.", + "INFEASIBLE": "The problem is infeasible.", + "UNBOUNDED": "The problem is unbounded.", + "INFEASIBLE_OR_UNBOUNDED": "The problem is infeasible or unbounded.", + "SOLUTION_LIM": "The solver reached the maximum number of solutions specified by the MAXSOLS= option.", + "NODE_LIM_SOL": "The solver reached the maximum number of nodes specified by the MAXNODES= option and found a solution.", + "NODE_LIM_NOSOL": "The solver reached the maximum number of nodes specified by the MAXNODES= option and did not find a solution.", + "ITERATION_LIMIT_REACHED": "The maximum allowable number of iterations was reached.", + "TIME_LIM_SOL": "The solver reached the execution time limit specified by the MAXTIME= option and found a solution.", + "TIME_LIM_NOSOL": "The solver reached the execution time limit specified by the MAXTIME= option and did not find a solution.", + "TIME_LIMIT_REACHED": "The solver reached its execution time limit.", + "ABORTED": "The solver was interrupted externally.", + "ABORT_SOL": "The solver was stopped by the user but still found a solution.", + "ABORT_NOSOL": "The solver was stopped by the user and did not find a solution.", + "OUTMEM_SOL": "The solver ran out of memory but still found a solution.", + "OUTMEM_NOSOL": "The solver ran out of memory and either did not find a solution or failed to output the solution due to insufficient memory.", + "FAILED": "The solver failed to converge, possibly due to numerical issues.", + "FAIL_SOL": "The solver stopped due to errors but still found a solution.", + "FAIL_NOSOL": "The solver stopped due to errors and did not find a solution.", +} + + +CAS_OPTION_NAMES = [ + "hostname", + "port", + "username", + "password", + "session", + "locale", + "name", + "nworkers", + "authinfo", + "protocol", + "path", + "ssl_ca_list", + "authcode", +] + + +@SolverFactory.register('sas', doc='The SAS LP/MIP solver') +class SAS(OptSolver): + """The SAS optimization solver""" + + def __new__(cls, *args, **kwds): + mode = kwds.pop('solver_io', None) + if mode != None: + return SolverFactory(mode) + else: + # Choose solver factory automatically + # bassed on what can be loaded. + s = SolverFactory('_sas94', **kwds) + if not s.available(): + s = SolverFactory('_sascas', **kwds) + return s + + +class SASAbc(ABC, OptSolver): + """Abstract base class for the SAS solver interfaces. Simply to avoid code duplication.""" + + def __init__(self, **kwds): + """Initialize the SAS solver interfaces.""" + kwds['type'] = 'sas' + super(SASAbc, self).__init__(**kwds) + + # + # Set up valid problem formats and valid results for each + # problem format + # + self._valid_problem_formats = [ProblemFormat.mps] + self._valid_result_formats = {ProblemFormat.mps: [ResultsFormat.soln]} + + self._keepfiles = False + self._capabilities.linear = True + self._capabilities.integer = True + + super(SASAbc, self).set_problem_format(ProblemFormat.mps) + + def _presolve(self, *args, **kwds): + """ "Set things up for the actual solve.""" + # create a context in the temporary file manager for + # this plugin - is "pop"ed in the _postsolve method. + TempfileManager.push() + + # Get the warmstart flag + self.warmstart_flag = kwds.pop('warmstart', False) + + # Call parent presolve function + super(SASAbc, self)._presolve(*args, **kwds) + + # Store the model, too bad this is not done in the base class + for arg in args: + if isinstance(arg, (_BlockData, IBlock)): + # Store the instance + self._instance = arg + self._vars = [] + for block in self._instance.block_data_objects(active=True): + for vardata in block.component_data_objects( + Var, active=True, descend_into=False + ): + self._vars.append(vardata) + # Store the symbal map, we need this for example when writing the warmstart file + if isinstance(self._instance, IBlock): + self._smap = getattr(self._instance, "._symbol_maps")[self._smap_id] + else: + self._smap = self._instance.solutions.symbol_map[self._smap_id] + + # Create the primalin data + if self.warmstart_flag: + filename = self._warm_start_file_name = TempfileManager.create_tempfile( + ".sol", text=True + ) + smap = self._smap + numWritten = 0 + with open(filename, 'w') as file: + file.write('_VAR_,_VALUE_\n') + for var in self._vars: + if (var.value is not None) and (id(var) in smap.byObject): + name = smap.byObject[id(var)] + file.write( + "{name},{value}\n".format(name=name, value=var.value) + ) + numWritten += 1 + if numWritten == 0: + # No solution available, disable warmstart + self.warmstart_flag = False + + def available(self, exception_flag=False): + """True if the solver is available""" + return self._python_api_exists + + def _has_integer_variables(self): + """True if the problem has integer variables.""" + for vardata in self._vars: + if vardata.is_binary() or vardata.is_integer(): + return True + return False + + def _create_results_from_status(self, status, solution_status): + """Create a results object and set the status code and messages.""" + results = SolverResults() + results.solver.name = "SAS" + results.solver.status = STATUS_TO_SOLVERSTATUS[status] + if results.solver.status == SolverStatus.ok: + results.solver.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[ + solution_status + ] + results.solver.message = ( + results.solver.termination_message + ) = SOLSTATUS_TO_MESSAGE[solution_status] + results.solver.status = TerminationCondition.to_solver_status( + results.solver.termination_condition + ) + elif results.solver.status == SolverStatus.aborted: + results.solver.termination_condition = TerminationCondition.userInterrupt + results.solver.message = ( + results.solver.termination_message + ) = SOLSTATUS_TO_MESSAGE["ABORTED"] + else: + results.solver.termination_condition = TerminationCondition.error + results.solver.message = ( + results.solver.termination_message + ) = SOLSTATUS_TO_MESSAGE["FAILED"] + return results + + @abstractmethod + def _apply_solver(self): + """The routine that performs the solve""" + raise NotImplemented("This is an abstract function and thus not implemented!") + + def _postsolve(self): + """Clean up at the end, especially the temp files.""" + # Let the base class deal with returning results. + results = super(SASAbc, self)._postsolve() + + # Finally, clean any temporary files registered with the temp file + # manager, created populated *directly* by this plugin. does not + # include, for example, the execution script. but does include + # the warm-start file. + TempfileManager.pop(remove=not self._keepfiles) + + return results + + def warm_start_capable(self): + """True if the solver interface supports MILP warmstarting.""" + return True + + +@SolverFactory.register('_sas94', doc='SAS 9.4 interface') +class SAS94(SASAbc): + """ + Solver interface for SAS 9.4 using saspy. See the saspy documentation about + how to create a connection. + """ + + def __init__(self, **kwds): + """Initialize the solver interface and see if the saspy package is available.""" + super(SAS94, self).__init__(**kwds) + + try: + import saspy + + self._sas = saspy + except ImportError: + self._python_api_exists = False + except Exception as e: + self._python_api_exists = False + # For other exceptions, raise it so that it does not get lost + raise e + else: + self._python_api_exists = True + self._sas.logger.setLevel(logger.level) + + def _create_statement_str(self, statement): + """Helper function to create the strings for the statements of the proc OPTLP/OPTMILP code.""" + stmt = self.options.pop(statement, None) + if stmt: + return ( + statement.strip() + + " " + + " ".join(option + "=" + str(value) for option, value in stmt.items()) + + ";" + ) + else: + return "" + + def _apply_solver(self): + """ "Prepare the options and run the solver. Then store the data to be returned.""" + logger.debug("Running SAS") + + # Set return code to issue an error if we get interrupted + self._rc = -1 + + # Figure out if the problem has integer variables + with_opt = self.options.pop("with", None) + if with_opt == "lp": + proc = "OPTLP" + elif with_opt == "milp": + proc = "OPTMILP" + else: + # Check if there are integer variables, this might be slow + proc = "OPTMILP" if self._has_integer_variables() else "OPTLP" + + # Remove CAS options in case they were specified + for opt in CAS_OPTION_NAMES: + self.options.pop(opt, None) + + # Get the rootnode options + decomp_str = self._create_statement_str("decomp") + decompmaster_str = self._create_statement_str("decompmaster") + decompmasterip_str = self._create_statement_str("decompmasterip") + decompsubprob_str = self._create_statement_str("decompsubprob") + rootnode_str = self._create_statement_str("rootnode") + + # Handle warmstart + warmstart_str = "" + if self.warmstart_flag: + # Set the warmstart basis option + if proc != "OPTLP": + warmstart_str = """ + proc import datafile='{primalin}' + out=primalin + dbms=csv + replace; + getnames=yes; + run; + """.format( + primalin=self._warm_start_file_name + ) + self.options["primalin"] = "primalin" + + # Convert options to string + opt_str = " ".join( + option + "=" + str(value) for option, value in self.options.items() + ) + + # Start a SAS session, submit the code and return the results`` + with self._sas.SASsession() as sas: + # Find the version of 9.4 we are using + if sas.sasver.startswith("9.04.01M5"): + # In 9.4M5 we have to create an MPS data set from an MPS file first + # Earlier versions will not work because the MPS format in incompatible + res = sas.submit( + """ + {warmstart} + %MPS2SASD(MPSFILE="{mpsfile}", OUTDATA=mpsdata, MAXLEN=256, FORMAT=FREE); + proc {proc} data=mpsdata {options} primalout=primalout dualout=dualout; + {decomp} + {decompmaster} + {decompmasterip} + {decompsubprob} + {rootnode} + run; + """.format( + warmstart=warmstart_str, + proc=proc, + mpsfile=self._problem_files[0], + options=opt_str, + decomp=decomp_str, + decompmaster=decompmaster_str, + decompmasterip=decompmasterip_str, + decompsubprob=decompsubprob_str, + rootnode=rootnode_str, + ), + results="TEXT", + ) + else: + # Since 9.4M6+ optlp/optmilp can read mps files directly + res = sas.submit( + """ + {warmstart} + proc {proc} mpsfile=\"{mpsfile}\" {options} primalout=primalout dualout=dualout; + {decomp} + {decompmaster} + {decompmasterip} + {decompsubprob} + {rootnode} + run; + """.format( + warmstart=warmstart_str, + proc=proc, + mpsfile=self._problem_files[0], + options=opt_str, + decomp=decomp_str, + decompmaster=decompmaster_str, + decompmasterip=decompmasterip_str, + decompsubprob=decompsubprob_str, + rootnode=rootnode_str, + ), + results="TEXT", + ) + + # Store log and ODS output + self._log = res["LOG"] + self._lst = res["LST"] + # Print log if requested by the user + if self._tee: + print(self._log) + if "ERROR 22-322: Syntax error" in self._log: + raise ValueError( + "An option passed to the SAS solver caused a syntax error: {log}".format( + log=self._log + ) + ) + self._macro = dict( + (key.strip(), value.strip()) + for key, value in ( + pair.split("=") for pair in sas.symget("_OR" + proc + "_").split() + ) + ) + primal_out = sas.sd2df("primalout") + dual_out = sas.sd2df("dualout") + + # Prepare the solver results + results = self.results = self._create_results_from_status( + self._macro.get("STATUS", "ERROR"), self._macro.get("SOLUTION_STATUS","ERROR") + ) + + if "Objective Sense Maximization" in self._lst: + results.problem.sense = ProblemSense.maximize + else: + results.problem.sense = ProblemSense.minimize + + # Prepare the solution information + if results.solver.termination_condition == TerminationCondition.optimal: + sol = results.solution.add() + + # Store status in solution + sol.status = SolutionStatus.feasible + sol.termination_condition = TerminationCondition.optimal + + # Store objective value in solution + sol.objective['__default_objective__'] = {'Value': self._macro["OBJECTIVE"]} + + if proc == "OPTLP": + # Convert primal out data set to variable dictionary + # Use panda functions for efficiency + primal_out = primal_out[['_VAR_', '_VALUE_', '_STATUS_', '_R_COST_']] + primal_out = primal_out.set_index('_VAR_', drop=True) + primal_out = primal_out.rename( + {'_VALUE_': 'Value', '_STATUS_': 'Status', '_R_COST_': 'rc'}, + axis='columns', + ) + sol.variable = primal_out.to_dict('index') + + # Convert dual out data set to constraint dictionary + # Use pandas functions for efficiency + dual_out = dual_out[['_ROW_', '_VALUE_', '_STATUS_', '_ACTIVITY_']] + dual_out = dual_out.set_index('_ROW_', drop=True) + dual_out = dual_out.rename( + {'_VALUE_': 'dual', '_STATUS_': 'Status', '_ACTIVITY_': 'slack'}, + axis='columns', + ) + sol.constraint = dual_out.to_dict('index') + else: + # Convert primal out data set to variable dictionary + # Use pandas functions for efficiency + primal_out = primal_out[['_VAR_', '_VALUE_']] + primal_out = primal_out.set_index('_VAR_', drop=True) + primal_out = primal_out.rename({'_VALUE_': 'Value'}, axis='columns') + sol.variable = primal_out.to_dict('index') + + self._rc = 0 + return Bunch(rc=self._rc, log=self._log) + + +class SASLogWriter: + """Helper class to take the log from stdout and put it also in a StringIO.""" + + def __init__(self, tee): + """Set up the two outputs.""" + self.tee = tee + self._log = StringIO() + self.stdout = sys.stdout + + def write(self, message): + """If the tee options is specified, write to both outputs.""" + if self.tee: + self.stdout.write(message) + self._log.write(message) + + def flush(self): + """Nothing to do, just here for compatibility reasons.""" + # Do nothing since we flush right away + pass + + def log(self): + """ "Get the log as a string.""" + return self._log.getvalue() + + +@SolverFactory.register('_sascas', doc='SAS Viya CAS Server interface') +class SASCAS(SASAbc): + """ + Solver interface connection to a SAS Viya CAS server using swat. + See the documentation for the swat package about how to create a connection. + The swat connection options can be passed as options to the solve function. + """ + + def __init__(self, **kwds): + """Initialize and try to load the swat package.""" + super(SASCAS, self).__init__(**kwds) + + try: + import swat + + self._sas = swat + except ImportError: + self._python_api_exists = False + except Exception as e: + self._python_api_exists = False + # For other exceptions, raise it so that it does not get lost + raise e + else: + self._python_api_exists = True + + def _apply_solver(self): + """ "Prepare the options and run the solver. Then store the data to be returned.""" + logger.debug("Running SAS Viya") + + # Set return code to issue an error if we get interrupted + self._rc = -1 + + # Extract CAS connection options + cas_opts = {} + for opt in CAS_OPTION_NAMES: + val = self.options.pop(opt, None) + if val != None: + cas_opts[opt] = val + + # Figure out if the problem has integer variables + with_opt = self.options.pop("with", None) + if with_opt == "lp": + action = "solveLp" + elif with_opt == "milp": + action = "solveMilp" + else: + # Check if there are integer variables, this might be slow + action = "solveMilp" if self._has_integer_variables() else "solveLp" + + # Connect to CAS server + with redirect_stdout(SASLogWriter(self._tee)) as self._log_writer: + s = self._sas.CAS(**cas_opts) + try: + # Load the optimization action set + s.loadactionset('optimization') + + # Upload mps file to CAS + if os.stat(self._problem_files[0]).st_size >= 2 * 1024**3: + # For large files, use convertMPS, first create file for upload + mpsWithIdFileName = TempfileManager.create_tempfile( + ".mps.csv", text=True + ) + with open(mpsWithIdFileName, 'w') as mpsWithId: + mpsWithId.write('_ID_\tText\n') + with open(self._problem_files[0], 'r') as f: + id = 0 + for line in f: + id += 1 + mpsWithId.write(str(id) + '\t' + line.rstrip() + '\n') + + # Upload .mps.csv file + s.upload_file( + mpsWithIdFileName, + casout={"name": "mpscsv", "replace": True}, + importoptions={"filetype": "CSV", "delimiter": "\t"}, + ) + + # Convert .mps.csv file to .mps + s.optimization.convertMps( + data="mpscsv", + casOut={"name": "mpsdata", "replace": True}, + format="FREE", + ) + else: + # For small files, use loadMPS + with open(self._problem_files[0], 'r') as mps_file: + s.optimization.loadMps( + mpsFileString=mps_file.read(), + casout={"name": "mpsdata", "replace": True}, + format="FREE", + ) + + if self.warmstart_flag: + # Upload warmstart file to CAS + s.upload_file( + self._warm_start_file_name, + casout={"name": "primalin", "replace": True}, + importoptions={"filetype": "CSV"}, + ) + self.options["primalin"] = "primalin" + + # Solve the problem in CAS + if action == "solveMilp": + r = s.optimization.solveMilp( + data={"name": "mpsdata"}, + primalOut={"name": "primalout", "replace": True}, + **self.options + ) + else: + r = s.optimization.solveLp( + data={"name": "mpsdata"}, + primalOut={"name": "primalout", "replace": True}, + dualOut={"name": "dualout", "replace": True}, + **self.options + ) + + # Prepare the solver results + if r: + # Get back the primal and dual solution data sets + results = self.results = self._create_results_from_status( + r.get("status", "ERROR"), r.get("solutionStatus", "ERROR") + ) + + if r.ProblemSummary["cValue1"][1] == "Maximization": + results.problem.sense = ProblemSense.maximize + else: + results.problem.sense = ProblemSense.minimize + + # Prepare the solution information + if ( + results.solver.termination_condition + == TerminationCondition.optimal + ): + sol = results.solution.add() + + # Store status in solution + sol.status = SolutionStatus.feasible + sol.termination_condition = TerminationCondition.optimal + + # Store objective value in solution + sol.objective['__default_objective__'] = { + 'Value': r["objective"] + } + + if action == "solveMilp": + primal_out = s.CASTable(name="primalout") + # Use pandas functions for efficiency + primal_out = primal_out[['_VAR_', '_VALUE_']] + sol.variable = {} + for row in primal_out.itertuples(index=False): + sol.variable[row[0]] = {'Value': row[1]} + else: + # Convert primal out data set to variable dictionary + # Use panda functions for efficiency + primal_out = s.CASTable(name="primalout") + primal_out = primal_out[ + ['_VAR_', '_VALUE_', '_STATUS_', '_R_COST_'] + ] + sol.variable = {} + for row in primal_out.itertuples(index=False): + sol.variable[row[0]] = { + 'Value': row[1], + 'Status': row[2], + 'rc': row[3], + } + + # Convert dual out data set to constraint dictionary + # Use pandas functions for efficiency + dual_out = s.CASTable(name="dualout") + dual_out = dual_out[ + ['_ROW_', '_VALUE_', '_STATUS_', '_ACTIVITY_'] + ] + sol.constraint = {} + for row in dual_out.itertuples(index=False): + sol.constraint[row[0]] = { + 'dual': row[1], + 'Status': row[2], + 'slack': row[3], + } + else: + results = self.results = SolverResults() + results.solver.name = "SAS" + results.solver.status = SolverStatus.error + raise ValueError( + "An option passed to the SAS solver caused a syntax error." + ) + + finally: + s.close() + + self._log = self._log_writer.log() + if self._tee: + print(self._log) + self._rc = 0 + return Bunch(rc=self._rc, log=self._log) diff --git a/pyomo/solvers/plugins/solvers/__init__.py b/pyomo/solvers/plugins/solvers/__init__.py index c5fbfa97e42..23b7fe06526 100644 --- a/pyomo/solvers/plugins/solvers/__init__.py +++ b/pyomo/solvers/plugins/solvers/__init__.py @@ -30,3 +30,4 @@ import pyomo.solvers.plugins.solvers.mosek_persistent import pyomo.solvers.plugins.solvers.xpress_direct import pyomo.solvers.plugins.solvers.xpress_persistent +import pyomo.solvers.plugins.solvers.SAS diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py new file mode 100644 index 00000000000..4592343b17f --- /dev/null +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -0,0 +1,462 @@ +import os +import pyomo.common.unittest as unittest +from pyomo.environ import ( + ConcreteModel, + Var, + Objective, + Constraint, + NonNegativeIntegers, + NonNegativeReals, + Reals, + Integers, + maximize, + minimize, + Suffix, +) +from pyomo.opt.results import ( + SolverStatus, + TerminationCondition, + ProblemSense, +) +from pyomo.opt import ( + SolverFactory, + check_available_solvers, +) + + +CAS_OPTIONS = { + "hostname": os.environ.get('CAS_SERVER', None), + "port": os.environ.get('CAS_PORT', None), + "authinfo": os.environ.get('CAS_AUTHINFO', None), +} + + +sas_available = check_available_solvers('sas') + + +class SASTestAbc: + solver_io = '_sas94' + base_options = {} + + def setObj(self): + X = self.instance.X + self.instance.Obj = Objective( + expr=2 * X[1] - 3 * X[2] - 4 * X[3], sense=minimize + ) + + def setX(self): + self.instance.X = Var([1, 2, 3], within=NonNegativeReals) + + def setUp(self): + instance = self.instance = ConcreteModel() + self.setX() + X = instance.X + instance.R1 = Constraint(expr=-2 * X[2] - 3 * X[3] >= -5) + instance.R2 = Constraint(expr=X[1] + X[2] + 2 * X[3] <= 4) + instance.R3 = Constraint(expr=X[1] + 2 * X[2] + 3 * X[3] <= 7) + self.setObj() + + # Declare suffixes for solution information + instance.status = Suffix(direction=Suffix.IMPORT) + instance.slack = Suffix(direction=Suffix.IMPORT) + instance.rc = Suffix(direction=Suffix.IMPORT) + instance.dual = Suffix(direction=Suffix.IMPORT) + + self.opt_sas = SolverFactory('sas', solver_io=self.solver_io) + + def tearDown(self): + del self.opt_sas + del self.instance + + def run_solver(self, **kwargs): + opt_sas = self.opt_sas + instance = self.instance + + # Add base options for connection data etc. + options = kwargs.get("options", {}) + if self.base_options: + kwargs["options"] = {**options, **self.base_options} + + # Call the solver + self.results = opt_sas.solve(instance, **kwargs) + + +class SASTestLP(SASTestAbc, unittest.TestCase): + def checkSolution(self): + instance = self.instance + results = self.results + # Get the objective sense, we use the same code for minimization and maximization tests + sense = instance.Obj.sense + + # Check status + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) + + # Check objective value + self.assertAlmostEqual(instance.Obj(), sense * -7.5) + + # Check primal solution values + self.assertAlmostEqual(instance.X[1].value, 0.0) + self.assertAlmostEqual(instance.X[2].value, 2.5) + self.assertAlmostEqual(instance.X[3].value, 0.0) + + # Check reduced cost + self.assertAlmostEqual(instance.rc[instance.X[1]], sense * 2.0) + self.assertAlmostEqual(instance.rc[instance.X[2]], sense * 0.0) + self.assertAlmostEqual(instance.rc[instance.X[3]], sense * 0.5) + + # Check slack + self.assertAlmostEqual(instance.slack[instance.R1], -5.0) + self.assertAlmostEqual(instance.slack[instance.R2], 2.5) + self.assertAlmostEqual(instance.slack[instance.R3], 5.0) + + # Check dual solution + self.assertAlmostEqual(instance.dual[instance.R1], sense * 1.5) + self.assertAlmostEqual(instance.dual[instance.R2], sense * 0.0) + self.assertAlmostEqual(instance.dual[instance.R3], sense * 0.0) + + # Check basis status + self.assertEqual(instance.status[instance.X[1]], 'L') + self.assertEqual(instance.status[instance.X[2]], 'B') + self.assertEqual(instance.status[instance.X[3]], 'L') + self.assertEqual(instance.status[instance.R1], 'U') + self.assertEqual(instance.status[instance.R2], 'B') + self.assertEqual(instance.status[instance.R3], 'B') + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_default(self): + self.run_solver() + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_primal(self): + self.run_solver(options={"algorithm": "ps"}) + self.assertIn("NOTE: The Primal Simplex algorithm is used.", self.opt_sas._log) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_ipm(self): + self.run_solver(options={"algorithm": "ip"}) + self.assertIn("NOTE: The Interior Point algorithm is used.", self.opt_sas._log) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_intoption(self): + self.run_solver(options={"maxiter": 20}) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_invalidoption(self): + with self.assertRaisesRegex(ValueError, "syntax error"): + self.run_solver(options={"foo": "bar"}) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_max(self): + X = self.instance.X + self.instance.Obj.set_value(expr=-2 * X[1] + 3 * X[2] + 4 * X[3]) + self.instance.Obj.sense = maximize + self.run_solver() + self.checkSolution() + self.assertEqual(self.results.problem.sense, ProblemSense.maximize) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_infeasible(self): + instance = self.instance + X = instance.X + instance.R4 = Constraint(expr=-2 * X[2] - 3 * X[3] <= -6) + self.run_solver() + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.infeasible + ) + self.assertEqual(results.solver.message, "The problem is infeasible.") + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_infeasible_or_unbounded(self): + self.instance.X.domain = Reals + self.run_solver() + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, + TerminationCondition.infeasibleOrUnbounded, + ) + self.assertEqual( + results.solver.message, "The problem is infeasible or unbounded." + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_unbounded(self): + self.instance.X.domain = Reals + self.run_solver(options={"presolver": "none", "algorithm": "primal"}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.unbounded + ) + self.assertEqual(results.solver.message, "The problem is unbounded.") + + def checkSolutionDecomp(self): + instance = self.instance + results = self.results + # Get the objective sense, we use the same code for minimization and maximization tests + sense = instance.Obj.sense + + # Check status + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) + + # Check objective value + self.assertAlmostEqual(instance.Obj(), sense * -7.5) + + # Check primal solution values + self.assertAlmostEqual(instance.X[1].value, 0.0) + self.assertAlmostEqual(instance.X[2].value, 2.5) + self.assertAlmostEqual(instance.X[3].value, 0.0) + + # Check reduced cost + self.assertAlmostEqual(instance.rc[instance.X[1]], sense * 2.0) + self.assertAlmostEqual(instance.rc[instance.X[2]], sense * 0.0) + self.assertAlmostEqual(instance.rc[instance.X[3]], sense * 0.5) + + # Check slack + self.assertAlmostEqual(instance.slack[instance.R1], -5.0) + self.assertAlmostEqual(instance.slack[instance.R2], 2.5) + self.assertAlmostEqual(instance.slack[instance.R3], 5.0) + + # Check dual solution + self.assertAlmostEqual(instance.dual[instance.R1], sense * 1.5) + self.assertAlmostEqual(instance.dual[instance.R2], sense * 0.0) + self.assertAlmostEqual(instance.dual[instance.R3], sense * 0.0) + + # Don't check basis status for decomp + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_decomp(self): + self.run_solver( + options={ + "decomp": {"absobjgap": 0.0}, + "decompmaster": {"algorithm": "dual"}, + "decompsubprob": {"presolver": "none"}, + } + ) + self.assertIn( + "NOTE: The DECOMP method value DEFAULT is applied.", self.opt_sas._log + ) + self.checkSolutionDecomp() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_iis(self): + self.run_solver(options={"iis": "true"}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.feasible + ) + self.assertIn("NOTE: The IIS= option is enabled.", self.opt_sas._log) + self.assertEqual( + results.solver.message, + "The problem is feasible. This status is displayed when the IIS=TRUE option is specified and the problem is feasible.", + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_maxiter(self): + self.run_solver(options={"maxiter": 1}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.maxIterations + ) + self.assertEqual( + results.solver.message, + "The maximum allowable number of iterations was reached.", + ) + + +class SASTestLPCAS(SASTestLP): + solver_io = '_sascas' + base_options = CAS_OPTIONS + + +class SASTestMILP(SASTestAbc, unittest.TestCase): + def setX(self): + self.instance.X = Var([1, 2, 3], within=NonNegativeIntegers) + + def checkSolution(self): + instance = self.instance + results = self.results + + # Get the objective sense, we use the same code for minimization and maximization tests + sense = instance.Obj.sense + + # Check status + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) + + # Check objective value + self.assertAlmostEqual(instance.Obj(), sense * -7) + + # Check primal solution values + self.assertAlmostEqual(instance.X[1].value, 0.0) + self.assertAlmostEqual(instance.X[2].value, 1.0) + self.assertAlmostEqual(instance.X[3].value, 1.0) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_default(self): + self.run_solver(options={}) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_presolve(self): + self.run_solver(options={"presolver": "none"}) + self.assertIn( + "NOTE: The MILP presolver value NONE is applied.", self.opt_sas._log + ) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_intoption(self): + self.run_solver(options={"maxnodes": 20}) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_invalidoption(self): + with self.assertRaisesRegex(ValueError, "syntax error"): + self.run_solver(options={"foo": "bar"}) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_max(self): + X = self.instance.X + self.instance.Obj.set_value(expr=-2 * X[1] + 3 * X[2] + 4 * X[3]) + self.instance.Obj.sense = maximize + self.run_solver() + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_infeasible(self): + instance = self.instance + X = instance.X + instance.R4 = Constraint(expr=-2 * X[2] - 3 * X[3] <= -6) + self.run_solver() + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.infeasible + ) + self.assertEqual(results.solver.message, "The problem is infeasible.") + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + @unittest.skip("Returns wrong status for some versions.") + def test_solver_infeasible_or_unbounded(self): + self.instance.X.domain = Integers + self.run_solver() + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, + TerminationCondition.infeasibleOrUnbounded, + ) + self.assertEqual( + results.solver.message, "The problem is infeasible or unbounded." + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_unbounded(self): + self.instance.X.domain = Integers + self.run_solver( + options={"presolver": "none", "rootnode": {"algorithm": "primal"}} + ) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.unbounded + ) + self.assertEqual(results.solver.message, "The problem is unbounded.") + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_decomp(self): + self.run_solver( + options={ + "decomp": {"hybrid": "off"}, + "decompmaster": {"algorithm": "dual"}, + "decompmasterip": {"presolver": "none"}, + "decompsubprob": {"presolver": "none"}, + } + ) + self.assertIn( + "NOTE: The DECOMP method value DEFAULT is applied.", self.opt_sas._log + ) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_rootnode(self): + self.run_solver(options={"rootnode": {"presolver": "automatic"}}) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_maxnodes(self): + self.run_solver(options={"maxnodes": 0}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.maxEvaluations + ) + self.assertEqual( + results.solver.message, + "The solver reached the maximum number of nodes specified by the MAXNODES= option and found a solution.", + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_maxsols(self): + self.run_solver(options={"maxsols": 1}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.maxEvaluations + ) + self.assertEqual( + results.solver.message, + "The solver reached the maximum number of solutions specified by the MAXSOLS= option.", + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_target(self): + self.run_solver(options={"target": -6.0}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) + self.assertEqual( + results.solver.message, + "The solution is not worse than the target specified by the TARGET= option.", + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_primalin(self): + X = self.instance.X + X[1] = None + X[2] = 3 + X[3] = 7 + self.run_solver(warmstart=True) + self.checkSolution() + self.assertIn( + "NOTE: The input solution is infeasible or incomplete. Repair heuristics are applied.", + self.opt_sas._log, + ) + + +class SASTestMILPCAS(SASTestMILP): + solver_io = '_sascas' + base_options = CAS_OPTIONS + + +if __name__ == '__main__': + unittest.main() From be423b99adfa97a6cc3d3cb20985398891acecbe Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Fri, 23 Jun 2023 09:56:54 -0400 Subject: [PATCH 02/20] Just some black adjustments --- pyomo/solvers/plugins/solvers/SAS.py | 3 ++- pyomo/solvers/tests/checks/test_SAS.py | 11 ++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index 7f50b7a2970..ed0e63d44d6 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -427,7 +427,8 @@ def _apply_solver(self): # Prepare the solver results results = self.results = self._create_results_from_status( - self._macro.get("STATUS", "ERROR"), self._macro.get("SOLUTION_STATUS","ERROR") + self._macro.get("STATUS", "ERROR"), + self._macro.get("SOLUTION_STATUS", "ERROR"), ) if "Objective Sense Maximization" in self._lst: diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 4592343b17f..654820f5060 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -13,15 +13,8 @@ minimize, Suffix, ) -from pyomo.opt.results import ( - SolverStatus, - TerminationCondition, - ProblemSense, -) -from pyomo.opt import ( - SolverFactory, - check_available_solvers, -) +from pyomo.opt.results import SolverStatus, TerminationCondition, ProblemSense +from pyomo.opt import SolverFactory, check_available_solvers CAS_OPTIONS = { From 284ab98e470cf67c5273837a28c11afbb2b153a5 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Fri, 23 Jun 2023 09:42:08 -0400 Subject: [PATCH 03/20] Initial version of the SAS solver interfaces and unit tests. --- pyomo/solvers/plugins/solvers/SAS.py | 700 ++++++++++++++++++++++ pyomo/solvers/plugins/solvers/__init__.py | 1 + pyomo/solvers/tests/checks/test_SAS.py | 462 ++++++++++++++ 3 files changed, 1163 insertions(+) create mode 100644 pyomo/solvers/plugins/solvers/SAS.py create mode 100644 pyomo/solvers/tests/checks/test_SAS.py diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py new file mode 100644 index 00000000000..7f50b7a2970 --- /dev/null +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -0,0 +1,700 @@ +__all__ = ['SAS'] + +import logging +import sys +import os + +from io import StringIO +from abc import ABC, abstractmethod +from contextlib import redirect_stdout + +from pyomo.opt.base import ProblemFormat, ResultsFormat, OptSolver +from pyomo.opt.base.solvers import SolverFactory +from pyomo.common.collections import Bunch +from pyomo.opt.results import ( + SolverResults, + SolverStatus, + TerminationCondition, + SolutionStatus, + ProblemSense, +) +from pyomo.common.tempfiles import TempfileManager +from pyomo.core.base import Var +from pyomo.core.base.block import _BlockData +from pyomo.core.kernel.block import IBlock + + +logger = logging.getLogger('pyomo.solvers') + + +STATUS_TO_SOLVERSTATUS = { + "OK": SolverStatus.ok, + "SYNTAX_ERROR": SolverStatus.error, + "DATA_ERROR": SolverStatus.error, + "OUT_OF_MEMORY": SolverStatus.aborted, + "IO_ERROR": SolverStatus.error, + "ERROR": SolverStatus.error, +} + +# This combines all status codes from OPTLP/solvelp and OPTMILP/solvemilp +SOLSTATUS_TO_TERMINATIONCOND = { + "OPTIMAL": TerminationCondition.optimal, + "OPTIMAL_AGAP": TerminationCondition.optimal, + "OPTIMAL_RGAP": TerminationCondition.optimal, + "OPTIMAL_COND": TerminationCondition.optimal, + "TARGET": TerminationCondition.optimal, + "CONDITIONAL_OPTIMAL": TerminationCondition.optimal, + "FEASIBLE": TerminationCondition.feasible, + "INFEASIBLE": TerminationCondition.infeasible, + "UNBOUNDED": TerminationCondition.unbounded, + "INFEASIBLE_OR_UNBOUNDED": TerminationCondition.infeasibleOrUnbounded, + "SOLUTION_LIM": TerminationCondition.maxEvaluations, + "NODE_LIM_SOL": TerminationCondition.maxEvaluations, + "NODE_LIM_NOSOL": TerminationCondition.maxEvaluations, + "ITERATION_LIMIT_REACHED": TerminationCondition.maxIterations, + "TIME_LIM_SOL": TerminationCondition.maxTimeLimit, + "TIME_LIM_NOSOL": TerminationCondition.maxTimeLimit, + "TIME_LIMIT_REACHED": TerminationCondition.maxTimeLimit, + "ABORTED": TerminationCondition.userInterrupt, + "ABORT_SOL": TerminationCondition.userInterrupt, + "ABORT_NOSOL": TerminationCondition.userInterrupt, + "OUTMEM_SOL": TerminationCondition.solverFailure, + "OUTMEM_NOSOL": TerminationCondition.solverFailure, + "FAILED": TerminationCondition.solverFailure, + "FAIL_SOL": TerminationCondition.solverFailure, + "FAIL_NOSOL": TerminationCondition.solverFailure, +} + + +SOLSTATUS_TO_MESSAGE = { + "OPTIMAL": "The solution is optimal.", + "OPTIMAL_AGAP": "The solution is optimal within the absolute gap specified by the ABSOBJGAP= option.", + "OPTIMAL_RGAP": "The solution is optimal within the relative gap specified by the RELOBJGAP= option.", + "OPTIMAL_COND": "The solution is optimal, but some infeasibilities (primal, bound, or integer) exceed tolerances due to scaling or choice of a small INTTOL= value.", + "TARGET": "The solution is not worse than the target specified by the TARGET= option.", + "CONDITIONAL_OPTIMAL": "The solution is optimal, but some infeasibilities (primal, dual or bound) exceed tolerances due to scaling or preprocessing.", + "FEASIBLE": "The problem is feasible. This status is displayed when the IIS=TRUE option is specified and the problem is feasible.", + "INFEASIBLE": "The problem is infeasible.", + "UNBOUNDED": "The problem is unbounded.", + "INFEASIBLE_OR_UNBOUNDED": "The problem is infeasible or unbounded.", + "SOLUTION_LIM": "The solver reached the maximum number of solutions specified by the MAXSOLS= option.", + "NODE_LIM_SOL": "The solver reached the maximum number of nodes specified by the MAXNODES= option and found a solution.", + "NODE_LIM_NOSOL": "The solver reached the maximum number of nodes specified by the MAXNODES= option and did not find a solution.", + "ITERATION_LIMIT_REACHED": "The maximum allowable number of iterations was reached.", + "TIME_LIM_SOL": "The solver reached the execution time limit specified by the MAXTIME= option and found a solution.", + "TIME_LIM_NOSOL": "The solver reached the execution time limit specified by the MAXTIME= option and did not find a solution.", + "TIME_LIMIT_REACHED": "The solver reached its execution time limit.", + "ABORTED": "The solver was interrupted externally.", + "ABORT_SOL": "The solver was stopped by the user but still found a solution.", + "ABORT_NOSOL": "The solver was stopped by the user and did not find a solution.", + "OUTMEM_SOL": "The solver ran out of memory but still found a solution.", + "OUTMEM_NOSOL": "The solver ran out of memory and either did not find a solution or failed to output the solution due to insufficient memory.", + "FAILED": "The solver failed to converge, possibly due to numerical issues.", + "FAIL_SOL": "The solver stopped due to errors but still found a solution.", + "FAIL_NOSOL": "The solver stopped due to errors and did not find a solution.", +} + + +CAS_OPTION_NAMES = [ + "hostname", + "port", + "username", + "password", + "session", + "locale", + "name", + "nworkers", + "authinfo", + "protocol", + "path", + "ssl_ca_list", + "authcode", +] + + +@SolverFactory.register('sas', doc='The SAS LP/MIP solver') +class SAS(OptSolver): + """The SAS optimization solver""" + + def __new__(cls, *args, **kwds): + mode = kwds.pop('solver_io', None) + if mode != None: + return SolverFactory(mode) + else: + # Choose solver factory automatically + # bassed on what can be loaded. + s = SolverFactory('_sas94', **kwds) + if not s.available(): + s = SolverFactory('_sascas', **kwds) + return s + + +class SASAbc(ABC, OptSolver): + """Abstract base class for the SAS solver interfaces. Simply to avoid code duplication.""" + + def __init__(self, **kwds): + """Initialize the SAS solver interfaces.""" + kwds['type'] = 'sas' + super(SASAbc, self).__init__(**kwds) + + # + # Set up valid problem formats and valid results for each + # problem format + # + self._valid_problem_formats = [ProblemFormat.mps] + self._valid_result_formats = {ProblemFormat.mps: [ResultsFormat.soln]} + + self._keepfiles = False + self._capabilities.linear = True + self._capabilities.integer = True + + super(SASAbc, self).set_problem_format(ProblemFormat.mps) + + def _presolve(self, *args, **kwds): + """ "Set things up for the actual solve.""" + # create a context in the temporary file manager for + # this plugin - is "pop"ed in the _postsolve method. + TempfileManager.push() + + # Get the warmstart flag + self.warmstart_flag = kwds.pop('warmstart', False) + + # Call parent presolve function + super(SASAbc, self)._presolve(*args, **kwds) + + # Store the model, too bad this is not done in the base class + for arg in args: + if isinstance(arg, (_BlockData, IBlock)): + # Store the instance + self._instance = arg + self._vars = [] + for block in self._instance.block_data_objects(active=True): + for vardata in block.component_data_objects( + Var, active=True, descend_into=False + ): + self._vars.append(vardata) + # Store the symbal map, we need this for example when writing the warmstart file + if isinstance(self._instance, IBlock): + self._smap = getattr(self._instance, "._symbol_maps")[self._smap_id] + else: + self._smap = self._instance.solutions.symbol_map[self._smap_id] + + # Create the primalin data + if self.warmstart_flag: + filename = self._warm_start_file_name = TempfileManager.create_tempfile( + ".sol", text=True + ) + smap = self._smap + numWritten = 0 + with open(filename, 'w') as file: + file.write('_VAR_,_VALUE_\n') + for var in self._vars: + if (var.value is not None) and (id(var) in smap.byObject): + name = smap.byObject[id(var)] + file.write( + "{name},{value}\n".format(name=name, value=var.value) + ) + numWritten += 1 + if numWritten == 0: + # No solution available, disable warmstart + self.warmstart_flag = False + + def available(self, exception_flag=False): + """True if the solver is available""" + return self._python_api_exists + + def _has_integer_variables(self): + """True if the problem has integer variables.""" + for vardata in self._vars: + if vardata.is_binary() or vardata.is_integer(): + return True + return False + + def _create_results_from_status(self, status, solution_status): + """Create a results object and set the status code and messages.""" + results = SolverResults() + results.solver.name = "SAS" + results.solver.status = STATUS_TO_SOLVERSTATUS[status] + if results.solver.status == SolverStatus.ok: + results.solver.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[ + solution_status + ] + results.solver.message = ( + results.solver.termination_message + ) = SOLSTATUS_TO_MESSAGE[solution_status] + results.solver.status = TerminationCondition.to_solver_status( + results.solver.termination_condition + ) + elif results.solver.status == SolverStatus.aborted: + results.solver.termination_condition = TerminationCondition.userInterrupt + results.solver.message = ( + results.solver.termination_message + ) = SOLSTATUS_TO_MESSAGE["ABORTED"] + else: + results.solver.termination_condition = TerminationCondition.error + results.solver.message = ( + results.solver.termination_message + ) = SOLSTATUS_TO_MESSAGE["FAILED"] + return results + + @abstractmethod + def _apply_solver(self): + """The routine that performs the solve""" + raise NotImplemented("This is an abstract function and thus not implemented!") + + def _postsolve(self): + """Clean up at the end, especially the temp files.""" + # Let the base class deal with returning results. + results = super(SASAbc, self)._postsolve() + + # Finally, clean any temporary files registered with the temp file + # manager, created populated *directly* by this plugin. does not + # include, for example, the execution script. but does include + # the warm-start file. + TempfileManager.pop(remove=not self._keepfiles) + + return results + + def warm_start_capable(self): + """True if the solver interface supports MILP warmstarting.""" + return True + + +@SolverFactory.register('_sas94', doc='SAS 9.4 interface') +class SAS94(SASAbc): + """ + Solver interface for SAS 9.4 using saspy. See the saspy documentation about + how to create a connection. + """ + + def __init__(self, **kwds): + """Initialize the solver interface and see if the saspy package is available.""" + super(SAS94, self).__init__(**kwds) + + try: + import saspy + + self._sas = saspy + except ImportError: + self._python_api_exists = False + except Exception as e: + self._python_api_exists = False + # For other exceptions, raise it so that it does not get lost + raise e + else: + self._python_api_exists = True + self._sas.logger.setLevel(logger.level) + + def _create_statement_str(self, statement): + """Helper function to create the strings for the statements of the proc OPTLP/OPTMILP code.""" + stmt = self.options.pop(statement, None) + if stmt: + return ( + statement.strip() + + " " + + " ".join(option + "=" + str(value) for option, value in stmt.items()) + + ";" + ) + else: + return "" + + def _apply_solver(self): + """ "Prepare the options and run the solver. Then store the data to be returned.""" + logger.debug("Running SAS") + + # Set return code to issue an error if we get interrupted + self._rc = -1 + + # Figure out if the problem has integer variables + with_opt = self.options.pop("with", None) + if with_opt == "lp": + proc = "OPTLP" + elif with_opt == "milp": + proc = "OPTMILP" + else: + # Check if there are integer variables, this might be slow + proc = "OPTMILP" if self._has_integer_variables() else "OPTLP" + + # Remove CAS options in case they were specified + for opt in CAS_OPTION_NAMES: + self.options.pop(opt, None) + + # Get the rootnode options + decomp_str = self._create_statement_str("decomp") + decompmaster_str = self._create_statement_str("decompmaster") + decompmasterip_str = self._create_statement_str("decompmasterip") + decompsubprob_str = self._create_statement_str("decompsubprob") + rootnode_str = self._create_statement_str("rootnode") + + # Handle warmstart + warmstart_str = "" + if self.warmstart_flag: + # Set the warmstart basis option + if proc != "OPTLP": + warmstart_str = """ + proc import datafile='{primalin}' + out=primalin + dbms=csv + replace; + getnames=yes; + run; + """.format( + primalin=self._warm_start_file_name + ) + self.options["primalin"] = "primalin" + + # Convert options to string + opt_str = " ".join( + option + "=" + str(value) for option, value in self.options.items() + ) + + # Start a SAS session, submit the code and return the results`` + with self._sas.SASsession() as sas: + # Find the version of 9.4 we are using + if sas.sasver.startswith("9.04.01M5"): + # In 9.4M5 we have to create an MPS data set from an MPS file first + # Earlier versions will not work because the MPS format in incompatible + res = sas.submit( + """ + {warmstart} + %MPS2SASD(MPSFILE="{mpsfile}", OUTDATA=mpsdata, MAXLEN=256, FORMAT=FREE); + proc {proc} data=mpsdata {options} primalout=primalout dualout=dualout; + {decomp} + {decompmaster} + {decompmasterip} + {decompsubprob} + {rootnode} + run; + """.format( + warmstart=warmstart_str, + proc=proc, + mpsfile=self._problem_files[0], + options=opt_str, + decomp=decomp_str, + decompmaster=decompmaster_str, + decompmasterip=decompmasterip_str, + decompsubprob=decompsubprob_str, + rootnode=rootnode_str, + ), + results="TEXT", + ) + else: + # Since 9.4M6+ optlp/optmilp can read mps files directly + res = sas.submit( + """ + {warmstart} + proc {proc} mpsfile=\"{mpsfile}\" {options} primalout=primalout dualout=dualout; + {decomp} + {decompmaster} + {decompmasterip} + {decompsubprob} + {rootnode} + run; + """.format( + warmstart=warmstart_str, + proc=proc, + mpsfile=self._problem_files[0], + options=opt_str, + decomp=decomp_str, + decompmaster=decompmaster_str, + decompmasterip=decompmasterip_str, + decompsubprob=decompsubprob_str, + rootnode=rootnode_str, + ), + results="TEXT", + ) + + # Store log and ODS output + self._log = res["LOG"] + self._lst = res["LST"] + # Print log if requested by the user + if self._tee: + print(self._log) + if "ERROR 22-322: Syntax error" in self._log: + raise ValueError( + "An option passed to the SAS solver caused a syntax error: {log}".format( + log=self._log + ) + ) + self._macro = dict( + (key.strip(), value.strip()) + for key, value in ( + pair.split("=") for pair in sas.symget("_OR" + proc + "_").split() + ) + ) + primal_out = sas.sd2df("primalout") + dual_out = sas.sd2df("dualout") + + # Prepare the solver results + results = self.results = self._create_results_from_status( + self._macro.get("STATUS", "ERROR"), self._macro.get("SOLUTION_STATUS","ERROR") + ) + + if "Objective Sense Maximization" in self._lst: + results.problem.sense = ProblemSense.maximize + else: + results.problem.sense = ProblemSense.minimize + + # Prepare the solution information + if results.solver.termination_condition == TerminationCondition.optimal: + sol = results.solution.add() + + # Store status in solution + sol.status = SolutionStatus.feasible + sol.termination_condition = TerminationCondition.optimal + + # Store objective value in solution + sol.objective['__default_objective__'] = {'Value': self._macro["OBJECTIVE"]} + + if proc == "OPTLP": + # Convert primal out data set to variable dictionary + # Use panda functions for efficiency + primal_out = primal_out[['_VAR_', '_VALUE_', '_STATUS_', '_R_COST_']] + primal_out = primal_out.set_index('_VAR_', drop=True) + primal_out = primal_out.rename( + {'_VALUE_': 'Value', '_STATUS_': 'Status', '_R_COST_': 'rc'}, + axis='columns', + ) + sol.variable = primal_out.to_dict('index') + + # Convert dual out data set to constraint dictionary + # Use pandas functions for efficiency + dual_out = dual_out[['_ROW_', '_VALUE_', '_STATUS_', '_ACTIVITY_']] + dual_out = dual_out.set_index('_ROW_', drop=True) + dual_out = dual_out.rename( + {'_VALUE_': 'dual', '_STATUS_': 'Status', '_ACTIVITY_': 'slack'}, + axis='columns', + ) + sol.constraint = dual_out.to_dict('index') + else: + # Convert primal out data set to variable dictionary + # Use pandas functions for efficiency + primal_out = primal_out[['_VAR_', '_VALUE_']] + primal_out = primal_out.set_index('_VAR_', drop=True) + primal_out = primal_out.rename({'_VALUE_': 'Value'}, axis='columns') + sol.variable = primal_out.to_dict('index') + + self._rc = 0 + return Bunch(rc=self._rc, log=self._log) + + +class SASLogWriter: + """Helper class to take the log from stdout and put it also in a StringIO.""" + + def __init__(self, tee): + """Set up the two outputs.""" + self.tee = tee + self._log = StringIO() + self.stdout = sys.stdout + + def write(self, message): + """If the tee options is specified, write to both outputs.""" + if self.tee: + self.stdout.write(message) + self._log.write(message) + + def flush(self): + """Nothing to do, just here for compatibility reasons.""" + # Do nothing since we flush right away + pass + + def log(self): + """ "Get the log as a string.""" + return self._log.getvalue() + + +@SolverFactory.register('_sascas', doc='SAS Viya CAS Server interface') +class SASCAS(SASAbc): + """ + Solver interface connection to a SAS Viya CAS server using swat. + See the documentation for the swat package about how to create a connection. + The swat connection options can be passed as options to the solve function. + """ + + def __init__(self, **kwds): + """Initialize and try to load the swat package.""" + super(SASCAS, self).__init__(**kwds) + + try: + import swat + + self._sas = swat + except ImportError: + self._python_api_exists = False + except Exception as e: + self._python_api_exists = False + # For other exceptions, raise it so that it does not get lost + raise e + else: + self._python_api_exists = True + + def _apply_solver(self): + """ "Prepare the options and run the solver. Then store the data to be returned.""" + logger.debug("Running SAS Viya") + + # Set return code to issue an error if we get interrupted + self._rc = -1 + + # Extract CAS connection options + cas_opts = {} + for opt in CAS_OPTION_NAMES: + val = self.options.pop(opt, None) + if val != None: + cas_opts[opt] = val + + # Figure out if the problem has integer variables + with_opt = self.options.pop("with", None) + if with_opt == "lp": + action = "solveLp" + elif with_opt == "milp": + action = "solveMilp" + else: + # Check if there are integer variables, this might be slow + action = "solveMilp" if self._has_integer_variables() else "solveLp" + + # Connect to CAS server + with redirect_stdout(SASLogWriter(self._tee)) as self._log_writer: + s = self._sas.CAS(**cas_opts) + try: + # Load the optimization action set + s.loadactionset('optimization') + + # Upload mps file to CAS + if os.stat(self._problem_files[0]).st_size >= 2 * 1024**3: + # For large files, use convertMPS, first create file for upload + mpsWithIdFileName = TempfileManager.create_tempfile( + ".mps.csv", text=True + ) + with open(mpsWithIdFileName, 'w') as mpsWithId: + mpsWithId.write('_ID_\tText\n') + with open(self._problem_files[0], 'r') as f: + id = 0 + for line in f: + id += 1 + mpsWithId.write(str(id) + '\t' + line.rstrip() + '\n') + + # Upload .mps.csv file + s.upload_file( + mpsWithIdFileName, + casout={"name": "mpscsv", "replace": True}, + importoptions={"filetype": "CSV", "delimiter": "\t"}, + ) + + # Convert .mps.csv file to .mps + s.optimization.convertMps( + data="mpscsv", + casOut={"name": "mpsdata", "replace": True}, + format="FREE", + ) + else: + # For small files, use loadMPS + with open(self._problem_files[0], 'r') as mps_file: + s.optimization.loadMps( + mpsFileString=mps_file.read(), + casout={"name": "mpsdata", "replace": True}, + format="FREE", + ) + + if self.warmstart_flag: + # Upload warmstart file to CAS + s.upload_file( + self._warm_start_file_name, + casout={"name": "primalin", "replace": True}, + importoptions={"filetype": "CSV"}, + ) + self.options["primalin"] = "primalin" + + # Solve the problem in CAS + if action == "solveMilp": + r = s.optimization.solveMilp( + data={"name": "mpsdata"}, + primalOut={"name": "primalout", "replace": True}, + **self.options + ) + else: + r = s.optimization.solveLp( + data={"name": "mpsdata"}, + primalOut={"name": "primalout", "replace": True}, + dualOut={"name": "dualout", "replace": True}, + **self.options + ) + + # Prepare the solver results + if r: + # Get back the primal and dual solution data sets + results = self.results = self._create_results_from_status( + r.get("status", "ERROR"), r.get("solutionStatus", "ERROR") + ) + + if r.ProblemSummary["cValue1"][1] == "Maximization": + results.problem.sense = ProblemSense.maximize + else: + results.problem.sense = ProblemSense.minimize + + # Prepare the solution information + if ( + results.solver.termination_condition + == TerminationCondition.optimal + ): + sol = results.solution.add() + + # Store status in solution + sol.status = SolutionStatus.feasible + sol.termination_condition = TerminationCondition.optimal + + # Store objective value in solution + sol.objective['__default_objective__'] = { + 'Value': r["objective"] + } + + if action == "solveMilp": + primal_out = s.CASTable(name="primalout") + # Use pandas functions for efficiency + primal_out = primal_out[['_VAR_', '_VALUE_']] + sol.variable = {} + for row in primal_out.itertuples(index=False): + sol.variable[row[0]] = {'Value': row[1]} + else: + # Convert primal out data set to variable dictionary + # Use panda functions for efficiency + primal_out = s.CASTable(name="primalout") + primal_out = primal_out[ + ['_VAR_', '_VALUE_', '_STATUS_', '_R_COST_'] + ] + sol.variable = {} + for row in primal_out.itertuples(index=False): + sol.variable[row[0]] = { + 'Value': row[1], + 'Status': row[2], + 'rc': row[3], + } + + # Convert dual out data set to constraint dictionary + # Use pandas functions for efficiency + dual_out = s.CASTable(name="dualout") + dual_out = dual_out[ + ['_ROW_', '_VALUE_', '_STATUS_', '_ACTIVITY_'] + ] + sol.constraint = {} + for row in dual_out.itertuples(index=False): + sol.constraint[row[0]] = { + 'dual': row[1], + 'Status': row[2], + 'slack': row[3], + } + else: + results = self.results = SolverResults() + results.solver.name = "SAS" + results.solver.status = SolverStatus.error + raise ValueError( + "An option passed to the SAS solver caused a syntax error." + ) + + finally: + s.close() + + self._log = self._log_writer.log() + if self._tee: + print(self._log) + self._rc = 0 + return Bunch(rc=self._rc, log=self._log) diff --git a/pyomo/solvers/plugins/solvers/__init__.py b/pyomo/solvers/plugins/solvers/__init__.py index c5fbfa97e42..23b7fe06526 100644 --- a/pyomo/solvers/plugins/solvers/__init__.py +++ b/pyomo/solvers/plugins/solvers/__init__.py @@ -30,3 +30,4 @@ import pyomo.solvers.plugins.solvers.mosek_persistent import pyomo.solvers.plugins.solvers.xpress_direct import pyomo.solvers.plugins.solvers.xpress_persistent +import pyomo.solvers.plugins.solvers.SAS diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py new file mode 100644 index 00000000000..4592343b17f --- /dev/null +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -0,0 +1,462 @@ +import os +import pyomo.common.unittest as unittest +from pyomo.environ import ( + ConcreteModel, + Var, + Objective, + Constraint, + NonNegativeIntegers, + NonNegativeReals, + Reals, + Integers, + maximize, + minimize, + Suffix, +) +from pyomo.opt.results import ( + SolverStatus, + TerminationCondition, + ProblemSense, +) +from pyomo.opt import ( + SolverFactory, + check_available_solvers, +) + + +CAS_OPTIONS = { + "hostname": os.environ.get('CAS_SERVER', None), + "port": os.environ.get('CAS_PORT', None), + "authinfo": os.environ.get('CAS_AUTHINFO', None), +} + + +sas_available = check_available_solvers('sas') + + +class SASTestAbc: + solver_io = '_sas94' + base_options = {} + + def setObj(self): + X = self.instance.X + self.instance.Obj = Objective( + expr=2 * X[1] - 3 * X[2] - 4 * X[3], sense=minimize + ) + + def setX(self): + self.instance.X = Var([1, 2, 3], within=NonNegativeReals) + + def setUp(self): + instance = self.instance = ConcreteModel() + self.setX() + X = instance.X + instance.R1 = Constraint(expr=-2 * X[2] - 3 * X[3] >= -5) + instance.R2 = Constraint(expr=X[1] + X[2] + 2 * X[3] <= 4) + instance.R3 = Constraint(expr=X[1] + 2 * X[2] + 3 * X[3] <= 7) + self.setObj() + + # Declare suffixes for solution information + instance.status = Suffix(direction=Suffix.IMPORT) + instance.slack = Suffix(direction=Suffix.IMPORT) + instance.rc = Suffix(direction=Suffix.IMPORT) + instance.dual = Suffix(direction=Suffix.IMPORT) + + self.opt_sas = SolverFactory('sas', solver_io=self.solver_io) + + def tearDown(self): + del self.opt_sas + del self.instance + + def run_solver(self, **kwargs): + opt_sas = self.opt_sas + instance = self.instance + + # Add base options for connection data etc. + options = kwargs.get("options", {}) + if self.base_options: + kwargs["options"] = {**options, **self.base_options} + + # Call the solver + self.results = opt_sas.solve(instance, **kwargs) + + +class SASTestLP(SASTestAbc, unittest.TestCase): + def checkSolution(self): + instance = self.instance + results = self.results + # Get the objective sense, we use the same code for minimization and maximization tests + sense = instance.Obj.sense + + # Check status + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) + + # Check objective value + self.assertAlmostEqual(instance.Obj(), sense * -7.5) + + # Check primal solution values + self.assertAlmostEqual(instance.X[1].value, 0.0) + self.assertAlmostEqual(instance.X[2].value, 2.5) + self.assertAlmostEqual(instance.X[3].value, 0.0) + + # Check reduced cost + self.assertAlmostEqual(instance.rc[instance.X[1]], sense * 2.0) + self.assertAlmostEqual(instance.rc[instance.X[2]], sense * 0.0) + self.assertAlmostEqual(instance.rc[instance.X[3]], sense * 0.5) + + # Check slack + self.assertAlmostEqual(instance.slack[instance.R1], -5.0) + self.assertAlmostEqual(instance.slack[instance.R2], 2.5) + self.assertAlmostEqual(instance.slack[instance.R3], 5.0) + + # Check dual solution + self.assertAlmostEqual(instance.dual[instance.R1], sense * 1.5) + self.assertAlmostEqual(instance.dual[instance.R2], sense * 0.0) + self.assertAlmostEqual(instance.dual[instance.R3], sense * 0.0) + + # Check basis status + self.assertEqual(instance.status[instance.X[1]], 'L') + self.assertEqual(instance.status[instance.X[2]], 'B') + self.assertEqual(instance.status[instance.X[3]], 'L') + self.assertEqual(instance.status[instance.R1], 'U') + self.assertEqual(instance.status[instance.R2], 'B') + self.assertEqual(instance.status[instance.R3], 'B') + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_default(self): + self.run_solver() + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_primal(self): + self.run_solver(options={"algorithm": "ps"}) + self.assertIn("NOTE: The Primal Simplex algorithm is used.", self.opt_sas._log) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_ipm(self): + self.run_solver(options={"algorithm": "ip"}) + self.assertIn("NOTE: The Interior Point algorithm is used.", self.opt_sas._log) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_intoption(self): + self.run_solver(options={"maxiter": 20}) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_invalidoption(self): + with self.assertRaisesRegex(ValueError, "syntax error"): + self.run_solver(options={"foo": "bar"}) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_max(self): + X = self.instance.X + self.instance.Obj.set_value(expr=-2 * X[1] + 3 * X[2] + 4 * X[3]) + self.instance.Obj.sense = maximize + self.run_solver() + self.checkSolution() + self.assertEqual(self.results.problem.sense, ProblemSense.maximize) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_infeasible(self): + instance = self.instance + X = instance.X + instance.R4 = Constraint(expr=-2 * X[2] - 3 * X[3] <= -6) + self.run_solver() + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.infeasible + ) + self.assertEqual(results.solver.message, "The problem is infeasible.") + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_infeasible_or_unbounded(self): + self.instance.X.domain = Reals + self.run_solver() + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, + TerminationCondition.infeasibleOrUnbounded, + ) + self.assertEqual( + results.solver.message, "The problem is infeasible or unbounded." + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_unbounded(self): + self.instance.X.domain = Reals + self.run_solver(options={"presolver": "none", "algorithm": "primal"}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.unbounded + ) + self.assertEqual(results.solver.message, "The problem is unbounded.") + + def checkSolutionDecomp(self): + instance = self.instance + results = self.results + # Get the objective sense, we use the same code for minimization and maximization tests + sense = instance.Obj.sense + + # Check status + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) + + # Check objective value + self.assertAlmostEqual(instance.Obj(), sense * -7.5) + + # Check primal solution values + self.assertAlmostEqual(instance.X[1].value, 0.0) + self.assertAlmostEqual(instance.X[2].value, 2.5) + self.assertAlmostEqual(instance.X[3].value, 0.0) + + # Check reduced cost + self.assertAlmostEqual(instance.rc[instance.X[1]], sense * 2.0) + self.assertAlmostEqual(instance.rc[instance.X[2]], sense * 0.0) + self.assertAlmostEqual(instance.rc[instance.X[3]], sense * 0.5) + + # Check slack + self.assertAlmostEqual(instance.slack[instance.R1], -5.0) + self.assertAlmostEqual(instance.slack[instance.R2], 2.5) + self.assertAlmostEqual(instance.slack[instance.R3], 5.0) + + # Check dual solution + self.assertAlmostEqual(instance.dual[instance.R1], sense * 1.5) + self.assertAlmostEqual(instance.dual[instance.R2], sense * 0.0) + self.assertAlmostEqual(instance.dual[instance.R3], sense * 0.0) + + # Don't check basis status for decomp + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_decomp(self): + self.run_solver( + options={ + "decomp": {"absobjgap": 0.0}, + "decompmaster": {"algorithm": "dual"}, + "decompsubprob": {"presolver": "none"}, + } + ) + self.assertIn( + "NOTE: The DECOMP method value DEFAULT is applied.", self.opt_sas._log + ) + self.checkSolutionDecomp() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_iis(self): + self.run_solver(options={"iis": "true"}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.feasible + ) + self.assertIn("NOTE: The IIS= option is enabled.", self.opt_sas._log) + self.assertEqual( + results.solver.message, + "The problem is feasible. This status is displayed when the IIS=TRUE option is specified and the problem is feasible.", + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_maxiter(self): + self.run_solver(options={"maxiter": 1}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.maxIterations + ) + self.assertEqual( + results.solver.message, + "The maximum allowable number of iterations was reached.", + ) + + +class SASTestLPCAS(SASTestLP): + solver_io = '_sascas' + base_options = CAS_OPTIONS + + +class SASTestMILP(SASTestAbc, unittest.TestCase): + def setX(self): + self.instance.X = Var([1, 2, 3], within=NonNegativeIntegers) + + def checkSolution(self): + instance = self.instance + results = self.results + + # Get the objective sense, we use the same code for minimization and maximization tests + sense = instance.Obj.sense + + # Check status + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) + + # Check objective value + self.assertAlmostEqual(instance.Obj(), sense * -7) + + # Check primal solution values + self.assertAlmostEqual(instance.X[1].value, 0.0) + self.assertAlmostEqual(instance.X[2].value, 1.0) + self.assertAlmostEqual(instance.X[3].value, 1.0) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_default(self): + self.run_solver(options={}) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_presolve(self): + self.run_solver(options={"presolver": "none"}) + self.assertIn( + "NOTE: The MILP presolver value NONE is applied.", self.opt_sas._log + ) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_intoption(self): + self.run_solver(options={"maxnodes": 20}) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_invalidoption(self): + with self.assertRaisesRegex(ValueError, "syntax error"): + self.run_solver(options={"foo": "bar"}) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_max(self): + X = self.instance.X + self.instance.Obj.set_value(expr=-2 * X[1] + 3 * X[2] + 4 * X[3]) + self.instance.Obj.sense = maximize + self.run_solver() + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_infeasible(self): + instance = self.instance + X = instance.X + instance.R4 = Constraint(expr=-2 * X[2] - 3 * X[3] <= -6) + self.run_solver() + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.infeasible + ) + self.assertEqual(results.solver.message, "The problem is infeasible.") + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + @unittest.skip("Returns wrong status for some versions.") + def test_solver_infeasible_or_unbounded(self): + self.instance.X.domain = Integers + self.run_solver() + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, + TerminationCondition.infeasibleOrUnbounded, + ) + self.assertEqual( + results.solver.message, "The problem is infeasible or unbounded." + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_unbounded(self): + self.instance.X.domain = Integers + self.run_solver( + options={"presolver": "none", "rootnode": {"algorithm": "primal"}} + ) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.warning) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.unbounded + ) + self.assertEqual(results.solver.message, "The problem is unbounded.") + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_decomp(self): + self.run_solver( + options={ + "decomp": {"hybrid": "off"}, + "decompmaster": {"algorithm": "dual"}, + "decompmasterip": {"presolver": "none"}, + "decompsubprob": {"presolver": "none"}, + } + ) + self.assertIn( + "NOTE: The DECOMP method value DEFAULT is applied.", self.opt_sas._log + ) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_rootnode(self): + self.run_solver(options={"rootnode": {"presolver": "automatic"}}) + self.checkSolution() + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_maxnodes(self): + self.run_solver(options={"maxnodes": 0}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.maxEvaluations + ) + self.assertEqual( + results.solver.message, + "The solver reached the maximum number of nodes specified by the MAXNODES= option and found a solution.", + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_maxsols(self): + self.run_solver(options={"maxsols": 1}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.maxEvaluations + ) + self.assertEqual( + results.solver.message, + "The solver reached the maximum number of solutions specified by the MAXSOLS= option.", + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_target(self): + self.run_solver(options={"target": -6.0}) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.ok) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) + self.assertEqual( + results.solver.message, + "The solution is not worse than the target specified by the TARGET= option.", + ) + + @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_primalin(self): + X = self.instance.X + X[1] = None + X[2] = 3 + X[3] = 7 + self.run_solver(warmstart=True) + self.checkSolution() + self.assertIn( + "NOTE: The input solution is infeasible or incomplete. Repair heuristics are applied.", + self.opt_sas._log, + ) + + +class SASTestMILPCAS(SASTestMILP): + solver_io = '_sascas' + base_options = CAS_OPTIONS + + +if __name__ == '__main__': + unittest.main() From bc0e1feb794f553d0e7a859120aeeabd297708e9 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Fri, 23 Jun 2023 09:56:54 -0400 Subject: [PATCH 04/20] Just some black adjustments --- pyomo/solvers/plugins/solvers/SAS.py | 3 ++- pyomo/solvers/tests/checks/test_SAS.py | 11 ++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index 7f50b7a2970..ed0e63d44d6 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -427,7 +427,8 @@ def _apply_solver(self): # Prepare the solver results results = self.results = self._create_results_from_status( - self._macro.get("STATUS", "ERROR"), self._macro.get("SOLUTION_STATUS","ERROR") + self._macro.get("STATUS", "ERROR"), + self._macro.get("SOLUTION_STATUS", "ERROR"), ) if "Objective Sense Maximization" in self._lst: diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 4592343b17f..654820f5060 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -13,15 +13,8 @@ minimize, Suffix, ) -from pyomo.opt.results import ( - SolverStatus, - TerminationCondition, - ProblemSense, -) -from pyomo.opt import ( - SolverFactory, - check_available_solvers, -) +from pyomo.opt.results import SolverStatus, TerminationCondition, ProblemSense +from pyomo.opt import SolverFactory, check_available_solvers CAS_OPTIONS = { From cda764d85ff7b37d72f8206c599f1592a0b477de Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Mon, 7 Aug 2023 09:44:00 -0400 Subject: [PATCH 05/20] Updated version for remote testing --- .github/workflows/typos.toml | 2 + .gitignore | 5 +- pyomo/solvers/plugins/solvers/SAS.py | 257 +++++++++++++++++-------- pyomo/solvers/tests/checks/test_SAS.py | 183 ++++++++++++------ 4 files changed, 307 insertions(+), 140 deletions(-) diff --git a/.github/workflows/typos.toml b/.github/workflows/typos.toml index c9fe9e804a2..71d9ad0355f 100644 --- a/.github/workflows/typos.toml +++ b/.github/workflows/typos.toml @@ -38,4 +38,6 @@ caf = "caf" WRONLY = "WRONLY" # Ignore the name Hax Hax = "Hax" +# Ignore dout (short for dual output in SAS solvers) +dout = "dout" # AS NEEDED: Add More Words Below diff --git a/.gitignore b/.gitignore index 09069552990..7309ff1e8a8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ .spyder* .ropeproject .vscode +.env +venv +sascfg_personal.py # Python generates numerous files when byte compiling / installing packages *.pyx *.pyc @@ -24,4 +27,4 @@ gurobi.log # Jupyterhub/Jupyterlab checkpoints .ipynb_checkpoints -cplex.log \ No newline at end of file +cplex.log diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index ed0e63d44d6..87a6a18c1f4 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -1,8 +1,9 @@ -__all__ = ['SAS'] +__all__ = ["SAS"] import logging import sys -import os +from os import stat +import uuid from io import StringIO from abc import ABC, abstractmethod @@ -24,7 +25,7 @@ from pyomo.core.kernel.block import IBlock -logger = logging.getLogger('pyomo.solvers') +logger = logging.getLogger("pyomo.solvers") STATUS_TO_SOLVERSTATUS = { @@ -112,20 +113,20 @@ ] -@SolverFactory.register('sas', doc='The SAS LP/MIP solver') +@SolverFactory.register("sas", doc="The SAS LP/MIP solver") class SAS(OptSolver): """The SAS optimization solver""" def __new__(cls, *args, **kwds): - mode = kwds.pop('solver_io', None) + mode = kwds.pop("solver_io", None) if mode != None: return SolverFactory(mode) else: # Choose solver factory automatically - # bassed on what can be loaded. - s = SolverFactory('_sas94', **kwds) + # based on what can be loaded. + s = SolverFactory("_sas94", **kwds) if not s.available(): - s = SolverFactory('_sascas', **kwds) + s = SolverFactory("_sascas", **kwds) return s @@ -134,7 +135,7 @@ class SASAbc(ABC, OptSolver): def __init__(self, **kwds): """Initialize the SAS solver interfaces.""" - kwds['type'] = 'sas' + kwds["type"] = "sas" super(SASAbc, self).__init__(**kwds) # @@ -157,7 +158,7 @@ def _presolve(self, *args, **kwds): TempfileManager.push() # Get the warmstart flag - self.warmstart_flag = kwds.pop('warmstart', False) + self.warmstart_flag = kwds.pop("warmstart", False) # Call parent presolve function super(SASAbc, self)._presolve(*args, **kwds) @@ -173,7 +174,7 @@ def _presolve(self, *args, **kwds): Var, active=True, descend_into=False ): self._vars.append(vardata) - # Store the symbal map, we need this for example when writing the warmstart file + # Store the symbol map, we need this for example when writing the warmstart file if isinstance(self._instance, IBlock): self._smap = getattr(self._instance, "._symbol_maps")[self._smap_id] else: @@ -186,8 +187,8 @@ def _presolve(self, *args, **kwds): ) smap = self._smap numWritten = 0 - with open(filename, 'w') as file: - file.write('_VAR_,_VALUE_\n') + with open(filename, "w") as file: + file.write("_VAR_,_VALUE_\n") for var in self._vars: if (var.value is not None) and (id(var) in smap.byObject): name = smap.byObject[id(var)] @@ -239,8 +240,7 @@ def _create_results_from_status(self, status, solution_status): @abstractmethod def _apply_solver(self): - """The routine that performs the solve""" - raise NotImplemented("This is an abstract function and thus not implemented!") + pass def _postsolve(self): """Clean up at the end, especially the temp files.""" @@ -260,7 +260,7 @@ def warm_start_capable(self): return True -@SolverFactory.register('_sas94', doc='SAS 9.4 interface') +@SolverFactory.register("_sas94", doc="SAS 9.4 interface") class SAS94(SASAbc): """ Solver interface for SAS 9.4 using saspy. See the saspy documentation about @@ -298,6 +298,9 @@ def _create_statement_str(self, statement): else: return "" + def sas_version(self): + return self._sasver + def _apply_solver(self): """ "Prepare the options and run the solver. Then store the data to be returned.""" logger.debug("Running SAS") @@ -326,39 +329,81 @@ def _apply_solver(self): decompsubprob_str = self._create_statement_str("decompsubprob") rootnode_str = self._create_statement_str("rootnode") + # Get a unique identifier, always use the same with different prefixes + unique = uuid.uuid4().hex[:16] + + # Create unique filename for output datasets + primalout_dataset_name = "pout" + unique + dualout_dataset_name = "dout" + unique + primalin_dataset_name = None + # Handle warmstart warmstart_str = "" if self.warmstart_flag: # Set the warmstart basis option + primalin_dataset_name = "pin" + unique if proc != "OPTLP": warmstart_str = """ proc import datafile='{primalin}' - out=primalin + out={primalin_dataset_name} dbms=csv replace; getnames=yes; run; """.format( - primalin=self._warm_start_file_name + primalin=self._warm_start_file_name, + primalin_dataset_name=primalin_dataset_name, ) - self.options["primalin"] = "primalin" + self.options["primalin"] = primalin_dataset_name # Convert options to string opt_str = " ".join( option + "=" + str(value) for option, value in self.options.items() ) - # Start a SAS session, submit the code and return the results`` + # Set some SAS options to make the log more clean + sas_options = "option notes nonumber nodate nosource pagesize=max;" + + # Start a SAS session, submit the code and return the results with self._sas.SASsession() as sas: # Find the version of 9.4 we are using - if sas.sasver.startswith("9.04.01M5"): + self._sasver = sas.sasver + + # Upload files, only if not accessible locally + upload_mps = False + if not sas.file_info(self._problem_files[0], quiet=True): + sas.upload( + self._problem_files[0], self._problem_files[0], overwrite=True + ) + upload_mps = True + + upload_pin = False + if self.warmstart_flag and not sas.file_info( + self._warm_start_file_name, quiet=True + ): + sas.upload( + self._warm_start_file_name, + self._warm_start_file_name, + overwrite=True, + ) + upload_pin = True + + # Using a function call to make it easier to moch the version check + version = self.sas_version().split("M", 1)[1][0] + if int(version) < 5: + raise NotImplementedError( + "Support for SAS 9.4 M4 and earlier is no implemented." + ) + elif int(version) == 5: # In 9.4M5 we have to create an MPS data set from an MPS file first # Earlier versions will not work because the MPS format in incompatible + mps_dataset_name = "mps" + unique res = sas.submit( """ + {sas_options} {warmstart} - %MPS2SASD(MPSFILE="{mpsfile}", OUTDATA=mpsdata, MAXLEN=256, FORMAT=FREE); - proc {proc} data=mpsdata {options} primalout=primalout dualout=dualout; + %MPS2SASD(MPSFILE="{mpsfile}", OUTDATA={mps_dataset_name}, MAXLEN=256, FORMAT=FREE); + proc {proc} data={mps_dataset_name} {options} primalout={primalout_dataset_name} dualout={dualout_dataset_name}; {decomp} {decompmaster} {decompmasterip} @@ -366,10 +411,14 @@ def _apply_solver(self): {rootnode} run; """.format( + sas_options=sas_options, warmstart=warmstart_str, proc=proc, mpsfile=self._problem_files[0], + mps_dataset_name=mps_dataset_name, options=opt_str, + primalout_dataset_name=primalout_dataset_name, + dualout_dataset_name=dualout_dataset_name, decomp=decomp_str, decompmaster=decompmaster_str, decompmasterip=decompmasterip_str, @@ -378,12 +427,14 @@ def _apply_solver(self): ), results="TEXT", ) + sas.sasdata(mps_dataset_name).delete(quiet=True) else: # Since 9.4M6+ optlp/optmilp can read mps files directly res = sas.submit( """ + {sas_options} {warmstart} - proc {proc} mpsfile=\"{mpsfile}\" {options} primalout=primalout dualout=dualout; + proc {proc} mpsfile=\"{mpsfile}\" {options} primalout={primalout_dataset_name} dualout={dualout_dataset_name}; {decomp} {decompmaster} {decompmasterip} @@ -391,10 +442,13 @@ def _apply_solver(self): {rootnode} run; """.format( + sas_options=sas_options, warmstart=warmstart_str, proc=proc, mpsfile=self._problem_files[0], options=opt_str, + primalout_dataset_name=primalout_dataset_name, + dualout_dataset_name=dualout_dataset_name, decomp=decomp_str, decompmaster=decompmaster_str, decompmasterip=decompmasterip_str, @@ -404,26 +458,40 @@ def _apply_solver(self): results="TEXT", ) + # Delete uploaded file + if upload_mps: + sas.file_delete(self._problem_files[0], quiet=True) + if self.warmstart_flag and upload_pin: + sas.file_delete(self._warm_start_file_name, quiet=True) + # Store log and ODS output self._log = res["LOG"] self._lst = res["LST"] - # Print log if requested by the user - if self._tee: - print(self._log) if "ERROR 22-322: Syntax error" in self._log: raise ValueError( "An option passed to the SAS solver caused a syntax error: {log}".format( log=self._log ) ) + else: + # Print log if requested by the user, only if we did not already print it + if self._tee: + print(self._log) self._macro = dict( (key.strip(), value.strip()) for key, value in ( pair.split("=") for pair in sas.symget("_OR" + proc + "_").split() ) ) - primal_out = sas.sd2df("primalout") - dual_out = sas.sd2df("dualout") + if self._macro.get("STATUS", "ERROR") == "OK": + primal_out = sas.sd2df(primalout_dataset_name) + dual_out = sas.sd2df(dualout_dataset_name) + + # Delete data sets, they will go away automatically, but does not hurt to delete them + if primalin_dataset_name: + sas.sasdata(primalin_dataset_name).delete(quiet=True) + sas.sasdata(primalout_dataset_name).delete(quiet=True) + sas.sasdata(dualout_dataset_name).delete(quiet=True) # Prepare the solver results results = self.results = self._create_results_from_status( @@ -445,35 +513,35 @@ def _apply_solver(self): sol.termination_condition = TerminationCondition.optimal # Store objective value in solution - sol.objective['__default_objective__'] = {'Value': self._macro["OBJECTIVE"]} + sol.objective["__default_objective__"] = {"Value": self._macro["OBJECTIVE"]} if proc == "OPTLP": # Convert primal out data set to variable dictionary # Use panda functions for efficiency - primal_out = primal_out[['_VAR_', '_VALUE_', '_STATUS_', '_R_COST_']] - primal_out = primal_out.set_index('_VAR_', drop=True) + primal_out = primal_out[["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"]] + primal_out = primal_out.set_index("_VAR_", drop=True) primal_out = primal_out.rename( - {'_VALUE_': 'Value', '_STATUS_': 'Status', '_R_COST_': 'rc'}, - axis='columns', + {"_VALUE_": "Value", "_STATUS_": "Status", "_R_COST_": "rc"}, + axis="columns", ) - sol.variable = primal_out.to_dict('index') + sol.variable = primal_out.to_dict("index") # Convert dual out data set to constraint dictionary # Use pandas functions for efficiency - dual_out = dual_out[['_ROW_', '_VALUE_', '_STATUS_', '_ACTIVITY_']] - dual_out = dual_out.set_index('_ROW_', drop=True) + dual_out = dual_out[["_ROW_", "_VALUE_", "_STATUS_", "_ACTIVITY_"]] + dual_out = dual_out.set_index("_ROW_", drop=True) dual_out = dual_out.rename( - {'_VALUE_': 'dual', '_STATUS_': 'Status', '_ACTIVITY_': 'slack'}, - axis='columns', + {"_VALUE_": "dual", "_STATUS_": "Status", "_ACTIVITY_": "slack"}, + axis="columns", ) - sol.constraint = dual_out.to_dict('index') + sol.constraint = dual_out.to_dict("index") else: # Convert primal out data set to variable dictionary # Use pandas functions for efficiency - primal_out = primal_out[['_VAR_', '_VALUE_']] - primal_out = primal_out.set_index('_VAR_', drop=True) - primal_out = primal_out.rename({'_VALUE_': 'Value'}, axis='columns') - sol.variable = primal_out.to_dict('index') + primal_out = primal_out[["_VAR_", "_VALUE_"]] + primal_out = primal_out.set_index("_VAR_", drop=True) + primal_out = primal_out.rename({"_VALUE_": "Value"}, axis="columns") + sol.variable = primal_out.to_dict("index") self._rc = 0 return Bunch(rc=self._rc, log=self._log) @@ -504,7 +572,7 @@ def log(self): return self._log.getvalue() -@SolverFactory.register('_sascas', doc='SAS Viya CAS Server interface') +@SolverFactory.register("_sascas", doc="SAS Viya CAS Server interface") class SASCAS(SASAbc): """ Solver interface connection to a SAS Viya CAS server using swat. @@ -553,70 +621,89 @@ def _apply_solver(self): # Check if there are integer variables, this might be slow action = "solveMilp" if self._has_integer_variables() else "solveLp" + # Get a unique identifier, always use the same with different prefixes + unique = uuid.uuid4().hex[:16] + # Connect to CAS server with redirect_stdout(SASLogWriter(self._tee)) as self._log_writer: s = self._sas.CAS(**cas_opts) try: # Load the optimization action set - s.loadactionset('optimization') + s.loadactionset("optimization") + + # Declare a unique table name for the mps table + mpsdata_table_name = "mps" + unique # Upload mps file to CAS - if os.stat(self._problem_files[0]).st_size >= 2 * 1024**3: - # For large files, use convertMPS, first create file for upload + if stat(self._problem_files[0]).st_size >= 2 * 1024**3: + # For files larger than 2 GB (this is a limitation of the loadMps action used in the else part). + # Use convertMPS, first create file for upload. mpsWithIdFileName = TempfileManager.create_tempfile( ".mps.csv", text=True ) - with open(mpsWithIdFileName, 'w') as mpsWithId: - mpsWithId.write('_ID_\tText\n') - with open(self._problem_files[0], 'r') as f: + with open(mpsWithIdFileName, "w") as mpsWithId: + mpsWithId.write("_ID_\tText\n") + with open(self._problem_files[0], "r") as f: id = 0 for line in f: id += 1 - mpsWithId.write(str(id) + '\t' + line.rstrip() + '\n') + mpsWithId.write(str(id) + "\t" + line.rstrip() + "\n") # Upload .mps.csv file + mpscsv_table_name = "csv" + unique s.upload_file( mpsWithIdFileName, - casout={"name": "mpscsv", "replace": True}, + casout={"name": mpscsv_table_name, "replace": True}, importoptions={"filetype": "CSV", "delimiter": "\t"}, ) # Convert .mps.csv file to .mps s.optimization.convertMps( - data="mpscsv", - casOut={"name": "mpsdata", "replace": True}, + data=mpscsv_table_name, + casOut={"name": mpsdata_table_name, "replace": True}, format="FREE", ) + + # Delete the table we don't need anymore + if mpscsv_table_name: + s.dropTable(name=mpscsv_table_name, quiet=True) else: - # For small files, use loadMPS - with open(self._problem_files[0], 'r') as mps_file: + # For small files (less than 2 GB), use loadMps + with open(self._problem_files[0], "r") as mps_file: s.optimization.loadMps( mpsFileString=mps_file.read(), - casout={"name": "mpsdata", "replace": True}, + casout={"name": mpsdata_table_name, "replace": True}, format="FREE", ) + primalin_table_name = None if self.warmstart_flag: + primalin_table_name = "pin" + unique # Upload warmstart file to CAS s.upload_file( self._warm_start_file_name, - casout={"name": "primalin", "replace": True}, + casout={"name": primalin_table_name, "replace": True}, importoptions={"filetype": "CSV"}, ) - self.options["primalin"] = "primalin" + self.options["primalin"] = primalin_table_name + + # Define output table names + primalout_table_name = "pout" + unique + dualout_table_name = None # Solve the problem in CAS if action == "solveMilp": r = s.optimization.solveMilp( - data={"name": "mpsdata"}, - primalOut={"name": "primalout", "replace": True}, + data={"name": mpsdata_table_name}, + primalOut={"name": primalout_table_name, "replace": True}, **self.options ) else: + dualout_table_name = "dout" + unique r = s.optimization.solveLp( - data={"name": "mpsdata"}, - primalOut={"name": "primalout", "replace": True}, - dualOut={"name": "dualout", "replace": True}, + data={"name": mpsdata_table_name}, + primalOut={"name": primalout_table_name, "replace": True}, + dualOut={"name": dualout_table_name, "replace": True}, **self.options ) @@ -644,44 +731,44 @@ def _apply_solver(self): sol.termination_condition = TerminationCondition.optimal # Store objective value in solution - sol.objective['__default_objective__'] = { - 'Value': r["objective"] + sol.objective["__default_objective__"] = { + "Value": r["objective"] } if action == "solveMilp": - primal_out = s.CASTable(name="primalout") + primal_out = s.CASTable(name=primalout_table_name) # Use pandas functions for efficiency - primal_out = primal_out[['_VAR_', '_VALUE_']] + primal_out = primal_out[["_VAR_", "_VALUE_"]] sol.variable = {} for row in primal_out.itertuples(index=False): - sol.variable[row[0]] = {'Value': row[1]} + sol.variable[row[0]] = {"Value": row[1]} else: # Convert primal out data set to variable dictionary # Use panda functions for efficiency - primal_out = s.CASTable(name="primalout") + primal_out = s.CASTable(name=primalout_table_name) primal_out = primal_out[ - ['_VAR_', '_VALUE_', '_STATUS_', '_R_COST_'] + ["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"] ] sol.variable = {} for row in primal_out.itertuples(index=False): sol.variable[row[0]] = { - 'Value': row[1], - 'Status': row[2], - 'rc': row[3], + "Value": row[1], + "Status": row[2], + "rc": row[3], } # Convert dual out data set to constraint dictionary # Use pandas functions for efficiency - dual_out = s.CASTable(name="dualout") + dual_out = s.CASTable(name=dualout_table_name) dual_out = dual_out[ - ['_ROW_', '_VALUE_', '_STATUS_', '_ACTIVITY_'] + ["_ROW_", "_VALUE_", "_STATUS_", "_ACTIVITY_"] ] sol.constraint = {} for row in dual_out.itertuples(index=False): sol.constraint[row[0]] = { - 'dual': row[1], - 'Status': row[2], - 'slack': row[3], + "dual": row[1], + "Status": row[2], + "slack": row[3], } else: results = self.results = SolverResults() @@ -692,10 +779,16 @@ def _apply_solver(self): ) finally: + if mpsdata_table_name: + s.dropTable(name=mpsdata_table_name, quiet=True) + if primalin_table_name: + s.dropTable(name=primalin_table_name, quiet=True) + if primalout_table_name: + s.dropTable(name=primalout_table_name, quiet=True) + if dualout_table_name: + s.dropTable(name=dualout_table_name, quiet=True) s.close() self._log = self._log_writer.log() - if self._tee: - print(self._log) self._rc = 0 return Bunch(rc=self._rc, log=self._log) diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 654820f5060..3a63e258600 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -1,5 +1,6 @@ import os import pyomo.common.unittest as unittest +from unittest import mock from pyomo.environ import ( ConcreteModel, Var, @@ -15,20 +16,20 @@ ) from pyomo.opt.results import SolverStatus, TerminationCondition, ProblemSense from pyomo.opt import SolverFactory, check_available_solvers - +import warnings CAS_OPTIONS = { - "hostname": os.environ.get('CAS_SERVER', None), - "port": os.environ.get('CAS_PORT', None), - "authinfo": os.environ.get('CAS_AUTHINFO', None), + "hostname": os.environ.get("CASHOST", None), + "port": os.environ.get("CASPORT", None), + "authinfo": os.environ.get("CASAUTHINFO", None), } -sas_available = check_available_solvers('sas') +sas_available = check_available_solvers("sas") class SASTestAbc: - solver_io = '_sas94' + solver_io = "_sas94" base_options = {} def setObj(self): @@ -41,6 +42,8 @@ def setX(self): self.instance.X = Var([1, 2, 3], within=NonNegativeReals) def setUp(self): + # Disable resource warnings + warnings.filterwarnings("ignore", category=ResourceWarning) instance = self.instance = ConcreteModel() self.setX() X = instance.X @@ -55,7 +58,7 @@ def setUp(self): instance.rc = Suffix(direction=Suffix.IMPORT) instance.dual = Suffix(direction=Suffix.IMPORT) - self.opt_sas = SolverFactory('sas', solver_io=self.solver_io) + self.opt_sas = SolverFactory("sas", solver_io=self.solver_io) def tearDown(self): del self.opt_sas @@ -74,7 +77,7 @@ def run_solver(self, **kwargs): self.results = opt_sas.solve(instance, **kwargs) -class SASTestLP(SASTestAbc, unittest.TestCase): +class SASTestLP(SASTestAbc): def checkSolution(self): instance = self.instance results = self.results @@ -111,41 +114,39 @@ def checkSolution(self): self.assertAlmostEqual(instance.dual[instance.R3], sense * 0.0) # Check basis status - self.assertEqual(instance.status[instance.X[1]], 'L') - self.assertEqual(instance.status[instance.X[2]], 'B') - self.assertEqual(instance.status[instance.X[3]], 'L') - self.assertEqual(instance.status[instance.R1], 'U') - self.assertEqual(instance.status[instance.R2], 'B') - self.assertEqual(instance.status[instance.R3], 'B') - - @unittest.skipIf(not sas_available, "The SAS solver is not available") + self.assertEqual(instance.status[instance.X[1]], "L") + self.assertEqual(instance.status[instance.X[2]], "B") + self.assertEqual(instance.status[instance.X[3]], "L") + self.assertEqual(instance.status[instance.R1], "U") + self.assertEqual(instance.status[instance.R2], "B") + self.assertEqual(instance.status[instance.R3], "B") + def test_solver_default(self): self.run_solver() self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") + def test_solver_tee(self): + self.run_solver(tee=True) + self.checkSolution() + def test_solver_primal(self): self.run_solver(options={"algorithm": "ps"}) self.assertIn("NOTE: The Primal Simplex algorithm is used.", self.opt_sas._log) self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_ipm(self): self.run_solver(options={"algorithm": "ip"}) self.assertIn("NOTE: The Interior Point algorithm is used.", self.opt_sas._log) self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_intoption(self): self.run_solver(options={"maxiter": 20}) self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_invalidoption(self): with self.assertRaisesRegex(ValueError, "syntax error"): self.run_solver(options={"foo": "bar"}) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_max(self): X = self.instance.X self.instance.Obj.set_value(expr=-2 * X[1] + 3 * X[2] + 4 * X[3]) @@ -154,7 +155,6 @@ def test_solver_max(self): self.checkSolution() self.assertEqual(self.results.problem.sense, ProblemSense.maximize) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_infeasible(self): instance = self.instance X = instance.X @@ -167,21 +167,23 @@ def test_solver_infeasible(self): ) self.assertEqual(results.solver.message, "The problem is infeasible.") - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_infeasible_or_unbounded(self): self.instance.X.domain = Reals self.run_solver() results = self.results self.assertEqual(results.solver.status, SolverStatus.warning) - self.assertEqual( + self.assertIn( results.solver.termination_condition, - TerminationCondition.infeasibleOrUnbounded, + [ + TerminationCondition.infeasibleOrUnbounded, + TerminationCondition.unbounded, + ], ) - self.assertEqual( - results.solver.message, "The problem is infeasible or unbounded." + self.assertIn( + results.solver.message, + ["The problem is infeasible or unbounded.", "The problem is unbounded."], ) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_unbounded(self): self.instance.X.domain = Reals self.run_solver(options={"presolver": "none", "algorithm": "primal"}) @@ -229,7 +231,6 @@ def checkSolutionDecomp(self): # Don't check basis status for decomp - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_decomp(self): self.run_solver( options={ @@ -243,7 +244,6 @@ def test_solver_decomp(self): ) self.checkSolutionDecomp() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_iis(self): self.run_solver(options={"iis": "true"}) results = self.results @@ -257,7 +257,6 @@ def test_solver_iis(self): "The problem is feasible. This status is displayed when the IIS=TRUE option is specified and the problem is feasible.", ) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_maxiter(self): self.run_solver(options={"maxiter": 1}) results = self.results @@ -270,13 +269,59 @@ def test_solver_maxiter(self): "The maximum allowable number of iterations was reached.", ) + def test_solver_with_milp(self): + self.run_solver(options={"with": "milp"}) + self.assertIn( + "WARNING: The problem has no integer variables.", self.opt_sas._log + ) + -class SASTestLPCAS(SASTestLP): - solver_io = '_sascas' +@unittest.skipIf(not sas_available, "The SAS solver is not available") +class SASTestLP94(SASTestLP, unittest.TestCase): + @mock.patch( + "pyomo.solvers.plugins.solvers.SAS.SAS94.sas_version", + return_value="2sd45s39M4234232", + ) + def test_solver_versionM4(self, sas): + with self.assertRaises(NotImplementedError): + self.run_solver() + + @mock.patch( + "pyomo.solvers.plugins.solvers.SAS.SAS94.sas_version", + return_value="234897293M5324u98", + ) + def test_solver_versionM5(self, sas): + self.run_solver() + self.checkSolution() + + @mock.patch("saspy.SASsession.submit", return_value={"LOG": "", "LST": ""}) + @mock.patch("saspy.SASsession.symget", return_value="STATUS=OUT_OF_MEMORY") + def test_solver_out_of_memory(self, submit_mock, symget_mocks): + self.run_solver(load_solutions=False) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.aborted) + + @mock.patch("saspy.SASsession.submit", return_value={"LOG": "", "LST": ""}) + @mock.patch("saspy.SASsession.symget", return_value="STATUS=ERROR") + def test_solver_error(self, submit_mock, symget_mock): + self.run_solver(load_solutions=False) + results = self.results + self.assertEqual(results.solver.status, SolverStatus.error) + + +@unittest.skipIf(not sas_available, "The SAS solver is not available") +class SASTestLPCAS(SASTestLP, unittest.TestCase): + solver_io = "_sascas" base_options = CAS_OPTIONS + @mock.patch("pyomo.solvers.plugins.solvers.SAS.stat") + def test_solver_large_file(self, os_stat): + os_stat.return_value.st_size = 3 * 1024**3 + self.run_solver() + self.checkSolution() + -class SASTestMILP(SASTestAbc, unittest.TestCase): +class SASTestMILP(SASTestAbc): def setX(self): self.instance.X = Var([1, 2, 3], within=NonNegativeIntegers) @@ -301,12 +346,14 @@ def checkSolution(self): self.assertAlmostEqual(instance.X[2].value, 1.0) self.assertAlmostEqual(instance.X[3].value, 1.0) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_default(self): - self.run_solver(options={}) + self.run_solver() + self.checkSolution() + + def test_solver_tee(self): + self.run_solver(tee=True) self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_presolve(self): self.run_solver(options={"presolver": "none"}) self.assertIn( @@ -314,17 +361,14 @@ def test_solver_presolve(self): ) self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_intoption(self): self.run_solver(options={"maxnodes": 20}) self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_invalidoption(self): with self.assertRaisesRegex(ValueError, "syntax error"): self.run_solver(options={"foo": "bar"}) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_max(self): X = self.instance.X self.instance.Obj.set_value(expr=-2 * X[1] + 3 * X[2] + 4 * X[3]) @@ -332,7 +376,6 @@ def test_solver_max(self): self.run_solver() self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_infeasible(self): instance = self.instance X = instance.X @@ -345,22 +388,23 @@ def test_solver_infeasible(self): ) self.assertEqual(results.solver.message, "The problem is infeasible.") - @unittest.skipIf(not sas_available, "The SAS solver is not available") - @unittest.skip("Returns wrong status for some versions.") def test_solver_infeasible_or_unbounded(self): self.instance.X.domain = Integers self.run_solver() results = self.results self.assertEqual(results.solver.status, SolverStatus.warning) - self.assertEqual( + self.assertIn( results.solver.termination_condition, - TerminationCondition.infeasibleOrUnbounded, + [ + TerminationCondition.infeasibleOrUnbounded, + TerminationCondition.unbounded, + ], ) - self.assertEqual( - results.solver.message, "The problem is infeasible or unbounded." + self.assertIn( + results.solver.message, + ["The problem is infeasible or unbounded.", "The problem is unbounded."], ) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_unbounded(self): self.instance.X.domain = Integers self.run_solver( @@ -373,7 +417,6 @@ def test_solver_unbounded(self): ) self.assertEqual(results.solver.message, "The problem is unbounded.") - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_decomp(self): self.run_solver( options={ @@ -388,12 +431,10 @@ def test_solver_decomp(self): ) self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_rootnode(self): self.run_solver(options={"rootnode": {"presolver": "automatic"}}) self.checkSolution() - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_maxnodes(self): self.run_solver(options={"maxnodes": 0}) results = self.results @@ -406,7 +447,6 @@ def test_solver_maxnodes(self): "The solver reached the maximum number of nodes specified by the MAXNODES= option and found a solution.", ) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_maxsols(self): self.run_solver(options={"maxsols": 1}) results = self.results @@ -419,7 +459,6 @@ def test_solver_maxsols(self): "The solver reached the maximum number of solutions specified by the MAXSOLS= option.", ) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_target(self): self.run_solver(options={"target": -6.0}) results = self.results @@ -432,7 +471,6 @@ def test_solver_target(self): "The solution is not worse than the target specified by the TARGET= option.", ) - @unittest.skipIf(not sas_available, "The SAS solver is not available") def test_solver_primalin(self): X = self.instance.X X[1] = None @@ -445,11 +483,42 @@ def test_solver_primalin(self): self.opt_sas._log, ) + def test_solver_primalin_nosol(self): + X = self.instance.X + X[1] = None + X[2] = None + X[3] = None + self.run_solver(warmstart=True) + self.checkSolution() + + @mock.patch("pyomo.solvers.plugins.solvers.SAS.stat") + def test_solver_large_file(self, os_stat): + os_stat.return_value.st_size = 3 * 1024**3 + self.run_solver() + self.checkSolution() + + def test_solver_with_lp(self): + self.run_solver(options={"with": "lp"}) + self.assertIn( + "contains integer variables; the linear relaxation will be solved.", + self.opt_sas._log, + ) + + def test_solver_warmstart_capable(self): + self.run_solver() + self.assertTrue(self.opt_sas.warm_start_capable()) + + +@unittest.skipIf(not sas_available, "The SAS solver is not available") +class SASTestMILP94(SASTestMILP, unittest.TestCase): + pass + -class SASTestMILPCAS(SASTestMILP): - solver_io = '_sascas' +@unittest.skipIf(not sas_available, "The SAS solver is not available") +class SASTestMILPCAS(SASTestMILP, unittest.TestCase): + solver_io = "_sascas" base_options = CAS_OPTIONS -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() From 4b6298da177aa6c16a1bf581dc6e0508e9ee6084 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Mon, 7 Aug 2023 11:26:10 -0400 Subject: [PATCH 06/20] Change SAS solver interfaces to keep the SAS connection --- pyomo/solvers/plugins/solvers/SAS.py | 266 +++++++++++++------------ pyomo/solvers/tests/checks/test_SAS.py | 11 +- 2 files changed, 149 insertions(+), 128 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index 87a6a18c1f4..a5f74849813 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -285,6 +285,14 @@ def __init__(self, **kwds): self._python_api_exists = True self._sas.logger.setLevel(logger.level) + # Create the session only as its needed + self._sas_session = None + + def __del__(self): + # Close the session, if we created one + if self._sas_session: + self._sas_session.endsas() + def _create_statement_str(self, statement): """Helper function to create the strings for the statements of the proc OPTLP/OPTMILP code.""" stmt = self.options.pop(statement, None) @@ -364,134 +372,133 @@ def _apply_solver(self): # Set some SAS options to make the log more clean sas_options = "option notes nonumber nodate nosource pagesize=max;" - # Start a SAS session, submit the code and return the results - with self._sas.SASsession() as sas: - # Find the version of 9.4 we are using - self._sasver = sas.sasver - - # Upload files, only if not accessible locally - upload_mps = False - if not sas.file_info(self._problem_files[0], quiet=True): - sas.upload( - self._problem_files[0], self._problem_files[0], overwrite=True - ) - upload_mps = True - - upload_pin = False - if self.warmstart_flag and not sas.file_info( - self._warm_start_file_name, quiet=True - ): - sas.upload( - self._warm_start_file_name, - self._warm_start_file_name, - overwrite=True, - ) - upload_pin = True + # Get the current SAS session, submit the code and return the results + sas = self._sas_session + if sas == None: + sas = self._sas_session = self._sas.SASsession() + + # Find the version of 9.4 we are using + self._sasver = sas.sasver + + # Upload files, only if not accessible locally + upload_mps = False + if not sas.file_info(self._problem_files[0], quiet=True): + sas.upload(self._problem_files[0], self._problem_files[0], overwrite=True) + upload_mps = True + + upload_pin = False + if self.warmstart_flag and not sas.file_info( + self._warm_start_file_name, quiet=True + ): + sas.upload( + self._warm_start_file_name, self._warm_start_file_name, overwrite=True + ) + upload_pin = True - # Using a function call to make it easier to moch the version check - version = self.sas_version().split("M", 1)[1][0] - if int(version) < 5: - raise NotImplementedError( - "Support for SAS 9.4 M4 and earlier is no implemented." - ) - elif int(version) == 5: - # In 9.4M5 we have to create an MPS data set from an MPS file first - # Earlier versions will not work because the MPS format in incompatible - mps_dataset_name = "mps" + unique - res = sas.submit( - """ - {sas_options} - {warmstart} - %MPS2SASD(MPSFILE="{mpsfile}", OUTDATA={mps_dataset_name}, MAXLEN=256, FORMAT=FREE); - proc {proc} data={mps_dataset_name} {options} primalout={primalout_dataset_name} dualout={dualout_dataset_name}; - {decomp} - {decompmaster} - {decompmasterip} - {decompsubprob} - {rootnode} - run; - """.format( - sas_options=sas_options, - warmstart=warmstart_str, - proc=proc, - mpsfile=self._problem_files[0], - mps_dataset_name=mps_dataset_name, - options=opt_str, - primalout_dataset_name=primalout_dataset_name, - dualout_dataset_name=dualout_dataset_name, - decomp=decomp_str, - decompmaster=decompmaster_str, - decompmasterip=decompmasterip_str, - decompsubprob=decompsubprob_str, - rootnode=rootnode_str, - ), - results="TEXT", - ) - sas.sasdata(mps_dataset_name).delete(quiet=True) - else: - # Since 9.4M6+ optlp/optmilp can read mps files directly - res = sas.submit( - """ - {sas_options} - {warmstart} - proc {proc} mpsfile=\"{mpsfile}\" {options} primalout={primalout_dataset_name} dualout={dualout_dataset_name}; - {decomp} - {decompmaster} - {decompmasterip} - {decompsubprob} - {rootnode} - run; - """.format( - sas_options=sas_options, - warmstart=warmstart_str, - proc=proc, - mpsfile=self._problem_files[0], - options=opt_str, - primalout_dataset_name=primalout_dataset_name, - dualout_dataset_name=dualout_dataset_name, - decomp=decomp_str, - decompmaster=decompmaster_str, - decompmasterip=decompmasterip_str, - decompsubprob=decompsubprob_str, - rootnode=rootnode_str, - ), - results="TEXT", - ) + # Using a function call to make it easier to moch the version check + version = self.sas_version().split("M", 1)[1][0] + if int(version) < 5: + raise NotImplementedError( + "Support for SAS 9.4 M4 and earlier is no implemented." + ) + elif int(version) == 5: + # In 9.4M5 we have to create an MPS data set from an MPS file first + # Earlier versions will not work because the MPS format in incompatible + mps_dataset_name = "mps" + unique + res = sas.submit( + """ + {sas_options} + {warmstart} + %MPS2SASD(MPSFILE="{mpsfile}", OUTDATA={mps_dataset_name}, MAXLEN=256, FORMAT=FREE); + proc {proc} data={mps_dataset_name} {options} primalout={primalout_dataset_name} dualout={dualout_dataset_name}; + {decomp} + {decompmaster} + {decompmasterip} + {decompsubprob} + {rootnode} + run; + """.format( + sas_options=sas_options, + warmstart=warmstart_str, + proc=proc, + mpsfile=self._problem_files[0], + mps_dataset_name=mps_dataset_name, + options=opt_str, + primalout_dataset_name=primalout_dataset_name, + dualout_dataset_name=dualout_dataset_name, + decomp=decomp_str, + decompmaster=decompmaster_str, + decompmasterip=decompmasterip_str, + decompsubprob=decompsubprob_str, + rootnode=rootnode_str, + ), + results="TEXT", + ) + sas.sasdata(mps_dataset_name).delete(quiet=True) + else: + # Since 9.4M6+ optlp/optmilp can read mps files directly + res = sas.submit( + """ + {sas_options} + {warmstart} + proc {proc} mpsfile=\"{mpsfile}\" {options} primalout={primalout_dataset_name} dualout={dualout_dataset_name}; + {decomp} + {decompmaster} + {decompmasterip} + {decompsubprob} + {rootnode} + run; + """.format( + sas_options=sas_options, + warmstart=warmstart_str, + proc=proc, + mpsfile=self._problem_files[0], + options=opt_str, + primalout_dataset_name=primalout_dataset_name, + dualout_dataset_name=dualout_dataset_name, + decomp=decomp_str, + decompmaster=decompmaster_str, + decompmasterip=decompmasterip_str, + decompsubprob=decompsubprob_str, + rootnode=rootnode_str, + ), + results="TEXT", + ) - # Delete uploaded file - if upload_mps: - sas.file_delete(self._problem_files[0], quiet=True) - if self.warmstart_flag and upload_pin: - sas.file_delete(self._warm_start_file_name, quiet=True) - - # Store log and ODS output - self._log = res["LOG"] - self._lst = res["LST"] - if "ERROR 22-322: Syntax error" in self._log: - raise ValueError( - "An option passed to the SAS solver caused a syntax error: {log}".format( - log=self._log - ) - ) - else: - # Print log if requested by the user, only if we did not already print it - if self._tee: - print(self._log) - self._macro = dict( - (key.strip(), value.strip()) - for key, value in ( - pair.split("=") for pair in sas.symget("_OR" + proc + "_").split() + # Delete uploaded file + if upload_mps: + sas.file_delete(self._problem_files[0], quiet=True) + if self.warmstart_flag and upload_pin: + sas.file_delete(self._warm_start_file_name, quiet=True) + + # Store log and ODS output + self._log = res["LOG"] + self._lst = res["LST"] + if "ERROR 22-322: Syntax error" in self._log: + raise ValueError( + "An option passed to the SAS solver caused a syntax error: {log}".format( + log=self._log ) ) - if self._macro.get("STATUS", "ERROR") == "OK": - primal_out = sas.sd2df(primalout_dataset_name) - dual_out = sas.sd2df(dualout_dataset_name) + else: + # Print log if requested by the user, only if we did not already print it + if self._tee: + print(self._log) + self._macro = dict( + (key.strip(), value.strip()) + for key, value in ( + pair.split("=") for pair in sas.symget("_OR" + proc + "_").split() + ) + ) + if self._macro.get("STATUS", "ERROR") == "OK": + primal_out = sas.sd2df(primalout_dataset_name) + dual_out = sas.sd2df(dualout_dataset_name) - # Delete data sets, they will go away automatically, but does not hurt to delete them - if primalin_dataset_name: - sas.sasdata(primalin_dataset_name).delete(quiet=True) - sas.sasdata(primalout_dataset_name).delete(quiet=True) - sas.sasdata(dualout_dataset_name).delete(quiet=True) + # Delete data sets, they will go away automatically, but does not hurt to delete them + if primalin_dataset_name: + sas.sasdata(primalin_dataset_name).delete(quiet=True) + sas.sasdata(primalout_dataset_name).delete(quiet=True) + sas.sasdata(dualout_dataset_name).delete(quiet=True) # Prepare the solver results results = self.results = self._create_results_from_status( @@ -597,6 +604,14 @@ def __init__(self, **kwds): else: self._python_api_exists = True + # Create the session only as its needed + self._sas_session = None + + def __del__(self): + # Close the session, if we created one + if self._sas_session: + self._sas_session.close() + def _apply_solver(self): """ "Prepare the options and run the solver. Then store the data to be returned.""" logger.debug("Running SAS Viya") @@ -626,7 +641,9 @@ def _apply_solver(self): # Connect to CAS server with redirect_stdout(SASLogWriter(self._tee)) as self._log_writer: - s = self._sas.CAS(**cas_opts) + s = self._sas_session + if s == None: + s = self._sas_session = self._sas.CAS(**cas_opts) try: # Load the optimization action set s.loadactionset("optimization") @@ -787,7 +804,6 @@ def _apply_solver(self): s.dropTable(name=primalout_table_name, quiet=True) if dualout_table_name: s.dropTable(name=dualout_table_name, quiet=True) - s.close() self._log = self._log_writer.log() self._rc = 0 diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 3a63e258600..1a6bbd80f1d 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -32,6 +32,14 @@ class SASTestAbc: solver_io = "_sas94" base_options = {} + @classmethod + def setUpClass(cls): + cls.opt_sas = SolverFactory("sas", solver_io=cls.solver_io) + + @classmethod + def tearDownClass(cls): + del cls.opt_sas + def setObj(self): X = self.instance.X self.instance.Obj = Objective( @@ -58,10 +66,7 @@ def setUp(self): instance.rc = Suffix(direction=Suffix.IMPORT) instance.dual = Suffix(direction=Suffix.IMPORT) - self.opt_sas = SolverFactory("sas", solver_io=self.solver_io) - def tearDown(self): - del self.opt_sas del self.instance def run_solver(self, **kwargs): From 619edf9805639ef2c95088c90f09d3eb5e8972c4 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Mon, 7 Aug 2023 14:31:58 -0400 Subject: [PATCH 07/20] Fix formatting issue in comment --- pyomo/solvers/plugins/solvers/SAS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index a5f74849813..f5840b5d6f3 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -152,7 +152,7 @@ def __init__(self, **kwds): super(SASAbc, self).set_problem_format(ProblemFormat.mps) def _presolve(self, *args, **kwds): - """ "Set things up for the actual solve.""" + """Set things up for the actual solve.""" # create a context in the temporary file manager for # this plugin - is "pop"ed in the _postsolve method. TempfileManager.push() From faddef1a12afdc8953a14ce78146a3d8ab7cef89 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Fri, 11 Aug 2023 03:23:50 -0400 Subject: [PATCH 08/20] Reset .gitignore to original state. --- .gitignore | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 7309ff1e8a8..09069552990 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,6 @@ .spyder* .ropeproject .vscode -.env -venv -sascfg_personal.py # Python generates numerous files when byte compiling / installing packages *.pyx *.pyc @@ -27,4 +24,4 @@ gurobi.log # Jupyterhub/Jupyterlab checkpoints .ipynb_checkpoints -cplex.log +cplex.log \ No newline at end of file From db18d03e5f4403d014f6fdaaa5a4ca494fee28c5 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Tue, 12 Mar 2024 05:43:11 -0400 Subject: [PATCH 09/20] Add session options, fix non-optimal return codes and version checking --- pyomo/solvers/plugins/solvers/SAS.py | 180 +++++++++++-------------- pyomo/solvers/tests/checks/test_SAS.py | 17 +-- 2 files changed, 87 insertions(+), 110 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index f5840b5d6f3..87ee31a08af 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -96,23 +96,6 @@ } -CAS_OPTION_NAMES = [ - "hostname", - "port", - "username", - "password", - "session", - "locale", - "name", - "nworkers", - "authinfo", - "protocol", - "path", - "ssl_ca_list", - "authcode", -] - - @SolverFactory.register("sas", doc="The SAS LP/MIP solver") class SAS(OptSolver): """The SAS optimization solver""" @@ -120,7 +103,7 @@ class SAS(OptSolver): def __new__(cls, *args, **kwds): mode = kwds.pop("solver_io", None) if mode != None: - return SolverFactory(mode) + return SolverFactory(mode, **kwds) else: # Choose solver factory automatically # based on what can be loaded. @@ -216,6 +199,7 @@ def _create_results_from_status(self, status, solution_status): results = SolverResults() results.solver.name = "SAS" results.solver.status = STATUS_TO_SOLVERSTATUS[status] + results.solver.hasSolution = False if results.solver.status == SolverStatus.ok: results.solver.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[ solution_status @@ -226,11 +210,14 @@ def _create_results_from_status(self, status, solution_status): results.solver.status = TerminationCondition.to_solver_status( results.solver.termination_condition ) + if "OPTIMAL" in solution_status or "_SOL" in solution_status: + results.solver.hasSolution = True elif results.solver.status == SolverStatus.aborted: results.solver.termination_condition = TerminationCondition.userInterrupt - results.solver.message = ( - results.solver.termination_message - ) = SOLSTATUS_TO_MESSAGE["ABORTED"] + if solution_status != "ERROR": + results.solver.message = ( + results.solver.termination_message + ) = SOLSTATUS_TO_MESSAGE[solution_status] else: results.solver.termination_condition = TerminationCondition.error results.solver.message = ( @@ -288,6 +275,9 @@ def __init__(self, **kwds): # Create the session only as its needed self._sas_session = None + # Store other options for the SAS session + self._session_options = kwds + def __del__(self): # Close the session, if we created one if self._sas_session: @@ -326,10 +316,6 @@ def _apply_solver(self): # Check if there are integer variables, this might be slow proc = "OPTMILP" if self._has_integer_variables() else "OPTLP" - # Remove CAS options in case they were specified - for opt in CAS_OPTION_NAMES: - self.options.pop(opt, None) - # Get the rootnode options decomp_str = self._create_statement_str("decomp") decompmaster_str = self._create_statement_str("decompmaster") @@ -373,9 +359,7 @@ def _apply_solver(self): sas_options = "option notes nonumber nodate nosource pagesize=max;" # Get the current SAS session, submit the code and return the results - sas = self._sas_session - if sas == None: - sas = self._sas_session = self._sas.SASsession() + sas = self._sas_session = self._sas.SASsession(**self._session_options) # Find the version of 9.4 we are using self._sasver = sas.sasver @@ -396,12 +380,13 @@ def _apply_solver(self): upload_pin = True # Using a function call to make it easier to moch the version check - version = self.sas_version().split("M", 1)[1][0] - if int(version) < 5: + major_version = self.sas_version()[0] + minor_version = self.sas_version().split("M", 1)[1][0] + if major_version == "9" and int(minor_version) < 5: raise NotImplementedError( "Support for SAS 9.4 M4 and earlier is no implemented." ) - elif int(version) == 5: + elif major_version == "9" and int(minor_version) == 5: # In 9.4M5 we have to create an MPS data set from an MPS file first # Earlier versions will not work because the MPS format in incompatible mps_dataset_name = "mps" + unique @@ -436,7 +421,7 @@ def _apply_solver(self): ) sas.sasdata(mps_dataset_name).delete(quiet=True) else: - # Since 9.4M6+ optlp/optmilp can read mps files directly + # Since 9.4M6+ optlp/optmilp can read mps files directly (this includes Viya-based local installs) res = sas.submit( """ {sas_options} @@ -512,12 +497,12 @@ def _apply_solver(self): results.problem.sense = ProblemSense.minimize # Prepare the solution information - if results.solver.termination_condition == TerminationCondition.optimal: + if results.solver.hasSolution: sol = results.solution.add() # Store status in solution sol.status = SolutionStatus.feasible - sol.termination_condition = TerminationCondition.optimal + sol.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[self._macro.get("SOLUTION_STATUS", "ERROR")] # Store objective value in solution sol.objective["__default_objective__"] = {"Value": self._macro["OBJECTIVE"]} @@ -606,6 +591,7 @@ def __init__(self, **kwds): # Create the session only as its needed self._sas_session = None + self._session_options = kwds def __del__(self): # Close the session, if we created one @@ -619,13 +605,6 @@ def _apply_solver(self): # Set return code to issue an error if we get interrupted self._rc = -1 - # Extract CAS connection options - cas_opts = {} - for opt in CAS_OPTION_NAMES: - val = self.options.pop(opt, None) - if val != None: - cas_opts[opt] = val - # Figure out if the problem has integer variables with_opt = self.options.pop("with", None) if with_opt == "lp": @@ -643,7 +622,7 @@ def _apply_solver(self): with redirect_stdout(SASLogWriter(self._tee)) as self._log_writer: s = self._sas_session if s == None: - s = self._sas_session = self._sas.CAS(**cas_opts) + s = self._sas_session = self._sas.CAS(**self._session_options) try: # Load the optimization action set s.loadactionset("optimization") @@ -651,8 +630,9 @@ def _apply_solver(self): # Declare a unique table name for the mps table mpsdata_table_name = "mps" + unique - # Upload mps file to CAS - if stat(self._problem_files[0]).st_size >= 2 * 1024**3: + # Upload mps file to CAS, if the file is larger than 2 GB, we need to use convertMps instead of loadMps + # Note that technically it is 2 Gibibytes file size that trigger the issue, but 2 GB is the safer threshold + if stat(self._problem_files[0]).st_size > 2E9: # For files larger than 2 GB (this is a limitation of the loadMps action used in the else part). # Use convertMPS, first create file for upload. mpsWithIdFileName = TempfileManager.create_tempfile( @@ -731,62 +711,64 @@ def _apply_solver(self): r.get("status", "ERROR"), r.get("solutionStatus", "ERROR") ) - if r.ProblemSummary["cValue1"][1] == "Maximization": - results.problem.sense = ProblemSense.maximize - else: - results.problem.sense = ProblemSense.minimize - - # Prepare the solution information - if ( - results.solver.termination_condition - == TerminationCondition.optimal - ): - sol = results.solution.add() - - # Store status in solution - sol.status = SolutionStatus.feasible - sol.termination_condition = TerminationCondition.optimal - - # Store objective value in solution - sol.objective["__default_objective__"] = { - "Value": r["objective"] - } - - if action == "solveMilp": - primal_out = s.CASTable(name=primalout_table_name) - # Use pandas functions for efficiency - primal_out = primal_out[["_VAR_", "_VALUE_"]] - sol.variable = {} - for row in primal_out.itertuples(index=False): - sol.variable[row[0]] = {"Value": row[1]} + if results.solver.status != SolverStatus.error: + if r.ProblemSummary["cValue1"][1] == "Maximization": + results.problem.sense = ProblemSense.maximize else: - # Convert primal out data set to variable dictionary - # Use panda functions for efficiency - primal_out = s.CASTable(name=primalout_table_name) - primal_out = primal_out[ - ["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"] - ] - sol.variable = {} - for row in primal_out.itertuples(index=False): - sol.variable[row[0]] = { - "Value": row[1], - "Status": row[2], - "rc": row[3], - } - - # Convert dual out data set to constraint dictionary - # Use pandas functions for efficiency - dual_out = s.CASTable(name=dualout_table_name) - dual_out = dual_out[ - ["_ROW_", "_VALUE_", "_STATUS_", "_ACTIVITY_"] - ] - sol.constraint = {} - for row in dual_out.itertuples(index=False): - sol.constraint[row[0]] = { - "dual": row[1], - "Status": row[2], - "slack": row[3], - } + results.problem.sense = ProblemSense.minimize + + # Prepare the solution information + if results.solver.hasSolution: + sol = results.solution.add() + + # Store status in solution + sol.status = SolutionStatus.feasible + sol.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[r.get("solutionStatus", "ERROR")] + + # Store objective value in solution + sol.objective["__default_objective__"] = { + "Value": r["objective"] + } + + if action == "solveMilp": + primal_out = s.CASTable(name=primalout_table_name) + # Use pandas functions for efficiency + primal_out = primal_out[["_VAR_", "_VALUE_"]] + sol.variable = {} + for row in primal_out.itertuples(index=False): + sol.variable[row[0]] = {"Value": row[1]} + else: + # Convert primal out data set to variable dictionary + # Use panda functions for efficiency + primal_out = s.CASTable(name=primalout_table_name) + primal_out = primal_out[ + ["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"] + ] + sol.variable = {} + for row in primal_out.itertuples(index=False): + sol.variable[row[0]] = { + "Value": row[1], + "Status": row[2], + "rc": row[3], + } + + # Convert dual out data set to constraint dictionary + # Use pandas functions for efficiency + dual_out = s.CASTable(name=dualout_table_name) + dual_out = dual_out[ + ["_ROW_", "_VALUE_", "_STATUS_", "_ACTIVITY_"] + ] + sol.constraint = {} + for row in dual_out.itertuples(index=False): + sol.constraint[row[0]] = { + "dual": row[1], + "Status": row[2], + "slack": row[3], + } + else: + raise ValueError( + "The SAS solver returned an error status." + ) else: results = self.results = SolverResults() results.solver.name = "SAS" diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 1a6bbd80f1d..7b0e2cccd9a 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -30,11 +30,11 @@ class SASTestAbc: solver_io = "_sas94" - base_options = {} + session_options = {} @classmethod def setUpClass(cls): - cls.opt_sas = SolverFactory("sas", solver_io=cls.solver_io) + cls.opt_sas = SolverFactory("sas", solver_io=cls.solver_io, **cls.session_options) @classmethod def tearDownClass(cls): @@ -73,11 +73,6 @@ def run_solver(self, **kwargs): opt_sas = self.opt_sas instance = self.instance - # Add base options for connection data etc. - options = kwargs.get("options", {}) - if self.base_options: - kwargs["options"] = {**options, **self.base_options} - # Call the solver self.results = opt_sas.solve(instance, **kwargs) @@ -285,7 +280,7 @@ def test_solver_with_milp(self): class SASTestLP94(SASTestLP, unittest.TestCase): @mock.patch( "pyomo.solvers.plugins.solvers.SAS.SAS94.sas_version", - return_value="2sd45s39M4234232", + return_value="9.sd45s39M4234232", ) def test_solver_versionM4(self, sas): with self.assertRaises(NotImplementedError): @@ -293,7 +288,7 @@ def test_solver_versionM4(self, sas): @mock.patch( "pyomo.solvers.plugins.solvers.SAS.SAS94.sas_version", - return_value="234897293M5324u98", + return_value="9.34897293M5324u98", ) def test_solver_versionM5(self, sas): self.run_solver() @@ -317,7 +312,7 @@ def test_solver_error(self, submit_mock, symget_mock): @unittest.skipIf(not sas_available, "The SAS solver is not available") class SASTestLPCAS(SASTestLP, unittest.TestCase): solver_io = "_sascas" - base_options = CAS_OPTIONS + session_options = CAS_OPTIONS @mock.patch("pyomo.solvers.plugins.solvers.SAS.stat") def test_solver_large_file(self, os_stat): @@ -522,7 +517,7 @@ class SASTestMILP94(SASTestMILP, unittest.TestCase): @unittest.skipIf(not sas_available, "The SAS solver is not available") class SASTestMILPCAS(SASTestMILP, unittest.TestCase): solver_io = "_sascas" - base_options = CAS_OPTIONS + session_options = CAS_OPTIONS if __name__ == "__main__": From e17a2e5b77d33c6d63e2836667de97dd854d3c41 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Tue, 12 Mar 2024 07:26:58 -0400 Subject: [PATCH 10/20] Black formatting --- pyomo/solvers/plugins/solvers/SAS.py | 32 ++++++++++++++------------ pyomo/solvers/tests/checks/test_SAS.py | 4 +++- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index 87ee31a08af..bd06f6a1ef7 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -204,9 +204,9 @@ def _create_results_from_status(self, status, solution_status): results.solver.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[ solution_status ] - results.solver.message = ( - results.solver.termination_message - ) = SOLSTATUS_TO_MESSAGE[solution_status] + results.solver.message = results.solver.termination_message = ( + SOLSTATUS_TO_MESSAGE[solution_status] + ) results.solver.status = TerminationCondition.to_solver_status( results.solver.termination_condition ) @@ -215,14 +215,14 @@ def _create_results_from_status(self, status, solution_status): elif results.solver.status == SolverStatus.aborted: results.solver.termination_condition = TerminationCondition.userInterrupt if solution_status != "ERROR": - results.solver.message = ( - results.solver.termination_message - ) = SOLSTATUS_TO_MESSAGE[solution_status] + results.solver.message = results.solver.termination_message = ( + SOLSTATUS_TO_MESSAGE[solution_status] + ) else: results.solver.termination_condition = TerminationCondition.error - results.solver.message = ( - results.solver.termination_message - ) = SOLSTATUS_TO_MESSAGE["FAILED"] + results.solver.message = results.solver.termination_message = ( + SOLSTATUS_TO_MESSAGE["FAILED"] + ) return results @abstractmethod @@ -502,7 +502,9 @@ def _apply_solver(self): # Store status in solution sol.status = SolutionStatus.feasible - sol.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[self._macro.get("SOLUTION_STATUS", "ERROR")] + sol.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[ + self._macro.get("SOLUTION_STATUS", "ERROR") + ] # Store objective value in solution sol.objective["__default_objective__"] = {"Value": self._macro["OBJECTIVE"]} @@ -632,7 +634,7 @@ def _apply_solver(self): # Upload mps file to CAS, if the file is larger than 2 GB, we need to use convertMps instead of loadMps # Note that technically it is 2 Gibibytes file size that trigger the issue, but 2 GB is the safer threshold - if stat(self._problem_files[0]).st_size > 2E9: + if stat(self._problem_files[0]).st_size > 2e9: # For files larger than 2 GB (this is a limitation of the loadMps action used in the else part). # Use convertMPS, first create file for upload. mpsWithIdFileName = TempfileManager.create_tempfile( @@ -723,7 +725,9 @@ def _apply_solver(self): # Store status in solution sol.status = SolutionStatus.feasible - sol.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[r.get("solutionStatus", "ERROR")] + sol.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[ + r.get("solutionStatus", "ERROR") + ] # Store objective value in solution sol.objective["__default_objective__"] = { @@ -766,9 +770,7 @@ def _apply_solver(self): "slack": row[3], } else: - raise ValueError( - "The SAS solver returned an error status." - ) + raise ValueError("The SAS solver returned an error status.") else: results = self.results = SolverResults() results.solver.name = "SAS" diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 7b0e2cccd9a..922209ef88b 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -34,7 +34,9 @@ class SASTestAbc: @classmethod def setUpClass(cls): - cls.opt_sas = SolverFactory("sas", solver_io=cls.solver_io, **cls.session_options) + cls.opt_sas = SolverFactory( + "sas", solver_io=cls.solver_io, **cls.session_options + ) @classmethod def tearDownClass(cls): From 07df0c69a9d62e0174d94077f3fff323bea53c87 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Tue, 26 Mar 2024 09:17:14 -0400 Subject: [PATCH 11/20] Use TeeStream, simplify _apply_solver and other small fixes --- pyomo/solvers/plugins/solvers/SAS.py | 392 +++++++++++++------------ pyomo/solvers/tests/checks/test_SAS.py | 11 + 2 files changed, 208 insertions(+), 195 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index bd06f6a1ef7..bccb7d34077 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -1,13 +1,20 @@ -__all__ = ["SAS"] +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ import logging import sys from os import stat import uuid - -from io import StringIO from abc import ABC, abstractmethod -from contextlib import redirect_stdout +from io import StringIO from pyomo.opt.base import ProblemFormat, ResultsFormat, OptSolver from pyomo.opt.base.solvers import SolverFactory @@ -23,6 +30,8 @@ from pyomo.core.base import Var from pyomo.core.base.block import _BlockData from pyomo.core.kernel.block import IBlock +from pyomo.common.log import LogStream +from pyomo.common.tee import capture_output, TeeStream logger = logging.getLogger("pyomo.solvers") @@ -252,6 +261,7 @@ class SAS94(SASAbc): """ Solver interface for SAS 9.4 using saspy. See the saspy documentation about how to create a connection. + The swat connection options can be specified on the SolverFactory call. """ def __init__(self, **kwds): @@ -541,37 +551,12 @@ def _apply_solver(self): return Bunch(rc=self._rc, log=self._log) -class SASLogWriter: - """Helper class to take the log from stdout and put it also in a StringIO.""" - - def __init__(self, tee): - """Set up the two outputs.""" - self.tee = tee - self._log = StringIO() - self.stdout = sys.stdout - - def write(self, message): - """If the tee options is specified, write to both outputs.""" - if self.tee: - self.stdout.write(message) - self._log.write(message) - - def flush(self): - """Nothing to do, just here for compatibility reasons.""" - # Do nothing since we flush right away - pass - - def log(self): - """ "Get the log as a string.""" - return self._log.getvalue() - - @SolverFactory.register("_sascas", doc="SAS Viya CAS Server interface") class SASCAS(SASAbc): """ Solver interface connection to a SAS Viya CAS server using swat. See the documentation for the swat package about how to create a connection. - The swat connection options can be passed as options to the solve function. + The swat connection options can be specified on the SolverFactory call. """ def __init__(self, **kwds): @@ -600,6 +585,106 @@ def __del__(self): if self._sas_session: self._sas_session.close() + def _uploadMpsFile(self, s, unique): + # Declare a unique table name for the mps table + mpsdata_table_name = "mps" + unique + + # Upload mps file to CAS, if the file is larger than 2 GB, we need to use convertMps instead of loadMps + # Note that technically it is 2 Gibibytes file size that trigger the issue, but 2 GB is the safer threshold + if stat(self._problem_files[0]).st_size > 2e9: + # For files larger than 2 GB (this is a limitation of the loadMps action used in the else part). + # Use convertMPS, first create file for upload. + mpsWithIdFileName = TempfileManager.create_tempfile(".mps.csv", text=True) + with open(mpsWithIdFileName, "w") as mpsWithId: + mpsWithId.write("_ID_\tText\n") + with open(self._problem_files[0], "r") as f: + id = 0 + for line in f: + id += 1 + mpsWithId.write(str(id) + "\t" + line.rstrip() + "\n") + + # Upload .mps.csv file + mpscsv_table_name = "csv" + unique + s.upload_file( + mpsWithIdFileName, + casout={"name": mpscsv_table_name, "replace": True}, + importoptions={"filetype": "CSV", "delimiter": "\t"}, + ) + + # Convert .mps.csv file to .mps + s.optimization.convertMps( + data=mpscsv_table_name, + casOut={"name": mpsdata_table_name, "replace": True}, + format="FREE", + ) + + # Delete the table we don't need anymore + if mpscsv_table_name: + s.dropTable(name=mpscsv_table_name, quiet=True) + else: + # For small files (less than 2 GB), use loadMps + with open(self._problem_files[0], "r") as mps_file: + s.optimization.loadMps( + mpsFileString=mps_file.read(), + casout={"name": mpsdata_table_name, "replace": True}, + format="FREE", + ) + return mpsdata_table_name + + def _uploadPrimalin(self, s, unique): + # Upload warmstart file to CAS with a unique name + primalin_table_name = "pin" + unique + s.upload_file( + self._warm_start_file_name, + casout={"name": primalin_table_name, "replace": True}, + importoptions={"filetype": "CSV"}, + ) + self.options["primalin"] = primalin_table_name + return primalin_table_name + + def _retrieveSolution( + self, s, r, results, action, primalout_table_name, dualout_table_name + ): + # Create solution + sol = results.solution.add() + + # Store status in solution + sol.status = SolutionStatus.feasible + sol.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[ + r.get("solutionStatus", "ERROR") + ] + + # Store objective value in solution + sol.objective["__default_objective__"] = {"Value": r["objective"]} + + if action == "solveMilp": + primal_out = s.CASTable(name=primalout_table_name) + # Use pandas functions for efficiency + primal_out = primal_out[["_VAR_", "_VALUE_"]] + sol.variable = {} + for row in primal_out.itertuples(index=False): + sol.variable[row[0]] = {"Value": row[1]} + else: + # Convert primal out data set to variable dictionary + # Use panda functions for efficiency + primal_out = s.CASTable(name=primalout_table_name) + primal_out = primal_out[["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"]] + sol.variable = {} + for row in primal_out.itertuples(index=False): + sol.variable[row[0]] = {"Value": row[1], "Status": row[2], "rc": row[3]} + + # Convert dual out data set to constraint dictionary + # Use pandas functions for efficiency + dual_out = s.CASTable(name=dualout_table_name) + dual_out = dual_out[["_ROW_", "_VALUE_", "_STATUS_", "_ACTIVITY_"]] + sol.constraint = {} + for row in dual_out.itertuples(index=False): + sol.constraint[row[0]] = { + "dual": row[1], + "Status": row[2], + "slack": row[3], + } + def _apply_solver(self): """ "Prepare the options and run the solver. Then store the data to be returned.""" logger.debug("Running SAS Viya") @@ -620,175 +705,92 @@ def _apply_solver(self): # Get a unique identifier, always use the same with different prefixes unique = uuid.uuid4().hex[:16] + # Creat the output stream, we want to print to a log string as well as to the console + self._log = StringIO() + ostreams = [LogStream(level=logging.INFO, logger=logger)] + ostreams.append(self._log) + if self._tee: + ostreams.append(sys.stdout) + # Connect to CAS server - with redirect_stdout(SASLogWriter(self._tee)) as self._log_writer: - s = self._sas_session - if s == None: - s = self._sas_session = self._sas.CAS(**self._session_options) - try: - # Load the optimization action set - s.loadactionset("optimization") - - # Declare a unique table name for the mps table - mpsdata_table_name = "mps" + unique - - # Upload mps file to CAS, if the file is larger than 2 GB, we need to use convertMps instead of loadMps - # Note that technically it is 2 Gibibytes file size that trigger the issue, but 2 GB is the safer threshold - if stat(self._problem_files[0]).st_size > 2e9: - # For files larger than 2 GB (this is a limitation of the loadMps action used in the else part). - # Use convertMPS, first create file for upload. - mpsWithIdFileName = TempfileManager.create_tempfile( - ".mps.csv", text=True - ) - with open(mpsWithIdFileName, "w") as mpsWithId: - mpsWithId.write("_ID_\tText\n") - with open(self._problem_files[0], "r") as f: - id = 0 - for line in f: - id += 1 - mpsWithId.write(str(id) + "\t" + line.rstrip() + "\n") - - # Upload .mps.csv file - mpscsv_table_name = "csv" + unique - s.upload_file( - mpsWithIdFileName, - casout={"name": mpscsv_table_name, "replace": True}, - importoptions={"filetype": "CSV", "delimiter": "\t"}, - ) - - # Convert .mps.csv file to .mps - s.optimization.convertMps( - data=mpscsv_table_name, - casOut={"name": mpsdata_table_name, "replace": True}, - format="FREE", - ) - - # Delete the table we don't need anymore - if mpscsv_table_name: - s.dropTable(name=mpscsv_table_name, quiet=True) - else: - # For small files (less than 2 GB), use loadMps - with open(self._problem_files[0], "r") as mps_file: - s.optimization.loadMps( - mpsFileString=mps_file.read(), - casout={"name": mpsdata_table_name, "replace": True}, - format="FREE", + with TeeStream(*ostreams) as t: + with capture_output(output=t.STDOUT, capture_fd=False): + s = self._sas_session + if s == None: + s = self._sas_session = self._sas.CAS(**self._session_options) + try: + # Load the optimization action set + s.loadactionset("optimization") + + mpsdata_table_name = self._uploadMpsFile(s, unique) + + primalin_table_name = None + if self.warmstart_flag: + primalin_table_name = self._uploadPrimalin(s, unique) + + # Define output table names + primalout_table_name = "pout" + unique + dualout_table_name = None + + # Solve the problem in CAS + if action == "solveMilp": + r = s.optimization.solveMilp( + data={"name": mpsdata_table_name}, + primalOut={"name": primalout_table_name, "replace": True}, + **self.options + ) + else: + dualout_table_name = "dout" + unique + r = s.optimization.solveLp( + data={"name": mpsdata_table_name}, + primalOut={"name": primalout_table_name, "replace": True}, + dualOut={"name": dualout_table_name, "replace": True}, + **self.options ) - primalin_table_name = None - if self.warmstart_flag: - primalin_table_name = "pin" + unique - # Upload warmstart file to CAS - s.upload_file( - self._warm_start_file_name, - casout={"name": primalin_table_name, "replace": True}, - importoptions={"filetype": "CSV"}, - ) - self.options["primalin"] = primalin_table_name - - # Define output table names - primalout_table_name = "pout" + unique - dualout_table_name = None - - # Solve the problem in CAS - if action == "solveMilp": - r = s.optimization.solveMilp( - data={"name": mpsdata_table_name}, - primalOut={"name": primalout_table_name, "replace": True}, - **self.options - ) - else: - dualout_table_name = "dout" + unique - r = s.optimization.solveLp( - data={"name": mpsdata_table_name}, - primalOut={"name": primalout_table_name, "replace": True}, - dualOut={"name": dualout_table_name, "replace": True}, - **self.options - ) - - # Prepare the solver results - if r: - # Get back the primal and dual solution data sets - results = self.results = self._create_results_from_status( - r.get("status", "ERROR"), r.get("solutionStatus", "ERROR") - ) - - if results.solver.status != SolverStatus.error: - if r.ProblemSummary["cValue1"][1] == "Maximization": - results.problem.sense = ProblemSense.maximize - else: - results.problem.sense = ProblemSense.minimize - - # Prepare the solution information - if results.solver.hasSolution: - sol = results.solution.add() - - # Store status in solution - sol.status = SolutionStatus.feasible - sol.termination_condition = SOLSTATUS_TO_TERMINATIONCOND[ - r.get("solutionStatus", "ERROR") - ] - - # Store objective value in solution - sol.objective["__default_objective__"] = { - "Value": r["objective"] - } - - if action == "solveMilp": - primal_out = s.CASTable(name=primalout_table_name) - # Use pandas functions for efficiency - primal_out = primal_out[["_VAR_", "_VALUE_"]] - sol.variable = {} - for row in primal_out.itertuples(index=False): - sol.variable[row[0]] = {"Value": row[1]} + # Prepare the solver results + if r: + # Get back the primal and dual solution data sets + results = self.results = self._create_results_from_status( + r.get("status", "ERROR"), r.get("solutionStatus", "ERROR") + ) + + if results.solver.status != SolverStatus.error: + if r.ProblemSummary["cValue1"][1] == "Maximization": + results.problem.sense = ProblemSense.maximize else: - # Convert primal out data set to variable dictionary - # Use panda functions for efficiency - primal_out = s.CASTable(name=primalout_table_name) - primal_out = primal_out[ - ["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"] - ] - sol.variable = {} - for row in primal_out.itertuples(index=False): - sol.variable[row[0]] = { - "Value": row[1], - "Status": row[2], - "rc": row[3], - } - - # Convert dual out data set to constraint dictionary - # Use pandas functions for efficiency - dual_out = s.CASTable(name=dualout_table_name) - dual_out = dual_out[ - ["_ROW_", "_VALUE_", "_STATUS_", "_ACTIVITY_"] - ] - sol.constraint = {} - for row in dual_out.itertuples(index=False): - sol.constraint[row[0]] = { - "dual": row[1], - "Status": row[2], - "slack": row[3], - } + results.problem.sense = ProblemSense.minimize + + # Prepare the solution information + if results.solver.hasSolution: + self._retrieveSolution( + s, + r, + results, + action, + primalout_table_name, + dualout_table_name, + ) + else: + raise ValueError("The SAS solver returned an error status.") else: - raise ValueError("The SAS solver returned an error status.") - else: - results = self.results = SolverResults() - results.solver.name = "SAS" - results.solver.status = SolverStatus.error - raise ValueError( - "An option passed to the SAS solver caused a syntax error." - ) - - finally: - if mpsdata_table_name: - s.dropTable(name=mpsdata_table_name, quiet=True) - if primalin_table_name: - s.dropTable(name=primalin_table_name, quiet=True) - if primalout_table_name: - s.dropTable(name=primalout_table_name, quiet=True) - if dualout_table_name: - s.dropTable(name=dualout_table_name, quiet=True) - - self._log = self._log_writer.log() + results = self.results = SolverResults() + results.solver.name = "SAS" + results.solver.status = SolverStatus.error + raise ValueError( + "An option passed to the SAS solver caused a syntax error." + ) + + finally: + if mpsdata_table_name: + s.dropTable(name=mpsdata_table_name, quiet=True) + if primalin_table_name: + s.dropTable(name=primalin_table_name, quiet=True) + if primalout_table_name: + s.dropTable(name=primalout_table_name, quiet=True) + if dualout_table_name: + s.dropTable(name=dualout_table_name, quiet=True) + + self._log = self._log.getvalue() self._rc = 0 return Bunch(rc=self._rc, log=self._log) diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 922209ef88b..75534e0e001 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -1,3 +1,14 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + import os import pyomo.common.unittest as unittest from unittest import mock From 6834c906bc299575f7588bb3b07558b63c5d7050 Mon Sep 17 00:00:00 2001 From: Miranda Rose Mundt Date: Wed, 5 Jun 2024 10:32:35 -0600 Subject: [PATCH 12/20] Add in cfgfile information; turn off SAS Viya tests for now --- pyomo/solvers/tests/checks/test_SAS.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 75534e0e001..843ca3f2d2b 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -29,6 +29,8 @@ from pyomo.opt import SolverFactory, check_available_solvers import warnings +CFGFILE = os.environ.get("SAS_CFG_FILE_PATH", None) + CAS_OPTIONS = { "hostname": os.environ.get("CASHOST", None), "port": os.environ.get("CASPORT", None), @@ -42,11 +44,12 @@ class SASTestAbc: solver_io = "_sas94" session_options = {} + cfgfile = CFGFILE @classmethod def setUpClass(cls): cls.opt_sas = SolverFactory( - "sas", solver_io=cls.solver_io, **cls.session_options + "sas", solver_io=cls.solver_io, cfgfile=cls.cfgfile, **cls.session_options ) @classmethod @@ -322,7 +325,8 @@ def test_solver_error(self, submit_mock, symget_mock): self.assertEqual(results.solver.status, SolverStatus.error) -@unittest.skipIf(not sas_available, "The SAS solver is not available") +#@unittest.skipIf(not sas_available, "The SAS solver is not available") +@unittest.skip("Tests not yet configured for SAS Viya interface.") class SASTestLPCAS(SASTestLP, unittest.TestCase): solver_io = "_sascas" session_options = CAS_OPTIONS @@ -528,6 +532,7 @@ class SASTestMILP94(SASTestMILP, unittest.TestCase): @unittest.skipIf(not sas_available, "The SAS solver is not available") +@unittest.skip("Tests not yet configured for SAS Viya interface.") class SASTestMILPCAS(SASTestMILP, unittest.TestCase): solver_io = "_sascas" session_options = CAS_OPTIONS From 6d5c931bf00b9ac9b0144314584eff2b52d8b92c Mon Sep 17 00:00:00 2001 From: Miranda Rose Mundt Date: Wed, 5 Jun 2024 10:38:20 -0600 Subject: [PATCH 13/20] Apply black to test_SAS --- pyomo/solvers/tests/checks/test_SAS.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 843ca3f2d2b..00b4d7a9b3e 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -325,7 +325,7 @@ def test_solver_error(self, submit_mock, symget_mock): self.assertEqual(results.solver.status, SolverStatus.error) -#@unittest.skipIf(not sas_available, "The SAS solver is not available") +# @unittest.skipIf(not sas_available, "The SAS solver is not available") @unittest.skip("Tests not yet configured for SAS Viya interface.") class SASTestLPCAS(SASTestLP, unittest.TestCase): solver_io = "_sascas" @@ -531,7 +531,7 @@ class SASTestMILP94(SASTestMILP, unittest.TestCase): pass -@unittest.skipIf(not sas_available, "The SAS solver is not available") +# @unittest.skipIf(not sas_available, "The SAS solver is not available") @unittest.skip("Tests not yet configured for SAS Viya interface.") class SASTestMILPCAS(SASTestMILP, unittest.TestCase): solver_io = "_sascas" From 60dcc7a3f9875ab7e32a2d9540edfcfa243f054d Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Wed, 5 Jun 2024 13:14:16 -0600 Subject: [PATCH 14/20] Import specific library from uuid --- pyomo/solvers/plugins/solvers/SAS.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index bccb7d34077..eb5014fbd1a 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -12,7 +12,7 @@ import logging import sys from os import stat -import uuid +from uuid import uuid4 from abc import ABC, abstractmethod from io import StringIO @@ -334,7 +334,7 @@ def _apply_solver(self): rootnode_str = self._create_statement_str("rootnode") # Get a unique identifier, always use the same with different prefixes - unique = uuid.uuid4().hex[:16] + unique = uuid4().hex[:16] # Create unique filename for output datasets primalout_dataset_name = "pout" + unique @@ -703,7 +703,7 @@ def _apply_solver(self): action = "solveMilp" if self._has_integer_variables() else "solveLp" # Get a unique identifier, always use the same with different prefixes - unique = uuid.uuid4().hex[:16] + unique = uuid4().hex[:16] # Creat the output stream, we want to print to a log string as well as to the console self._log = StringIO() From d84a8a019c715082fbbe4167b66adb5f87ce113d Mon Sep 17 00:00:00 2001 From: Miranda Mundt <55767766+mrmundt@users.noreply.github.com> Date: Mon, 10 Jun 2024 12:21:52 -0600 Subject: [PATCH 15/20] Update test_SAS.py --- pyomo/solvers/tests/checks/test_SAS.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyomo/solvers/tests/checks/test_SAS.py b/pyomo/solvers/tests/checks/test_SAS.py index 00b4d7a9b3e..6dd662bdb21 100644 --- a/pyomo/solvers/tests/checks/test_SAS.py +++ b/pyomo/solvers/tests/checks/test_SAS.py @@ -526,7 +526,8 @@ def test_solver_warmstart_capable(self): self.assertTrue(self.opt_sas.warm_start_capable()) -@unittest.skipIf(not sas_available, "The SAS solver is not available") +# @unittest.skipIf(not sas_available, "The SAS solver is not available") +@unittest.skip("MILP94 tests disabled.") class SASTestMILP94(SASTestMILP, unittest.TestCase): pass From 1c0a0307e00848f03f4cf5c46efd8c4a114b1adf Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Wed, 26 Jun 2024 08:07:30 -0400 Subject: [PATCH 16/20] Fix warning and improve session handling --- pyomo/solvers/plugins/solvers/SAS.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index eb5014fbd1a..fbbd6e5c3eb 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -28,7 +28,7 @@ ) from pyomo.common.tempfiles import TempfileManager from pyomo.core.base import Var -from pyomo.core.base.block import _BlockData +from pyomo.core.base.block import BlockData from pyomo.core.kernel.block import IBlock from pyomo.common.log import LogStream from pyomo.common.tee import capture_output, TeeStream @@ -157,7 +157,7 @@ def _presolve(self, *args, **kwds): # Store the model, too bad this is not done in the base class for arg in args: - if isinstance(arg, (_BlockData, IBlock)): + if isinstance(arg, (BlockData, IBlock)): # Store the instance self._instance = arg self._vars = [] @@ -282,16 +282,17 @@ def __init__(self, **kwds): self._python_api_exists = True self._sas.logger.setLevel(logger.level) - # Create the session only as its needed - self._sas_session = None - # Store other options for the SAS session self._session_options = kwds + # Create the session + self._sas_session = self._sas.SASsession(**self._session_options) + def __del__(self): # Close the session, if we created one if self._sas_session: self._sas_session.endsas() + del self._sas_session def _create_statement_str(self, statement): """Helper function to create the strings for the statements of the proc OPTLP/OPTMILP code.""" @@ -369,7 +370,10 @@ def _apply_solver(self): sas_options = "option notes nonumber nodate nosource pagesize=max;" # Get the current SAS session, submit the code and return the results - sas = self._sas_session = self._sas.SASsession(**self._session_options) + if not self._sas_session: + sas = self._sas_session = self._sas.SASsession(**self._session_options) + else: + sas = self._sas_session # Find the version of 9.4 we are using self._sasver = sas.sasver @@ -576,14 +580,16 @@ def __init__(self, **kwds): else: self._python_api_exists = True - # Create the session only as its needed - self._sas_session = None self._session_options = kwds + # Create the session + self._sas_session = self._sas.CAS(**self._session_options) + def __del__(self): # Close the session, if we created one if self._sas_session: self._sas_session.close() + del self._sas_session def _uploadMpsFile(self, s, unique): # Declare a unique table name for the mps table From 6cce08fba429f16c6294fd8ff4a463a09d802af6 Mon Sep 17 00:00:00 2001 From: Miranda Mundt <55767766+mrmundt@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:44:42 -0600 Subject: [PATCH 17/20] Change `_sas_session` check --- pyomo/solvers/plugins/solvers/SAS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index fbbd6e5c3eb..bc46f1c7465 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -290,7 +290,7 @@ def __init__(self, **kwds): def __del__(self): # Close the session, if we created one - if self._sas_session: + if hasattr(self, '_sas_session'): self._sas_session.endsas() del self._sas_session From ee9d446a882642dd8b7c2e269b9bd6a77a846169 Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Tue, 2 Jul 2024 15:44:57 -0600 Subject: [PATCH 18/20] Add a try-except block around session creation --- pyomo/solvers/plugins/solvers/SAS.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index bc46f1c7465..80ec8d2877d 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -286,11 +286,14 @@ def __init__(self, **kwds): self._session_options = kwds # Create the session - self._sas_session = self._sas.SASsession(**self._session_options) + try: + self._sas_session = self._sas.SASsession(**self._session_options) + except: + self._sas_session = None def __del__(self): # Close the session, if we created one - if hasattr(self, '_sas_session'): + if self._sas_session: self._sas_session.endsas() del self._sas_session @@ -583,7 +586,10 @@ def __init__(self, **kwds): self._session_options = kwds # Create the session - self._sas_session = self._sas.CAS(**self._session_options) + try: + self._sas_session = self._sas.CAS(**self._session_options) + except: + self._sas_session = None def __del__(self): # Close the session, if we created one From 0af995b5eeea07e59f014786d8c60e00ed82da8c Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Wed, 3 Jul 2024 07:45:43 -0600 Subject: [PATCH 19/20] Change uuid import to attempt_import --- pyomo/solvers/plugins/solvers/SAS.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index 80ec8d2877d..865efa36dc3 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -12,13 +12,13 @@ import logging import sys from os import stat -from uuid import uuid4 from abc import ABC, abstractmethod from io import StringIO from pyomo.opt.base import ProblemFormat, ResultsFormat, OptSolver from pyomo.opt.base.solvers import SolverFactory from pyomo.common.collections import Bunch +from pyomo.common.dependencies import attempt_import from pyomo.opt.results import ( SolverResults, SolverStatus, @@ -34,6 +34,7 @@ from pyomo.common.tee import capture_output, TeeStream +uuid, uuid_available = attempt_import('uuid') logger = logging.getLogger("pyomo.solvers") @@ -338,7 +339,7 @@ def _apply_solver(self): rootnode_str = self._create_statement_str("rootnode") # Get a unique identifier, always use the same with different prefixes - unique = uuid4().hex[:16] + unique = uuid.uuid4().hex[:16] # Create unique filename for output datasets primalout_dataset_name = "pout" + unique From 51568d1a2e92685d640f96e66ddedee3c35727b8 Mon Sep 17 00:00:00 2001 From: Bethany Nicholson Date: Wed, 3 Jul 2024 11:59:00 -0600 Subject: [PATCH 20/20] Fix typos from code review --- pyomo/solvers/plugins/solvers/SAS.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index 865efa36dc3..f2cfe279fdc 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -397,12 +397,12 @@ def _apply_solver(self): ) upload_pin = True - # Using a function call to make it easier to moch the version check + # Using a function call to make it easier to mock the version check major_version = self.sas_version()[0] minor_version = self.sas_version().split("M", 1)[1][0] if major_version == "9" and int(minor_version) < 5: raise NotImplementedError( - "Support for SAS 9.4 M4 and earlier is no implemented." + "Support for SAS 9.4 M4 and earlier is not implemented." ) elif major_version == "9" and int(minor_version) == 5: # In 9.4M5 we have to create an MPS data set from an MPS file first @@ -529,7 +529,7 @@ def _apply_solver(self): if proc == "OPTLP": # Convert primal out data set to variable dictionary - # Use panda functions for efficiency + # Use pandas functions for efficiency primal_out = primal_out[["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"]] primal_out = primal_out.set_index("_VAR_", drop=True) primal_out = primal_out.rename(