From 467e56309b1fd09f6ff79823388ec746073e50f4 Mon Sep 17 00:00:00 2001 From: Nickolas Comeau Date: Wed, 28 Feb 2024 09:53:03 -0500 Subject: [PATCH 1/2] Use billing instead of cpu/gpu for TRES limit (#363) * use billing instead of cpu/gpu for TRES limit * getting the lock state should look for billing as well --- bank/system/slurm.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/bank/system/slurm.py b/bank/system/slurm.py index cbaefed0..e8b56489 100644 --- a/bank/system/slurm.py +++ b/bank/system/slurm.py @@ -128,7 +128,7 @@ def get_locked_state(self, cluster: str) -> bool: raise ClusterNotFoundError(f'Cluster {cluster} is not configured with Slurm') cmd = f'sacctmgr -n -P show assoc account={self.account_name} format=GrpTresRunMins clusters={cluster}' - return 'cpu=0' in ShellCmd(cmd).out + return 'billing=0' in ShellCmd(cmd).out def set_locked_state(self, lock_state: bool, cluster: str) -> None: """Lock or unlock the current slurm account @@ -146,10 +146,7 @@ def set_locked_state(self, lock_state: bool, cluster: str) -> None: raise ClusterNotFoundError(f'Cluster {cluster} is not configured with Slurm') lock_state_int = 0 if lock_state else -1 - ShellCmd(f'sacctmgr -i modify account where account={self.account_name} cluster={cluster} ' - f'set GrpTresRunMins=cpu={lock_state_int}').raise_if_err() - ShellCmd(f'sacctmgr -i modify account where account={self.account_name} cluster={cluster} ' - f'set GrpTresRunMins=gres/gpu={lock_state_int}').raise_if_err() + ShellCmd(f'sacctmgr -i modify account where account={self.account_name} cluster={cluster} set GrpTresRunMins=billing={lock_state_int}').raise_if_err() def get_cluster_usage_per_user(self, cluster: str, start: date, end: date, in_hours: bool = True) -> Dict[str, int]: """Return the raw account usage per user on a given cluster From 9f6e3333b95a7cf59aec1633c975d408466d7eba Mon Sep 17 00:00:00 2001 From: Nickolas Comeau Date: Tue, 5 Mar 2024 11:48:13 -0500 Subject: [PATCH 2/2] Update usage output (#364) * Only add/remove SUs to total (not current) on investments * Add current totals to floating and investment SU table output * make last investment row a dividier * Pull duplicate header items from investment output --- bank/account_logic.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/bank/account_logic.py b/bank/account_logic.py index 72800dab..1a3a300c 100644 --- a/bank/account_logic.py +++ b/bank/account_logic.py @@ -543,7 +543,6 @@ def add_sus(self, inv_id: Optional[int], sus: int) -> None: with DBConnection.session() as session: investment = session.execute(query).scalars().first() investment.service_units += sus - investment.current_sus += sus session.commit() @@ -573,7 +572,6 @@ def subtract_sus(self, inv_id: Optional[int], sus: int) -> None: f'Cannot subtract {sus}. Investment {inv_id} only has {investment.current_sus} available.') investment.service_units -= sus - investment.current_sus -= sus session.commit() @@ -794,28 +792,33 @@ def _build_usage_table(self) -> PrettyTable: usage_percentage = self._calculate_percentage(aggregate_usage_total, allocation_total) floating_su_percent = self._calculate_percentage(floating_su_usage, floating_su_total) - output_table.add_row(['Floating SUs', "SUs Remaining", "% Used"], divider=True) - output_table.add_row([f'*Floating SUs', "", ""]) - output_table.add_row([f'are applied on', "", ""]) - output_table.add_row([f'any cluster to', str(floating_su_remaining)+'*', floating_su_percent]) - output_table.add_row([f'cover usage above', "", ""]) - output_table.add_row([f'Total SUs', "", ""], divider=True) + output_table.add_row(["Floating SUs", "", ""], divider=True) + output_table.add_row(["Floating SUs", "", ""]) + output_table.add_row(["are applied on", "", ""]) + output_table.add_row(["any cluster to", "", ""]) + output_table.add_row(["cover usage above", "", ""]) + output_table.add_row(["Total Proposal SUs", "", ""]) + output_table.add_row(["Total", "SUs Remaining", "% Used"]) + output_table.add_row([floating_su_total, floating_su_remaining, floating_su_percent], divider=True) # Add another inner table describing aggregate usage if not investments: output_table.add_row(['Aggregate Usage', usage_percentage, ""], divider=True) else: investment_total = sum(inv.service_units for inv in investments) - investment_remaining = sum(inv.current_sus for inv in investments) + investment_remaining = sum(inv.current_sus for inv in investments) investment_used = investment_total - investment_remaining investment_percentage = self._calculate_percentage(investment_used, investment_total) - output_table.add_row(['Investment SUs', "SUs Remaining", "% Used"], divider=True) - output_table.add_row([f'**Investment SUs', "",""]) - output_table.add_row([f'are applied on', "", ""]) - output_table.add_row([f'any cluster to', str(investment_remaining)+"**", investment_percentage]) - output_table.add_row([f'cover usage above',"",""]) - output_table.add_row([f'Total SUs', "", ""], divider=True) + output_table.add_row(["Investment SUs", "", ""], divider=True) + output_table.add_row(["Investment SUs", "", ""]) + output_table.add_row(["are applied on", "", ""]) + output_table.add_row(["any cluster to", "", ""]) + output_table.add_row(["cover usage above", "", ""]) + output_table.add_row(["Total Proposal SUs", "", ""]) + output_table.add_row(["Total", "SUs Remaining", "% Used"]) + output_table.add_row([investment_total, investment_remaining, investment_percentage], divider=True) + output_table.add_row(['Aggregate Usage', usage_percentage, ""]) output_table.add_row(['(no investments)', "", ""])