Skip to content

Commit

Permalink
feat: --assert_all_regions_parallelizable flag that has the same func…
Browse files Browse the repository at this point in the history
…tion as old assert_compiler_success flag; new assert_compiler_success flag now will not exit with error when regions are unparallelizable and exit with error when general exceptions are caught
  • Loading branch information
YUUU23 committed Jun 16, 2024
1 parent c11365c commit cd4c265
Show file tree
Hide file tree
Showing 7 changed files with 66 additions and 11 deletions.
7 changes: 6 additions & 1 deletion compiler/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,12 @@ def add_pash_args(self):
)
self.add_argument(
"--assert_compiler_success",
help="assert that the compiler succeeded (used to make tests more robust)",
help="assert that the compiler succeeded with no general error occuring",
action="store_true",
)
self.add_argument(
"--assert_all_regions_parallelizable",
help="assert that the compiler succeeded with all regions being parallelizable and no general error occuring (used to make tests more robust)",
action="store_true",
)
self.add_argument(
Expand Down
9 changes: 8 additions & 1 deletion compiler/custom_error.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,11 @@ class UnparallelizableError(Exception):
pass

class AdjLineNotImplementedError(Exception):
pass
pass

# to be raised in pash_compiler if a UnparallelizableError is caught at any point running the compiler
# primarily to differentiate
# --assert_compiler_success (exit with error only under general exceptions caught)
# --assert_all_regions_parallelizable (exit with error when regions are found not parallelizable + general exceptions)
class NotAllRegionParallelizableError(Exception):
pass
5 changes: 5 additions & 0 deletions compiler/orchestrator_runtime/pash_init_setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ export pash_output_time_flag=1
export pash_execute_flag=1
export pash_dry_run_compiler_flag=0
export pash_assert_compiler_success_flag=0
export pash_assert_all_regions_parallelizable_flag=0
export pash_checking_log_file=0
export pash_checking_debug_level=0
export pash_avoid_pash_runtime_completion_flag=0
Expand Down Expand Up @@ -51,6 +52,10 @@ do
export pash_assert_compiler_success_flag=1
fi

if [ "--assert_all_regions_parallelizable" == "$item" ]; then
export pash_assert_all_regions_parallelizable_flag=1
fi

if [ "--log_file" == "$item" ]; then
pash_checking_log_file=1
fi
Expand Down
21 changes: 20 additions & 1 deletion compiler/orchestrator_runtime/pash_prepare_call_compiler.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@ pash_redir_output echo "$$: (2) Before asking the daemon for compilation..."
msg="Compile:${pash_compiled_script_file}| Variable File:${pash_runtime_shell_variables_file}| Input IR File:${pash_input_ir_file}"
daemon_response=$(pash_communicate_daemon "$msg") # Blocking step, daemon will not send response until it's safe to continue

if [[ "$daemon_response" == *"not all regions are parallelizable"* ]]; then
pash_all_region_parallelizable=1
else
pash_all_region_parallelizable=0
fi

if [[ "$daemon_response" == *"OK:"* ]]; then
pash_runtime_return_code=0
elif [ -z "$daemon_response" ]; then
Expand All @@ -51,7 +57,20 @@ response_args=($daemon_response)
process_id=${response_args[1]}

pash_redir_output echo "$$: (2) Compiler exited with code: $pash_runtime_return_code"
if [ "$pash_runtime_return_code" -ne 0 ] && [ "$pash_assert_compiler_success_flag" -eq 1 ]; then

## only when --assert_all_regions_parallellizable is used do we care about all regions being parallelizable
if [ "$pash_all_region_parallelizable" -ne 0 ] && [ "$pash_assert_all_regions_parallelizable_flag" -eq 1 ]; then
pash_redir_output echo "$$: ERROR: (2) Compiler failed with error code because some regions were not parallelizable: $pash_all_region_parallelizable while assert_all_regions_parallelizable_flag was enabled! Exiting PaSh..."
exit 1
fi

if [ "$pash_runtime_return_code" -ne 0 ] && [ "$pash_assert_all_regions_parallelizable_flag" -eq 1 ]; then
pash_redir_output echo "$$: ERROR: (2) Compiler failed with error code: $pash_runtime_return_code while assert_all_regions_parallelizable_flag was enabled! Exiting PaSh..."
exit 1
fi

## for pash_assert_compiler_success_flag, exit when return code is 0 (general exception caught) and not when all regions are parallelizable
if [ "$pash_runtime_return_code" -ne 0 ] && [ "$pash_all_region_parallelizable" -eq 0 ] && [ "$pash_assert_compiler_success_flag" -eq 1 ]; then
pash_redir_output echo "$$: ERROR: (2) Compiler failed with error code: $pash_runtime_return_code while assert_compiler_success was enabled! Exiting PaSh..."
exit 1
fi
Expand Down
31 changes: 24 additions & 7 deletions compiler/pash_compilation_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import server_util

from cli import BaseParser
from custom_error import *

##
## A Daemon (not with the strict Unix sense)
Expand Down Expand Up @@ -252,6 +253,7 @@ def compile_and_add(self, compiled_script_file, var_file, input_ir_file):
process_id = self.get_next_id()
run_parallel = False
compile_success = False
all_region_parallelizable = True

variable_reading_start_time = datetime.now()
# Read any shell variables files if present
Expand All @@ -269,9 +271,15 @@ def compile_and_add(self, compiled_script_file, var_file, input_ir_file):
## Add the process_id -> input_ir mapping
self.add_proc_id_map(process_id, input_ir_file, compiler_config)

ast_or_ir = pash_compiler.compile_ir(
input_ir_file, compiled_script_file, config.pash_args, compiler_config
)
# check if any general exceptions are caught to report to --assert_compiler_success flag
try:
ast_or_ir = pash_compiler.compile_ir(
input_ir_file, compiled_script_file, config.pash_args, compiler_config
)
except NotAllRegionParallelizableError:
ast_or_ir = None
all_region_parallelizable = False


daemon_compile_end_time = datetime.now()
print_time_delta(
Expand Down Expand Up @@ -321,19 +329,28 @@ def compile_and_add(self, compiled_script_file, var_file, input_ir_file):
else:
## Wait if we have more pipelines running than our current limit
self.wait_until_limit(config.pash_args.parallel_pipelines_limit)

if compile_success:
response = server_util.success_response(
f"{process_id} {compiled_script_file} {var_file} {input_ir_file}"
)
elif not all_region_parallelizable:
# send specified message to say not all regions are parallelizable instead of general exception caught
response = server_util.error_response(f"{process_id} not all regions are parallelizable; failed to compile")
self.unsafe_running = True
else:
response = server_util.error_response(f"{process_id} failed to compile")
self.unsafe_running = True


## Do not increase the running procs if assert_compiler_success is enabled
## Do not increase the running procs if assert_all_regions_parallelizable is enabled
## and compilation failed, since nothing will run then.
if not compile_success and config.pash_args.assert_compiler_success:
pass
## Do not increase when compile is not successful but regions are parallelizable (in the case that general exceptions are caught),
## nothing will run in this case also
if (not compile_success and config.pash_args.assert_all_regions_parallelizable):
pass
elif (not compile_success and all_region_parallelizable and config.pash_args.assert_compiler_success):
pass
else:
self.running_procs += 1

Expand Down
2 changes: 2 additions & 0 deletions compiler/pash_compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,10 @@ def compile_ir(ir_filename, compiled_script_file, args, compiler_config):
)
except ExpansionError as e:
log("WARNING: Exception caught because some region(s) are not expandable and therefore unparallelizable:", e)
raise NotAllRegionParallelizableError()
except UnparallelizableError as e:
log("WARNING: Exception caught because some region(s) are unparallelizable:", e)
raise NotAllRegionParallelizableError()
# log(traceback.format_exc()) # uncomment for exact trace report (PaSh user should see informative messages for unparellizable regions)
except (AdjLineNotImplementedError, NotImplementedError) as e:
log("WARNING: Exception caught because some part is not implemented:", e)
Expand Down
2 changes: 1 addition & 1 deletion evaluation/tests/test_evaluation_scripts.sh
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ execute_tests() {
}

execute_tests "" "${script_microbenchmarks[@]}"
execute_tests "--assert_compiler_success" "${pipeline_microbenchmarks[@]}"
execute_tests "--assert_all_regions_parallelizable" "${pipeline_microbenchmarks[@]}"

#cat ${results_time} | sed 's/,/./' > /tmp/a
#cat /tmp/a | sed 's/@/,/' > ${results_time}
Expand Down

0 comments on commit cd4c265

Please sign in to comment.