From 8a8b0a5c38729415ff4b35a37915d9db084ad819 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sat, 27 Jul 2024 18:50:15 +0200 Subject: [PATCH 1/8] tvm backends: add relax_mode config (WIP) --- mlonmcu/flow/tvm/backend/backend.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mlonmcu/flow/tvm/backend/backend.py b/mlonmcu/flow/tvm/backend/backend.py index 41317f39..e79b233a 100644 --- a/mlonmcu/flow/tvm/backend/backend.py +++ b/mlonmcu/flow/tvm/backend/backend.py @@ -84,6 +84,7 @@ class TVMBackend(Backend): "relay_debug": None, # Use "DEFAULT=2" to have most verbosity. Needs USE_RELAY_DEBUG during setup. "refresh_model_info": False, "generate_wrapper": "auto", + "relax_mode": False, } REQUIRED = set() @@ -247,6 +248,11 @@ def use_tuning_results(self): value = self.config["use_tuning_results"] return str2bool(value) if not isinstance(value, (bool, int)) else value + @property + def relax_mode(self): + value = self.config["relax_mode"] + return str2bool(value) if not isinstance(value, (bool, int)) else value + @property def tvmc_extra_args(self): return self.config["tvmc_extra_args"] @@ -381,6 +387,8 @@ def get_tvmc_compile_args(self, out, dump=None): *["-f", self.fmt], *["--model-format", self.model_format], ] + if self.relax_mode: + args.append("--relax") return args def invoke_tvmc(self, command, *args, cwd=None): From 279146ea1830c75a734fb9185bc4de1efec700d6 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sat, 27 Jul 2024 18:51:14 +0200 Subject: [PATCH 2/8] tvm backends: pass config improvements --- mlonmcu/flow/tvm/backend/backend.py | 12 +++++++++--- mlonmcu/flow/tvm/backend/tvmaot.py | 10 +++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/mlonmcu/flow/tvm/backend/backend.py b/mlonmcu/flow/tvm/backend/backend.py index e79b233a..6d7bafe9 100644 --- a/mlonmcu/flow/tvm/backend/backend.py +++ b/mlonmcu/flow/tvm/backend/backend.py @@ -155,14 +155,20 @@ def disable_vectorize(self): return str2bool(temp) @property - def pass_config(self): - base = {"tir.disable_vectorize": self.disable_vectorize} + def extra_pass_config(self): extra = self.config["extra_pass_config"] + if extra is None: + extra = {} if isinstance(extra, str): import ast - extra = ast.literal_eval(extra) assert isinstance(extra, dict) + return extra + + @property + def pass_config(self): + base = {"tir.disable_vectorize": self.disable_vectorize} + extra = self.extra_pass_config base.update(extra) return base diff --git a/mlonmcu/flow/tvm/backend/tvmaot.py b/mlonmcu/flow/tvm/backend/tvmaot.py index 77832432..bb5802ca 100644 --- a/mlonmcu/flow/tvm/backend/tvmaot.py +++ b/mlonmcu/flow/tvm/backend/tvmaot.py @@ -43,9 +43,6 @@ class TVMAOTBackend(TVMBackend): "arena_size": None, # Determined automatically "unpacked_api": False, "alignment_bytes": 16, - "extra_pass_config": { - "tir.usmp.enable": False, - }, } name = "tvmaot" @@ -55,6 +52,13 @@ def __init__(self, runtime="crt", fmt="mlf", system_lib=False, features=None, co executor="aot", runtime=runtime, fmt=fmt, system_lib=system_lib, features=features, config=config ) + @property + def extra_pass_config(self): + default = {"tir.usmp.enable": False} + extra = super().extra_pass_config + extra.update(default) + return extra + @property def arena_size(self): size = self.config["arena_size"] From 2dc481cf14ae9e9aa0df10e053623cff180a5adc Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sat, 27 Jul 2024 18:52:58 +0200 Subject: [PATCH 3/8] tvm backends: add assertion if input tensors list is empty --- mlonmcu/flow/tvm/backend/backend.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mlonmcu/flow/tvm/backend/backend.py b/mlonmcu/flow/tvm/backend/backend.py index 6d7bafe9..47904aad 100644 --- a/mlonmcu/flow/tvm/backend/backend.py +++ b/mlonmcu/flow/tvm/backend/backend.py @@ -464,6 +464,9 @@ def load_model(self, model, input_shapes=None, output_shapes=None, input_types=N # TODO: take care of refresh_model_info if self.input_shapes: self.model_info.in_tensors = [t for t in self.model_info.in_tensors if t.name in self.input_shapes] + assert ( + len(self.model_info.in_tensors) > 0 + ), "Missmatch between provided input names and detected ones" else: self.input_shapes = {tensor.name: tensor.shape for tensor in self.model_info.in_tensors} if self.model_info: From f248808904bee76816fdde7e264d8d0540077fbd Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sat, 27 Jul 2024 18:53:33 +0200 Subject: [PATCH 4/8] wrapper.py: add workaround for output tensors names in relax mode --- mlonmcu/flow/tvm/backend/tvmaot.py | 1 + mlonmcu/flow/tvm/backend/wrapper.py | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/mlonmcu/flow/tvm/backend/tvmaot.py b/mlonmcu/flow/tvm/backend/tvmaot.py index bb5802ca..d663eb00 100644 --- a/mlonmcu/flow/tvm/backend/tvmaot.py +++ b/mlonmcu/flow/tvm/backend/tvmaot.py @@ -118,6 +118,7 @@ def generate(self) -> Tuple[dict, dict]: self.prefix, api="c" if self.unpacked_api else "packed", debug_arena=self.debug_arena, + relax_mode=self.relax_mode, ) artifacts.append(Artifact("aot_wrapper.c", content=wrapper_src, fmt=ArtifactFormat.SOURCE)) header_src = generate_wrapper_header() diff --git a/mlonmcu/flow/tvm/backend/wrapper.py b/mlonmcu/flow/tvm/backend/wrapper.py index 17c4026e..21e46097 100644 --- a/mlonmcu/flow/tvm/backend/wrapper.py +++ b/mlonmcu/flow/tvm/backend/wrapper.py @@ -382,7 +382,7 @@ def getMeta(tensors, withNames=False): return out -def generate_tvmaot_wrapper(model_info, workspace_size, mod_name, api="c", debug_arena=False): +def generate_tvmaot_wrapper(model_info, workspace_size, mod_name, api="c", debug_arena=False, relax_mode=False): modPrefix = f"tvmgen_{mod_name}" def writeTensors(in_tensors, out_tensors, modPrefix, api): @@ -391,7 +391,7 @@ def writeTensors(in_tensors, out_tensors, modPrefix, api): // Define data for input and output tensors """ - def writeTensorsHelper(tensors, prefix, out=False): + def writeTensorsHelper(tensors, prefix, out=False, relax_mode=False): lenTensors = len(tensors) direction = "out" if out else "in" ret = "" @@ -401,13 +401,19 @@ def writeTensorsHelper(tensors, prefix, out=False): ret += f"void* {direction}puts[] = {{" + ", ".join(names) + "};\n" ret += f"struct {prefix}_{direction}puts {prefix}_{direction}puts = {{" + "\n" for i, t in enumerate(tensors): - tensor_name = t.name.replace(":", "_").replace("/", "_").replace(".", "_").replace(";", "_") - ret += f" .{tensor_name} = {names[i]}," + "\n" + if not relax_mode or not out: + tensor_name = t.name.replace(":", "_").replace("/", "_").replace(".", "_").replace(";", "_") + ret += f" .{tensor_name} = {names[i]}," + "\n" + else: + if i > 0: + ret += f" .{direction}put{i} = {names[i]}," + "\n" + else: + ret += f" .{direction}put = {names[i]}," + "\n" ret += "};\n" return ret - retStr += writeTensorsHelper(in_tensors, modPrefix, False) - retStr += writeTensorsHelper(out_tensors, modPrefix, True) + retStr += writeTensorsHelper(in_tensors, modPrefix, False, relax_mode=relax_mode) + retStr += writeTensorsHelper(out_tensors, modPrefix, True, relax_mode=relax_mode) return retStr elif api == "packed": retStr = """ From c44e9e8a40a090c65a5c41377034f2e1dc28bd89 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sat, 27 Jul 2024 18:57:24 +0200 Subject: [PATCH 5/8] mlonmcu/setup/utils.py: add comment --- mlonmcu/setup/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mlonmcu/setup/utils.py b/mlonmcu/setup/utils.py index 200f8d46..fbcd2f05 100644 --- a/mlonmcu/setup/utils.py +++ b/mlonmcu/setup/utils.py @@ -226,6 +226,8 @@ def execute( logger.debug("- Executing: %s", str(args)) if "cwd" in kwargs: logger.debug("- CWD: %s", str(kwargs["cwd"])) + # if "env" in kwargs: + # logger.debug("- ENV: %s", str(kwargs["env"])) if ignore_output: assert not live subprocess.run(args, **kwargs, check=True) From e7efda7267f4221fe1738e78b6de1b2041635ef2 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 9 Sep 2024 09:03:18 +0200 Subject: [PATCH 6/8] mlonmcu/flow/tvm/backend/model_info.py: fix fallback output name --- mlonmcu/flow/tvm/backend/model_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlonmcu/flow/tvm/backend/model_info.py b/mlonmcu/flow/tvm/backend/model_info.py index c17ac8c5..6f8e57f1 100644 --- a/mlonmcu/flow/tvm/backend/model_info.py +++ b/mlonmcu/flow/tvm/backend/model_info.py @@ -154,7 +154,7 @@ def parse_relay_main(line): if len(output_tensor_names_str) > 0: output_tensor_names = re.compile(r"\"([a-zA-Z0-9_]+)\"").findall(output_tensor_names_str[0]) else: - output_tensor_names = [f"output{i}" for i in range(len(output_tensor_strs))] + output_tensor_names = [(f"output{i}" if i > 0 else "output") for i in range(len(output_tensor_strs))] assert len(output_tensor_names) == len(output_tensor_strs) From 2a50c8997ee4b1192ca34881656a9478169fedd3 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Mon, 9 Sep 2024 09:03:52 +0200 Subject: [PATCH 7/8] mlonmcu/flow/tvm/backend/wrapper.py: no relax mode needed? --- mlonmcu/flow/tvm/backend/wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlonmcu/flow/tvm/backend/wrapper.py b/mlonmcu/flow/tvm/backend/wrapper.py index 21e46097..ab8c26a5 100644 --- a/mlonmcu/flow/tvm/backend/wrapper.py +++ b/mlonmcu/flow/tvm/backend/wrapper.py @@ -401,7 +401,7 @@ def writeTensorsHelper(tensors, prefix, out=False, relax_mode=False): ret += f"void* {direction}puts[] = {{" + ", ".join(names) + "};\n" ret += f"struct {prefix}_{direction}puts {prefix}_{direction}puts = {{" + "\n" for i, t in enumerate(tensors): - if not relax_mode or not out: + if True: tensor_name = t.name.replace(":", "_").replace("/", "_").replace(".", "_").replace(";", "_") ret += f" .{tensor_name} = {names[i]}," + "\n" else: From eed2545afb068ad7b2bfd2ac0b8b69b88546981e Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Wed, 11 Sep 2024 09:51:07 +0200 Subject: [PATCH 8/8] wrapper.py: fix relax moe --- mlonmcu/flow/tvm/backend/wrapper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mlonmcu/flow/tvm/backend/wrapper.py b/mlonmcu/flow/tvm/backend/wrapper.py index 43ffd5b4..98763156 100644 --- a/mlonmcu/flow/tvm/backend/wrapper.py +++ b/mlonmcu/flow/tvm/backend/wrapper.py @@ -402,7 +402,10 @@ def writeTensorsHelper(tensors, prefix, out=False, relax_mode=False): ret += f"void* {direction}puts[] = {{" + ", ".join(names) + "};\n" ret += f"struct {prefix}_{direction}puts {prefix}_{direction}puts = {{" + "\n" for i, t in enumerate(tensors): - if True: + # if True: + # if not relax_mode: + # if not out: + if (relax_mode and not out) or not relax_mode: tensor_name = t.name.replace(":", "_").replace("/", "_").replace(".", "_").replace(";", "_") ret += f" .{tensor_name} = {names[i]}," + "\n" else: