[wip] JSON IDL V2 #2741

Future-Outlier · 2024-09-10T08:04:54Z

Flyte Backend

flyteorg/flyte#5735

Tracking issue

flyteorg/flyte#5318

Why are the changes needed?

What changes were proposed in this pull request?

The attribute access roadmap:

How was this patch tested?

List and nested List

code

import typing
from dataclasses import dataclass, fields, field
from typing import Dict
from flytekit import task, workflow, ImageSpec


flytekit_hash = "313dc8f43897a5fc597493df348fa1bdc63a1518"
flyteidl_hash = "2c9cbd89dce53acd1cb90cc042012637ae6c31f8"

json_idl = f"git+https://github.com/flyteorg/flyte.git@{flyteidl_hash}#subdirectory=flyteidl"
flytekit = f"git+https://github.com/flyteorg/flytekit.git@{flytekit_hash}"

image = ImageSpec(
    packages=[json_idl, flytekit],
    apt_packages=["git"],
    registry="localhost:30000",
)

@dataclass
class DC:
    a: int = 1
    b: float = 2.0
    c: str = "string"
    d: bool = True
    list_int: typing.List[int] = field(default_factory=lambda: [1, 2, 3])
    list_float: typing.List[float] = field(default_factory=lambda: [1.0, 2.0, 3.0])
    list_str: typing.List[str] = field(default_factory=lambda: ["a", "b", "c"])
    list_bool: typing.List[bool] = field(default_factory=lambda: [True, False, True])
    e: typing.List[typing.List[int]] = field(default_factory=lambda: [[1, 2], [3, 4]])
    f: typing.List[typing.List[float]] = field(default_factory=lambda: [[1.0, 2.0], [3.0, 4.0]])
    g: typing.List[typing.List[str]] = field(default_factory=lambda: [["a", "b"], ["c", "d"]])
    h: typing.List[typing.List[bool]] = field(default_factory=lambda: [[True, False], [False, True]])
    i: typing.List[Dict[str, int]] = field(default_factory=lambda: [{"key1": 1, "key2": 2}])
    j: typing.List[Dict[str, float]] = field(default_factory=lambda: [{"key1": 1.0, "key2": 2.0}])
    k: typing.List[Dict[str, str]] = field(default_factory=lambda: [{"key1": "a", "key2": "b"}])
    l: typing.List[Dict[str, bool]] = field(default_factory=lambda: [{"key1": True, "key2": False}])

@task(container_image=image)
def t_int(a: int):
    assert isinstance(a, int), f"Expected type int, but got {type(a)}"
    print("t_int:", a, type(a))

@task(container_image=image)
def t_float(b: float):
    assert isinstance(b, float), f"Expected type float, but got {type(b)}"
    print("t_float:", b, type(b))

@task(container_image=image)
def t_str(c: str):
    assert isinstance(c, str), f"Expected type str, but got {type(c)}"
    print("t_str:", c, type(c))

@task(container_image=image)
def t_bool(d: bool):
    assert isinstance(d, bool), f"Expected type bool, but got {type(d)}"
    print("t_bool:", d, type(d))

# Tasks to handle List[int], List[float], List[str], and List[bool]
@task(container_image=image)
def t_list_int(list_int: typing.List[int]):
    assert isinstance(list_int, list), f"Expected type list[int], but got {type(list_int)}"
    for idx, val in enumerate(list_int):
        assert isinstance(val, int), f"Expected type int in list, but got {type(val)} at index {idx}"
    print("t_list_int:", list_int)

@task(container_image=image)
def t_list_float(list_float: typing.List[float]):
    assert isinstance(list_float, list), f"Expected type list[float], but got {type(list_float)}"
    for idx, val in enumerate(list_float):
        assert isinstance(val, float), f"Expected type float in list, but got {type(val)} at index {idx}"
    print("t_list_float:", list_float)

@task(container_image=image)
def t_list_str(list_str: typing.List[str]):
    assert isinstance(list_str, list), f"Expected type list[str], but got {type(list_str)}"
    for idx, val in enumerate(list_str):
        assert isinstance(val, str), f"Expected type str in list, but got {type(val)} at index {idx}"
    print("t_list_str:", list_str)

@task(container_image=image)
def t_list_bool(list_bool: typing.List[bool]):
    assert isinstance(list_bool, list), f"Expected type list[bool], but got {type(list_bool)}"
    for idx, val in enumerate(list_bool):
        assert isinstance(val, bool), f"Expected type bool in list, but got {type(val)} at index {idx}"
    print("t_list_bool:", list_bool)

# Tasks to handle nested list of int
@task(container_image=image)
def t_list_list_int(e: typing.List[typing.List[int]]):
    assert isinstance(e, list), f"Expected type list[list[int]], but got {type(e)}"
    for idx, sublist in enumerate(e):
        assert isinstance(sublist, list), f"Expected type list at index {idx}, but got {type(sublist)}"
        for i, val in enumerate(sublist):
            assert isinstance(val, int), f"Expected type int in sublist, but got {type(val)} at index {i}"
    print("t_list_list_int:", e)

# Tasks to handle list of dicts with various types
@task(container_image=image)
def t_list_dict_str_int(i: typing.List[Dict[str, int]]):
    assert isinstance(i, list), f"Expected type list[dict[str, int]], but got {type(i)}"
    for idx, d in enumerate(i):
        assert isinstance(d, dict), f"Expected type dict at index {idx}, but got {type(d)}"
        for k, v in d.items():
            assert isinstance(k, str), f"Expected key type str, but got {type(k)}"
            assert isinstance(v, int), f"Expected value type int, but got {type(v)}"
    print("t_list_dict_str_int:", i)

@task(container_image=image)
def t_list_dict_str_float(j: typing.List[Dict[str, float]]):
    assert isinstance(j, list), f"Expected type list[dict[str, float]], but got {type(j)}"
    for idx, d in enumerate(j):
        assert isinstance(d, dict), f"Expected type dict at index {idx}, but got {type(d)}"
        for k, v in d.items():
            assert isinstance(k, str), f"Expected key type str, but got {type(k)}"
            assert isinstance(v, float), f"Expected value type float, but got {type(v)}"
    print("t_list_dict_str_float:", j)

@task(container_image=image)
def t_list_dict_str_str(k: typing.List[Dict[str, str]]):
    assert isinstance(k, list), f"Expected type list[dict[str, str]], but got {type(k)}"
    for idx, d in enumerate(k):
        assert isinstance(d, dict), f"Expected type dict at index {idx}, but got {type(d)}"
        for k, v in d.items():
            assert isinstance(k, str), f"Expected key type str, but got {type(k)}"
            assert isinstance(v, str), f"Expected value type str, but got {type(v)}"
    print("t_list_dict_str_str:", k)

@task(container_image=image)
def t_list_dict_str_bool(l: typing.List[Dict[str, bool]]):
    assert isinstance(l, list), f"Expected type list[dict[str, bool]], but got {type(l)}"
    for idx, d in enumerate(l):
        assert isinstance(d, dict), f"Expected type dict at index {idx}, but got {type(d)}"
        for k, v in d.items():
            assert isinstance(k, str), f"Expected key type str, but got {type(k)}"
            assert isinstance(v, bool), f"Expected value type bool, but got {type(v)}"
    print("t_list_dict_str_bool:", l)


@workflow
def dataclass_wf(input: DC) -> DC:
    t_int(a=input.a)
    t_float(b=input.b)
    t_str(c=input.c)
    t_bool(d=input.d)
    t_list_int(list_int=input.list_int)
    t_list_float(list_float=input.list_float)
    t_list_str(list_str=input.list_str)
    t_list_bool(list_bool=input.list_bool)
    t_list_list_int(e=input.e)
    t_list_dict_str_int(i=input.i)
    t_list_dict_str_float(j=input.j)
    t_list_dict_str_str(k=input.k)
    t_list_dict_str_bool(l=input.l)
    return input

if __name__ == "__main__":
    from flytekit.clis.sdk_in_container import pyflyte
    from click.testing import CliRunner

    runner = CliRunner()
    path = "/Users/future-outlier/code/dev/flytekit/build/PR/JSON/demo/dataclass_simple_list_1_and_2_level.py"
    result = runner.invoke(pyflyte.main, ["run", path, "dataclass_wf", "--input", '{"a": 1}'])

    print("Local Execution: ", result.output)

    result = runner.invoke(pyflyte.main, ["run", "--remote", path, "dataclass_wf", "--input", '{"a": 1}'])
    print("Remote Execution: ", result.output)

Dict Transformer

code

import typing
from dataclasses import dataclass, fields, field
from typing import Dict
from flytekit import task, workflow, ImageSpec


flytekit_hash = "717104edde0cb90de0a2ade6a51a9e4d9b75635c"
flyteidl_hash = "dd21c31811ae6cc1ab93259defdd6dbff57a6102"

json_idl = f"git+https://github.com/flyteorg/flyte.git@{flyteidl_hash}#subdirectory=flyteidl"
flytekit = f"git+https://github.com/flyteorg/flytekit.git@{flytekit_hash}"

image = ImageSpec(
    packages=[json_idl, flytekit],
    apt_packages=["git"],
    registry="localhost:30000",
)

@dataclass
class DC:
    a: int = 1
    b: float = 2.0
    c: str = "string"
    d: bool = True
    e: Dict[str, int] = field(default_factory=lambda: {"key1": 1, "key2": 2})
    f: Dict[str, float] = field(default_factory=lambda: {"key1": 1.0, "key2": 2.0})
    g: Dict[str, str] = field(default_factory=lambda: {"key1": "a", "key2": "b"})
    h: Dict[str, bool] = field(default_factory=lambda: {"key1": True, "key2": False})
    i: dict = field(default_factory=lambda: {"key1": 1, "key2": 2})


@task(container_image=image)
def t_int(a: int):
    assert isinstance(a, int), f"Expected type int, but got {type(a)}"
    print("t_int:", a, type(a))

@task(container_image=image)
def t_float(b: float):
    assert isinstance(b, float), f"Expected type float, but got {type(b)}"
    print("t_float:", b, type(b))

@task(container_image=image)
def t_str(c: str):
    assert isinstance(c, str), f"Expected type str, but got {type(c)}"
    print("t_str:", c, type(c))

@task(container_image=image)
def t_bool(d: bool):
    assert isinstance(d, bool), f"Expected type bool, but got {type(d)}"
    print("t_bool:", d, type(d))

# Tasks to handle Dict[str, int], Dict[str, float], Dict[str, str], and Dict[str, bool]
@task(container_image=image)
def t_dict_int(e: Dict[str, int]):
    assert isinstance(e, dict), f"Expected type dict, but got {type(e)}"
    print("t_dict_int:", e, type(e))

@task(container_image=image)
def t_dict_float(f: Dict[str, float]):
    assert isinstance(f, dict), f"Expected type dict, but got {type(f)}"
    print("t_dict_float:", f, type(f))

@task(container_image=image)
def t_dict_str(g: Dict[str, str]):
    assert isinstance(g, dict), f"Expected type dict, but got {type(g)}"
    print("t_dict_str:", g, type(g))

@task(container_image=image)
def t_dict_bool(h: Dict[str, bool]):
    assert isinstance(h, dict), f"Expected type dict, but got {type(h)}"
    print("t_dict_bool:", h, type(h))

@task(container_image=image)
def t_dict_any(i: dict):
    assert isinstance(i, dict), f"Expected type dict, but got {type(i)}"
    print("t_dict_any:", i, type(i))

@workflow
def dataclass_wf(input: DC) -> DC:
    t_int(a=input.a)
    t_float(b=input.b)
    t_str(c=input.c)
    t_bool(d=input.d)
    t_dict_int(e=input.e)
    t_dict_float(f=input.f)
    t_dict_str(g=input.g)
    t_dict_bool(h=input.h)
    t_dict_any(i=input.i)
    return input

if __name__ == "__main__":
    from flytekit.clis.sdk_in_container import pyflyte
    from click.testing import CliRunner

    runner = CliRunner()
    path = "/Users/future-outlier/code/dev/flytekit/build/PR/JSON/demo/dataclass_simple_dict_1_and_2_level.py"

    result = runner.invoke(pyflyte.main, ["run", path, "dataclass_wf", "--input", '{"a": 1}'])

    print("Local Execution: ", result.output)

    result = runner.invoke(pyflyte.main, ["run", "--remote", path, "dataclass_wf", "--input", '{"a": 1}'])
    print("Remote Execution: ", result.output)

Dataclass Transformer

code

import typing
from dataclasses import dataclass, field
from typing import Dict, List
from flytekit import task, workflow, ImageSpec

flytekit_hash = "717104edde0cb90de0a2ade6a51a9e4d9b75635c"
flyteidl_hash = "dd21c31811ae6cc1ab93259defdd6dbff57a6102"

json_idl = f"git+https://github.com/flyteorg/flyte.git@{flyteidl_hash}#subdirectory=flyteidl"
flytekit = f"git+https://github.com/flyteorg/flytekit.git@{flytekit_hash}"

image = ImageSpec(
    packages=[json_idl, flytekit],
    apt_packages=["git"],
    registry="localhost:30000",
)

@dataclass
class InnerDC:
    x: int = 1
    y: float = 2.0
    z: str = "inner_string"

@dataclass
class MiddleDC:
    inner: InnerDC = field(default_factory=InnerDC)
    b_list: List[InnerDC] = field(default_factory=lambda: [InnerDC(), InnerDC()])
    c_dict: Dict[str, InnerDC] = field(default_factory=lambda: {"key1": InnerDC(), "key2": InnerDC()})

@dataclass
class OuterDC:
    middle: MiddleDC = field(default_factory=MiddleDC)
    list_of_lists: List[List[InnerDC]] = field(default_factory=lambda: [[InnerDC(), InnerDC()], [InnerDC()]])
    dict_of_dicts: Dict[str, Dict[str, InnerDC]] = field(default_factory=lambda: {"key1": {"subkey1": InnerDC()}})
    dict_of_lists: Dict[str, List[InnerDC]] = field(default_factory=lambda: {"key1": [InnerDC(), InnerDC()]})
    list_of_dicts: List[Dict[str, InnerDC]] = field(
        default_factory=lambda: [{"key1": InnerDC(), "key2": InnerDC()}])

@task(container_image=image)
def t_x(x: int):
    assert isinstance(x, int), f"Expected int, but got {type(x)}"
    print(f"x: {x}")
    print("================")

@task(container_image=image)
def t_y(y: float):
    assert isinstance(y, float), f"Expected float, but got {type(y)}"
    print(f"y: {y}")
    print("================")

@task(container_image=image)
def t_middle(middle: MiddleDC):
    assert isinstance(middle, MiddleDC), f"Expected MiddleDC, but got {type(middle)}"
    print(f"middle.inner.x: {middle.inner.x}, middle.a.y: {middle.inner.y}")
    print(f"middle.b_list: {middle.b_list}")
    print(f"middle.c_dict: {middle.c_dict}")
    print("================")

@task(container_image=image)
def t_inner(inner: InnerDC):
    assert isinstance(inner, InnerDC), f"Expected InnerDC, but got {type(inner)}"
    print(f"inner.x: {inner.x}, inner.y: {inner.y}, inner.z: {inner.z}")
    print("================")

@task(container_image=image)
def t_list_of_dc(list_of_dc: List[InnerDC]):
    for i, inner in enumerate(list_of_dc):
        assert isinstance(inner, InnerDC), f"Expected InnerDC, but got {type(inner)}"
        print(f"list_of_dc[{i}] - x: {inner.x}, y: {inner.y}, z: {inner.z}")
    print("================")

@task(container_image=image)
def t_list_of_lists(list_of_lists: List[List[InnerDC]]):
    for i, inner_list in enumerate(list_of_lists):
        for j, inner in enumerate(inner_list):
            assert isinstance(inner, InnerDC), f"Expected InnerDC, but got {type(inner)}"
            print(f"list_of_lists[{i}][{j}] - x: {inner.x}, y: {inner.y}, z: {inner.z}")
    print("================")

@task(container_image=image)
def create_outer_dc() -> OuterDC:
    return OuterDC()

@workflow
def dataclass_wf() -> OuterDC:
    input = create_outer_dc()
    t_x(x=input.middle.inner.x)
    t_y(y=input.middle.inner.y)
    t_middle(middle=input.middle)
    t_inner(inner=input.middle.inner)
    t_list_of_dc(list_of_dc=input.list_of_lists[0])
    t_list_of_lists(list_of_lists=input.list_of_lists)
    t_x(x=input.list_of_lists[0][0].x)
    t_x(x=input.list_of_lists[0][1].x)
    t_x(x=input.dict_of_dicts["key1"]["subkey1"].x)

    return input

if __name__ == "__main__":
    from flytekit.clis.sdk_in_container import pyflyte
    from click.testing import CliRunner

    runner = CliRunner()
    path = "/Users/future-outlier/code/dev/flytekit/build/PR/JSON/demo/dataclasss_simple_dataclass_1_and_2_level.py"

    # result = runner.invoke(pyflyte.main, ["run", path, "dataclass_wf", "--input", '{}'])
    result = runner.invoke(pyflyte.main, ["run", path, "dataclass_wf"])
    print("Local Execution: ", result.output)
    print("================")
    #
    result = runner.invoke(pyflyte.main, ["run", "--remote", path, "dataclass_wf"])
    print("Remote Execution: ", result.output)

Setup process

Screenshots

List Transformer

Dict Transformer

Dataclass Transformer

Check all the applicable boxes

I updated the documentation accordingly.
All new and existing tests passed.
All commits are signed-off.

Related PRs

Docs link

Signed-off-by: Future-Outlier <[email protected]>

Future-Outlier · 2024-09-10T08:09:16Z

flytekit/core/type_engine.py

+    def to_json(self, ctx: FlyteContext, python_val: T, python_type: Type[T], expected: LiteralType) -> Literal:
+        if isinstance(python_val, dict):
+            json_str = json.dumps(python_val)
+            json_bytes = json_str.encode("UTF-8")
+            return Literal(scalar=Scalar(json=Json(value=json_bytes, serialization_format="UTF-8")))
+
+        if not dataclasses.is_dataclass(python_val):


I don't want to reuse the code in to_literal and to_python_val because I think this will be far more readable and more easier to customize behavior in the future when we want to have more flexible change to the JSON IDL object.

I haven't come up with the scenario, but this is just my instinct.

pingsutw · 2024-09-10T08:10:00Z

flytekit/core/type_engine.py

+        serialization_format = json_idl_object.serialization_format
+        json_str = None
+        if serialization_format == "UTF-8":
+            json_str = value.decode("UTF-8")
+        if json_str is None:
+            json_str = value.decode("UTF-8") # default to UTF-8


Suggested change

serialization_format = json_idl_object.serialization_format

json_str = None

if serialization_format == "UTF-8":

json_str = value.decode("UTF-8")

if json_str is None:

json_str = value.decode("UTF-8") # default to UTF-8

serialization_format = json_idl_object.serialization_format or "UTF-8"

json_str = None

if serialization_format == "UTF-8":

json_str = value.decode("UTF-8")

else:

raise ValueError(...)

will do this later, thank you.

Signed-off-by: Future-Outlier <[email protected]> Co-authored-by: pingsutw <[email protected]>

codecov · 2024-09-10T08:55:25Z

Codecov Report

Attention: Patch coverage is 35.60209% with 123 lines in your changes missing coverage. Please review.

Project coverage is 45.20%. Comparing base (15d82ef) to head (b1e4c66).
Report is 2 commits behind head on master.

Files with missing lines	Patch %	Lines
flytekit/core/type_engine.py	33.09%	82 Missing and 11 partials ⚠️
flytekit/core/promise.py	13.79%	24 Missing and 1 partial ⚠️
flytekit/interaction/string_literals.py	0.00%	2 Missing ⚠️
flytekit/models/literals.py	89.47%	2 Missing ⚠️
flytekit/interaction/click_types.py	0.00%	1 Missing ⚠️

❗ There is a different number of reports uploaded between BASE (15d82ef) and HEAD (b1e4c66). Click for more details.

HEAD has 8 uploads less than BASE

Flag BASE (15d82ef) HEAD (b1e4c66)

10 2

Additional details and impacted files

@@             Coverage Diff             @@
##           master    #2741       +/-   ##
===========================================
- Coverage   78.61%   45.20%   -33.42%     
===========================================
  Files         193      194        +1     
  Lines       19691    19850      +159     
  Branches     4101     2887     -1214     
===========================================
- Hits        15481     8973     -6508     
- Misses       3492    10424     +6932     
+ Partials      718      453      -265

☔ View full report in Codecov by Sentry.
📢 Have feedback on the report? Share it here.

Signed-off-by: Future-Outlier <[email protected]>

Future-Outlier · 2024-09-10T10:12:27Z

flytekit/core/type_engine.py

+        else:
+            raise ValueError(
+                f"Bytes can't be converted to JSON String.\n"
+                f"Unsupported serialization format: {serialization_format}"
+            )
+        return json.loads(json_str)


Questoin: Should we assert expected_python_type == type(json.loads(json_str))?

Signed-off-by: Future-Outlier <[email protected]>

Future-Outlier · 2024-09-10T10:39:04Z

flytekit/core/type_engine.py

+        python_val =  json.loads(json_str)
+        expected_python_val = expected_python_type(python_val)


In golang, if we use json.Unmarshal, it will discard .0, which will make json.loads(float) to int, if the value is x.0 (for example 2.0).

Signed-off-by: Future-Outlier <[email protected]>

Future-Outlier · 2024-09-10T17:20:28Z

flytekit/core/type_engine.py

+    def from_json(self, ctx: FlyteContext, json_idl_object: Json, expected_python_type: Type[T]) -> typing.List[T]:
+        """
+        Process JSON IDL object and convert it to the corresponding Python value.
+        Handles both simple types and recursive structures like List[List[int]] or List[List[float]].
+        """
+
+        def recursive_from_json(ctx: FlyteContext, json_value: typing.Any, expected_python_type: Type[T]) -> typing.Any:
+            """
+            Recursively process JSON objects, converting them to their corresponding Python values based on
+            the expected Python type (e.g., handling List[List[int]] or List[List[float]]).
+            """
+            # Check if the type is a List
+            if typing.get_origin(expected_python_type) is list:
+                # Get the subtype, which should be the type of the list's elements
+                sub_type = self.get_sub_type(expected_python_type)
+                # Recursively process each element in the list
+                return [recursive_from_json(ctx, item, sub_type) for item in json_value]
+
+            # Check if the type is a Dict
+            elif typing.get_origin(expected_python_type) is dict:
+                # For Dicts, get key and value types
+                key_type, val_type = typing.get_args(expected_python_type)
+                # Recursively process each key and value in the dict
+                return {recursive_from_json(ctx, k, key_type): recursive_from_json(ctx, v, val_type) for k, v in
+                        json_value.items()}
+
+            # Base case: if it's not a list or dict, we assume it's a simple type and return it
+            try:
+                return expected_python_type(json_value)  # Cast to the expected type
+            except Exception as e:
+                raise ValueError(f"Could not cast {json_value} to {expected_python_type}: {e}")
+
+        # Handle the serialization format
+        value = json_idl_object.value
+        serialization_format = json_idl_object.serialization_format
+        if serialization_format == "UTF-8":
+            # Decode JSON string
+            json_value = json.loads(value.decode("utf-8"))
+        else:
+            raise ValueError(f"Unknown serialization format {serialization_format}")
+
+        # Call the recursive function to handle nested structures
+        return recursive_from_json(ctx, json_value, expected_python_type)
+
    def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> typing.List[typing.Any]:  # type: ignore
+        scalar = lv.scalar
+        if scalar and scalar.json:
+            return self.from_json(ctx, scalar.json, expected_python_type)


Hi, @wild-endeavor
Does this look good to you?
propeller will return json idl object to the List Transformer, and we will handle it recursively.

propeller get the list json str: https://github.com/flyteorg/flyte/pull/5735/files#diff-ee7f936e440a7e043b3bc7acb4ea255ba991dea8f3144d24ab276c3a292de018R103-R113

Signed-off-by: Future-Outlier <[email protected]>

…propeller and get literal type error in dataclass (needs to use type hints) Signed-off-by: Future-Outlier <[email protected]>

Future-Outlier · 2024-09-11T14:57:26Z

flytekit/core/type_engine.py

+        hints = get_type_hints(t)
        # Get the type of each field from dataclass
        for field in t.__dataclass_fields__.values():  # type: ignore
            try:
-                literal_type[field.name] = TypeEngine.to_literal_type(field.type)
+                name = field.name
+                python_type = hints.get(field.name, field.type)
+                literal_type[name] = TypeEngine.to_literal_type(python_type)


This is for issue here: flyteorg/flyte#5427
If we use t.__dataclass_fields__.values() only, we will set sub_fruit as str, but not JSON.

…need that Signed-off-by: Future-Outlier <[email protected]>

JSON IDL V2

158ae01

Signed-off-by: Future-Outlier <[email protected]>

Future-Outlier requested review from wild-endeavor, kumare3, eapolinario, pingsutw, cosmicBboy, samhita-alla and thomasjpfan as code owners September 10, 2024 08:04

Future-Outlier commented Sep 10, 2024

View reviewed changes

pingsutw reviewed Sep 10, 2024

View reviewed changes

Update kevin's advice

d31a82d

Signed-off-by: Future-Outlier <[email protected]> Co-authored-by: pingsutw <[email protected]>

Future-Outlier added 4 commits September 10, 2024 16:58

print json idl and serialization format, debuggin flytefile in it

0adf1ae

Signed-off-by: Future-Outlier <[email protected]>

print json str

b7e2bdb

Signed-off-by: Future-Outlier <[email protected]>

support simpleTransformer from_json

dcf9b4b

Signed-off-by: Future-Outlier <[email protected]>

local execution with attribute access

24fe614

Signed-off-by: Future-Outlier <[email protected]>

Future-Outlier commented Sep 10, 2024

View reviewed changes

debuggin deserialization for simple type

70e298c

Signed-off-by: Future-Outlier <[email protected]>

Future-Outlier commented Sep 10, 2024

View reviewed changes

Future-Outlier added 5 commits September 10, 2024 18:40

remove @@@

848bd4b

Signed-off-by: Future-Outlier <[email protected]>

add type ignore for lint

b145637

Signed-off-by: Future-Outlier <[email protected]>

recursive construct List type

e42b667

Signed-off-by: Future-Outlier <[email protected]>

fix bug

1782dab

Signed-off-by: Future-Outlier <[email protected]>

better recursive function

313dc8f

Signed-off-by: Future-Outlier <[email protected]>

Future-Outlier commented Sep 10, 2024

View reviewed changes

Future-Outlier added 3 commits September 11, 2024 11:49

dict iterative

717104e

Signed-off-by: Future-Outlier <[email protected]>

fix promise resolve test

69d9e8b

Signed-off-by: Future-Outlier <[email protected]>

fix promise attribute access, make it's behavior exactly the same as …

b1e4c66

…propeller and get literal type error in dataclass (needs to use type hints) Signed-off-by: Future-Outlier <[email protected]>

Future-Outlier commented Sep 11, 2024

View reviewed changes

remove type in resolve_attr_path_in_promise, since propeller doesn't …

ea45e1e

…need that Signed-off-by: Future-Outlier <[email protected]>

Future-Outlier closed this Sep 14, 2024

Future-Outlier mentioned this pull request Sep 11, 2024

[wip] JSON IDL V2 flyteorg/flyte#5735

Closed

3 tasks

Future-Outlier mentioned this pull request Sep 24, 2024

[flytepropeller][flyteadmin] Compiler unknown literal type error handling flyteorg/flyte#5651

Merged

3 tasks

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[wip] JSON IDL V2 #2741

[wip] JSON IDL V2 #2741

Future-Outlier commented Sep 10, 2024 •

edited

Loading

Future-Outlier Sep 10, 2024 •

edited

Loading

Future-Outlier Sep 10, 2024

pingsutw Sep 10, 2024

Future-Outlier Sep 10, 2024

codecov bot commented Sep 10, 2024 •

edited

Loading

Future-Outlier Sep 10, 2024

Future-Outlier Sep 10, 2024

Future-Outlier Sep 10, 2024

Future-Outlier Sep 11, 2024

		python_val = json.loads(json_str)
		expected_python_val = expected_python_type(python_val)

[wip] JSON IDL V2 #2741

[wip] JSON IDL V2 #2741

Conversation

Future-Outlier commented Sep 10, 2024 • edited Loading

Flyte Backend

Tracking issue

Why are the changes needed?

What changes were proposed in this pull request?

How was this patch tested?

List and nested List

Dict Transformer

Dataclass Transformer

Setup process

Screenshots

List Transformer

Dict Transformer

Dataclass Transformer

Check all the applicable boxes

Related PRs

Docs link

Future-Outlier Sep 10, 2024 • edited Loading

Choose a reason for hiding this comment

Future-Outlier Sep 10, 2024

Choose a reason for hiding this comment

pingsutw Sep 10, 2024

Choose a reason for hiding this comment

Future-Outlier Sep 10, 2024

Choose a reason for hiding this comment

codecov bot commented Sep 10, 2024 • edited Loading

Codecov Report

Future-Outlier Sep 10, 2024

Choose a reason for hiding this comment

Future-Outlier Sep 10, 2024

Choose a reason for hiding this comment

Future-Outlier Sep 10, 2024

Choose a reason for hiding this comment

Future-Outlier Sep 11, 2024

Choose a reason for hiding this comment

Future-Outlier commented Sep 10, 2024 •

edited

Loading

Future-Outlier Sep 10, 2024 •

edited

Loading

codecov bot commented Sep 10, 2024 •

edited

Loading