diff --git a/Cargo.toml b/Cargo.toml index 0d33d453..191ee953 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,9 +34,7 @@ members = [ # tools "tools/phoenix_cargo", ] -exclude = [ - "experimental/mrpc", -] +exclude = ["experimental/mrpc"] [workspace.dependencies] phoenix-api = { path = "src/phoenix-api" } diff --git a/eval/policy/delay/attach.toml b/eval/policy/delay/attach.toml new file mode 100644 index 00000000..6d86f6f0 --- /dev/null +++ b/eval/policy/delay/attach.toml @@ -0,0 +1,12 @@ +addon_engine = "DelayEngine" +tx_channels_replacements = [ + ["MrpcEngine", "DelayEngine", 0, 0], + ["DelayEngine", "TcpRpcAdapterEngine", 0, 0], +] +rx_channels_replacements = [] +group = ["MrpcEngine", "TcpRpcAdapterEngine"] +op = "attach" +config_string = ''' +delay_probability = 0.2 +delay_ms = 100 +''' diff --git a/eval/policy/delay/collect.py b/eval/policy/delay/collect.py new file mode 100755 index 00000000..04d08256 --- /dev/null +++ b/eval/policy/delay/collect.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +from typing import List +import glob +import sys + +OD = "/tmp/mrpc-eval" +if len(sys.argv) >= 2: + OD = sys.argv[1] + + +def convert_msg_size(s: str) -> int: + if s.endswith('gb'): + return int(s[:-2]) * 1024 * 1024 * 1024 + if s.endswith('mb'): + return int(s[:-2]) * 1024 * 1024 + if s.endswith('kb'): + return int(s[:-2]) * 1024 + if s.endswith('b'): + return int(s[:-1]) + + raise ValueError(f"unknown input: {s}") + + +def get_rate(path: str) -> List[float]: + rates = [] + with open(path, 'r') as fin: + for line in fin: + words = line.strip().split(' ') + if words[-3] == 'rps,': + rate = float(words[-4]) + rates.append(rate) + return rates[1:] + + +def load_result(sol_before, sol_after, f: str): + # print(f) + rates = get_rate(f) + before = rates[5:25] + after = rates[-25:-5] + for r in before: + print(f'{round(r/1000,2)},{sol_before},w/o Fault') + for r in after: + print(f'{round(r/1000,2)},{sol_after},w/ Fault') + + +for f in glob.glob(OD+"/policy/delay/rpc_bench_tput_32b/rpc_bench_client_danyang-04.stdout"): + load_result('mRPC', 'Native mRPC', f) diff --git a/eval/policy/delay/config.toml b/eval/policy/delay/config.toml new file mode 100644 index 00000000..d6ed4a5b --- /dev/null +++ b/eval/policy/delay/config.toml @@ -0,0 +1,9 @@ +workdir = "~/nfs/Developing/livingshade/phoenix/experimental/mrpc" + +[env] +RUST_BACKTRACE = "1" +RUST_LOG_STYLE = "never" +CARGO_TERM_COLOR = "never" +PHOENIX_LOG = "info" +PROTOC = "/usr/bin/protoc" +PHOENIX_PREFIX = "/tmp/phoenix" diff --git a/eval/policy/delay/detach.toml b/eval/policy/delay/detach.toml new file mode 100644 index 00000000..bfd5b661 --- /dev/null +++ b/eval/policy/delay/detach.toml @@ -0,0 +1,4 @@ +addon_engine = "DelayEngine" +tx_channels_replacements = [["MrpcEngine", "TcpRpcAdapterEngine", 0, 0]] +rx_channels_replacements = [["TcpRpcAdapterEngine", "MrpcEngine", 0, 0]] +op = "detach" diff --git a/eval/policy/delay/rpc_bench_tput_32b.toml b/eval/policy/delay/rpc_bench_tput_32b.toml new file mode 100644 index 00000000..67720038 --- /dev/null +++ b/eval/policy/delay/rpc_bench_tput_32b.toml @@ -0,0 +1,15 @@ +name = "policy/delay/rpc_bench_tput_32b" +description = "Run rpc_bench benchmark" +group = "delay" +timeout_secs = 70 + +[[worker]] +host = "danyang-06" +bin = "rpc_bench_server" +args = "--port 5002 -l info --transport tcp" + +[[worker]] +host = "danyang-04" +bin = "rpc_bench_client" +args = "--transport tcp -c rdma0.danyang-06 --concurrency 128 --req-size 32 --duration 65 -i 1 --port 5002 -l error" +dependencies = [0] diff --git a/eval/policy/delay/start_traffic.sh b/eval/policy/delay/start_traffic.sh new file mode 100755 index 00000000..f0c5c3d1 --- /dev/null +++ b/eval/policy/delay/start_traffic.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM SIGHUP EXIT + +OD=/tmp/mrpc-eval +if [[ $# -ge 1 ]]; then + OD=$1 +fi + +WORKDIR=$(dirname $(realpath $0)) +cd $WORKDIR + +# concurrency = 128 +cargo rr --bin launcher -- --output-dir ${OD} --timeout=120 --benchmark ./rpc_bench_tput_32b.toml --configfile ./config.toml & + +sleep 30 + +LIST_OUTPUT="${OD}"/policy/list.json +cargo rr --bin list -- --dump "${LIST_OUTPUT}" # Need to specifiy PHOENIX_PREFIX +cat "${LIST_OUTPUT}" +ARG_PID=$(cat "${LIST_OUTPUT}" | jq '.[] | select(.service == "Mrpc") | .pid') +ARG_SID=$(cat "${LIST_OUTPUT}" | jq '.[] | select(.service == "Mrpc") | .sid') +echo $ARG_SID + +sleep 1 + +cargo run --bin addonctl -- --config ./attach.toml --pid ${ARG_PID} --sid ${ARG_SID} # Need to specifiy PHOENIX_PREFIX + +wait diff --git a/eval/policy/fault/attach.toml b/eval/policy/fault/attach.toml new file mode 100644 index 00000000..a8b4d30e --- /dev/null +++ b/eval/policy/fault/attach.toml @@ -0,0 +1,33 @@ +addon_engine = "FaultEngine" +tx_channels_replacements = [ + [ + "MrpcEngine", + "FaultEngine", + 0, + 0, + ], + [ + "FaultEngine", + "TcpRpcAdapterEngine", + 0, + 0, + ], +] +rx_channels_replacements = [ + [ + "TcpRpcAdapterEngine", + "FaultEngine", + 0, + 0, + ], + [ + "FaultEngine", + "MrpcEngine", + 0, + 0, + ], +] +group = ["MrpcEngine", "TcpRpcAdapterEngine"] +op = "attach" +config_string = ''' +''' diff --git a/eval/policy/fault/collect.py b/eval/policy/fault/collect.py new file mode 100755 index 00000000..009eec0d --- /dev/null +++ b/eval/policy/fault/collect.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +from typing import List +import glob +import sys + +OD = "/tmp/mrpc-eval" +if len(sys.argv) >= 2: + OD = sys.argv[1] + + +def convert_msg_size(s: str) -> int: + if s.endswith('gb'): + return int(s[:-2]) * 1024 * 1024 * 1024 + if s.endswith('mb'): + return int(s[:-2]) * 1024 * 1024 + if s.endswith('kb'): + return int(s[:-2]) * 1024 + if s.endswith('b'): + return int(s[:-1]) + + raise ValueError(f"unknown input: {s}") + + +def get_rate(path: str) -> List[float]: + rates = [] + with open(path, 'r') as fin: + for line in fin: + words = line.strip().split(' ') + if words[-3] == 'rps,': + rate = float(words[-4]) + rates.append(rate) + return rates[1:] + + +def load_result(sol_before, sol_after, f: str): + # print(f) + rates = get_rate(f) + before = rates[5:25] + after = rates[-25:-5] + for r in before: + print(f'{round(r/1000,2)},{sol_before},w/o Fault') + for r in after: + print(f'{round(r/1000,2)},{sol_after},w/ Fault') + + +for f in glob.glob(OD+"/policy/fault/rpc_bench_tput_32b/rpc_bench_client_danyang-04.stdout"): + load_result('mRPC', 'ADN+mRPC', f) diff --git a/eval/policy/fault/config.toml b/eval/policy/fault/config.toml new file mode 100644 index 00000000..d6ed4a5b --- /dev/null +++ b/eval/policy/fault/config.toml @@ -0,0 +1,9 @@ +workdir = "~/nfs/Developing/livingshade/phoenix/experimental/mrpc" + +[env] +RUST_BACKTRACE = "1" +RUST_LOG_STYLE = "never" +CARGO_TERM_COLOR = "never" +PHOENIX_LOG = "info" +PROTOC = "/usr/bin/protoc" +PHOENIX_PREFIX = "/tmp/phoenix" diff --git a/eval/policy/fault/detach.toml b/eval/policy/fault/detach.toml new file mode 100644 index 00000000..02b4e34b --- /dev/null +++ b/eval/policy/fault/detach.toml @@ -0,0 +1,4 @@ +addon_engine = "FaultEngine" +tx_channels_replacements = [["MrpcEngine", "TcpRpcAdapterEngine", 0, 0]] +rx_channels_replacements = [["TcpRpcAdapterEngine", "MrpcEngine", 0, 0]] +op = "detach" diff --git a/eval/policy/fault/rpc_bench_tput_32b.toml b/eval/policy/fault/rpc_bench_tput_32b.toml new file mode 100644 index 00000000..d1d46cd5 --- /dev/null +++ b/eval/policy/fault/rpc_bench_tput_32b.toml @@ -0,0 +1,15 @@ +name = "policy/fault/rpc_bench_tput_32b" +description = "Run rpc_bench benchmark" +group = "fault" +timeout_secs = 70 + +[[worker]] +host = "danyang-06" +bin = "rpc_bench_server" +args = "--port 5002 -l info --transport tcp" + +[[worker]] +host = "danyang-04" +bin = "rpc_bench_client" +args = "--transport tcp -c rdma0.danyang-06 --concurrency 128 --req-size 32 --duration 65 -i 1 --port 5002 -l error" +dependencies = [0] diff --git a/eval/policy/fault/start_traffic.sh b/eval/policy/fault/start_traffic.sh new file mode 100755 index 00000000..f0c5c3d1 --- /dev/null +++ b/eval/policy/fault/start_traffic.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM SIGHUP EXIT + +OD=/tmp/mrpc-eval +if [[ $# -ge 1 ]]; then + OD=$1 +fi + +WORKDIR=$(dirname $(realpath $0)) +cd $WORKDIR + +# concurrency = 128 +cargo rr --bin launcher -- --output-dir ${OD} --timeout=120 --benchmark ./rpc_bench_tput_32b.toml --configfile ./config.toml & + +sleep 30 + +LIST_OUTPUT="${OD}"/policy/list.json +cargo rr --bin list -- --dump "${LIST_OUTPUT}" # Need to specifiy PHOENIX_PREFIX +cat "${LIST_OUTPUT}" +ARG_PID=$(cat "${LIST_OUTPUT}" | jq '.[] | select(.service == "Mrpc") | .pid') +ARG_SID=$(cat "${LIST_OUTPUT}" | jq '.[] | select(.service == "Mrpc") | .sid') +echo $ARG_SID + +sleep 1 + +cargo run --bin addonctl -- --config ./attach.toml --pid ${ARG_PID} --sid ${ARG_SID} # Need to specifiy PHOENIX_PREFIX + +wait diff --git a/eval/policy/fault2/attach.toml b/eval/policy/fault2/attach.toml new file mode 100644 index 00000000..d8d20ed5 --- /dev/null +++ b/eval/policy/fault2/attach.toml @@ -0,0 +1,33 @@ +addon_engine = "Fault2Engine" +tx_channels_replacements = [ + [ + "MrpcEngine", + "Fault2Engine", + 0, + 0, + ], + [ + "Fault2Engine", + "TcpRpcAdapterEngine", + 0, + 0, + ], +] +rx_channels_replacements = [ + [ + "TcpRpcAdapterEngine", + "Fault2Engine", + 0, + 0, + ], + [ + "Fault2Engine", + "MrpcEngine", + 0, + 0, + ], +] +group = ["MrpcEngine", "TcpRpcAdapterEngine"] +op = "attach" +config_string = ''' +''' diff --git a/eval/policy/fault2/collect.py b/eval/policy/fault2/collect.py new file mode 100755 index 00000000..520423c7 --- /dev/null +++ b/eval/policy/fault2/collect.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +from typing import List +import glob +import sys + +OD = "/tmp/mrpc-eval" +if len(sys.argv) >= 2: + OD = sys.argv[1] + + +def convert_msg_size(s: str) -> int: + if s.endswith('gb'): + return int(s[:-2]) * 1024 * 1024 * 1024 + if s.endswith('mb'): + return int(s[:-2]) * 1024 * 1024 + if s.endswith('kb'): + return int(s[:-2]) * 1024 + if s.endswith('b'): + return int(s[:-1]) + + raise ValueError(f"unknown input: {s}") + + +def get_rate(path: str) -> List[float]: + rates = [] + with open(path, 'r') as fin: + for line in fin: + words = line.strip().split(' ') + if words[-3] == 'rps,': + rate = float(words[-4]) + rates.append(rate) + return rates[1:] + + +def load_result(sol_before, sol_after, f: str): + # print(f) + rates = get_rate(f) + before = rates[5:25] + after = rates[-25:-5] + for r in before: + print(f'{round(r/1000,2)},{sol_before},w/o Fault') + for r in after: + print(f'{round(r/1000,2)},{sol_after},w/ Fault') + + +for f in glob.glob(OD+"/policy/fault2/rpc_bench_tput_32b/rpc_bench_client_danyang-04.stdout"): + load_result('mRPC', 'Native mRPC', f) diff --git a/eval/policy/fault2/config.toml b/eval/policy/fault2/config.toml new file mode 100644 index 00000000..d6ed4a5b --- /dev/null +++ b/eval/policy/fault2/config.toml @@ -0,0 +1,9 @@ +workdir = "~/nfs/Developing/livingshade/phoenix/experimental/mrpc" + +[env] +RUST_BACKTRACE = "1" +RUST_LOG_STYLE = "never" +CARGO_TERM_COLOR = "never" +PHOENIX_LOG = "info" +PROTOC = "/usr/bin/protoc" +PHOENIX_PREFIX = "/tmp/phoenix" diff --git a/eval/policy/fault2/detach.toml b/eval/policy/fault2/detach.toml new file mode 100644 index 00000000..fee4d9ec --- /dev/null +++ b/eval/policy/fault2/detach.toml @@ -0,0 +1,4 @@ +addon_engine = "Fault2Engine" +tx_channels_replacements = [["MrpcEngine", "TcpRpcAdapterEngine", 0, 0]] +rx_channels_replacements = [["TcpRpcAdapterEngine", "MrpcEngine", 0, 0]] +op = "detach" diff --git a/eval/policy/fault2/rpc_bench_tput_32b.toml b/eval/policy/fault2/rpc_bench_tput_32b.toml new file mode 100644 index 00000000..2ffc7e94 --- /dev/null +++ b/eval/policy/fault2/rpc_bench_tput_32b.toml @@ -0,0 +1,15 @@ +name = "policy/fault2/rpc_bench_tput_32b" +description = "Run rpc_bench benchmark" +group = "fault2" +timeout_secs = 70 + +[[worker]] +host = "danyang-06" +bin = "rpc_bench_server" +args = "--port 5002 -l info --transport tcp" + +[[worker]] +host = "danyang-04" +bin = "rpc_bench_client" +args = "--transport tcp -c rdma0.danyang-06 --concurrency 128 --req-size 32 --duration 65 -i 1 --port 5002 -l error" +dependencies = [0] diff --git a/eval/policy/fault2/start_traffic.sh b/eval/policy/fault2/start_traffic.sh new file mode 100755 index 00000000..f0c5c3d1 --- /dev/null +++ b/eval/policy/fault2/start_traffic.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM SIGHUP EXIT + +OD=/tmp/mrpc-eval +if [[ $# -ge 1 ]]; then + OD=$1 +fi + +WORKDIR=$(dirname $(realpath $0)) +cd $WORKDIR + +# concurrency = 128 +cargo rr --bin launcher -- --output-dir ${OD} --timeout=120 --benchmark ./rpc_bench_tput_32b.toml --configfile ./config.toml & + +sleep 30 + +LIST_OUTPUT="${OD}"/policy/list.json +cargo rr --bin list -- --dump "${LIST_OUTPUT}" # Need to specifiy PHOENIX_PREFIX +cat "${LIST_OUTPUT}" +ARG_PID=$(cat "${LIST_OUTPUT}" | jq '.[] | select(.service == "Mrpc") | .pid') +ARG_SID=$(cat "${LIST_OUTPUT}" | jq '.[] | select(.service == "Mrpc") | .sid') +echo $ARG_SID + +sleep 1 + +cargo run --bin addonctl -- --config ./attach.toml --pid ${ARG_PID} --sid ${ARG_SID} # Need to specifiy PHOENIX_PREFIX + +wait diff --git a/eval/policy/hello-acl-sender/attach.toml b/eval/policy/hello-acl-sender/attach.toml new file mode 100644 index 00000000..460dc5c6 --- /dev/null +++ b/eval/policy/hello-acl-sender/attach.toml @@ -0,0 +1,33 @@ +addon_engine = "HelloAclSenderEngine" +tx_channels_replacements = [ + [ + "MrpcEngine", + "HelloAclSenderEngine", + 0, + 0, + ], + [ + "HelloAclSenderEngine", + "TcpRpcAdapterEngine", + 0, + 0, + ], +] +rx_channels_replacements = [ + [ + "TcpRpcAdapterEngine", + "HelloAclSenderEngine", + 0, + 0, + ], + [ + "HelloAclSenderEngine", + "MrpcEngine", + 0, + 0, + ], +] +group = ["MrpcEngine", "TcpRpcAdapterEngine"] +op = "attach" +config_string = ''' +''' diff --git a/eval/policy/hello-acl-sender/collect.py b/eval/policy/hello-acl-sender/collect.py new file mode 100755 index 00000000..4bb88ed1 --- /dev/null +++ b/eval/policy/hello-acl-sender/collect.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +from typing import List +import glob +import sys + +OD = "/tmp/mrpc-eval" +if len(sys.argv) >= 2: + OD = sys.argv[1] + + +def convert_msg_size(s: str) -> int: + if s.endswith('gb'): + return int(s[:-2]) * 1024 * 1024 * 1024 + if s.endswith('mb'): + return int(s[:-2]) * 1024 * 1024 + if s.endswith('kb'): + return int(s[:-2]) * 1024 + if s.endswith('b'): + return int(s[:-1]) + + raise ValueError(f"unknown input: {s}") + + +def get_rate(path: str) -> List[float]: + rates = [] + with open(path, 'r') as fin: + for line in fin: + words = line.strip().split(' ') + if words[-3] == 'rps,': + rate = float(words[-4]) + rates.append(rate) + return rates[1:] + + +def load_result(sol_before, sol_after, f: str): + # print(f) + rates = get_rate(f) + before = rates[5:25] + after = rates[-25:-5] + for r in before: + print(f'{round(r/1000,2)},{sol_before},w/o ACL') + for r in after: + print(f'{round(r/1000,2)},{sol_after},w/ ACL') + + +for f in glob.glob(OD+"/policy/hello-acl-sender/rpc_bench_tput_32b/rpc_bench_client2_danyang-04.stdout"): + load_result('mRPC', 'Native mRPC', f) diff --git a/eval/policy/hello-acl-sender/config.toml b/eval/policy/hello-acl-sender/config.toml new file mode 100644 index 00000000..d6ed4a5b --- /dev/null +++ b/eval/policy/hello-acl-sender/config.toml @@ -0,0 +1,9 @@ +workdir = "~/nfs/Developing/livingshade/phoenix/experimental/mrpc" + +[env] +RUST_BACKTRACE = "1" +RUST_LOG_STYLE = "never" +CARGO_TERM_COLOR = "never" +PHOENIX_LOG = "info" +PROTOC = "/usr/bin/protoc" +PHOENIX_PREFIX = "/tmp/phoenix" diff --git a/eval/policy/hello-acl-sender/detach.toml b/eval/policy/hello-acl-sender/detach.toml new file mode 100644 index 00000000..958c09d6 --- /dev/null +++ b/eval/policy/hello-acl-sender/detach.toml @@ -0,0 +1,4 @@ +addon_engine = "HelloAclSenderEngine" +tx_channels_replacements = [["MrpcEngine", "TcpRpcAdapterEngine", 0, 0]] +rx_channels_replacements = [["TcpRpcAdapterEngine", "MrpcEngine", 0, 0]] +op = "detach" diff --git a/eval/policy/hello-acl-sender/rpc_bench_tput_32b.toml b/eval/policy/hello-acl-sender/rpc_bench_tput_32b.toml new file mode 100644 index 00000000..1261cdd5 --- /dev/null +++ b/eval/policy/hello-acl-sender/rpc_bench_tput_32b.toml @@ -0,0 +1,15 @@ +name = "policy/hello-acl-sender/rpc_bench_tput_32b" +description = "Run rpc_bench benchmark" +group = "hello-acl-sender" +timeout_secs = 70 + +[[worker]] +host = "danyang-06" +bin = "rpc_bench_server" +args = "--port 5002 -l info --transport tcp" + +[[worker]] +host = "danyang-04" +bin = "rpc_bench_client2" +args = "--transport tcp -c rdma0.danyang-06 --concurrency 128 --req-size 32 --duration 65 -i 1 --port 5002 -l error" +dependencies = [0] diff --git a/eval/policy/hello-acl-sender/start_traffic.sh b/eval/policy/hello-acl-sender/start_traffic.sh new file mode 100755 index 00000000..f0c5c3d1 --- /dev/null +++ b/eval/policy/hello-acl-sender/start_traffic.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM SIGHUP EXIT + +OD=/tmp/mrpc-eval +if [[ $# -ge 1 ]]; then + OD=$1 +fi + +WORKDIR=$(dirname $(realpath $0)) +cd $WORKDIR + +# concurrency = 128 +cargo rr --bin launcher -- --output-dir ${OD} --timeout=120 --benchmark ./rpc_bench_tput_32b.toml --configfile ./config.toml & + +sleep 30 + +LIST_OUTPUT="${OD}"/policy/list.json +cargo rr --bin list -- --dump "${LIST_OUTPUT}" # Need to specifiy PHOENIX_PREFIX +cat "${LIST_OUTPUT}" +ARG_PID=$(cat "${LIST_OUTPUT}" | jq '.[] | select(.service == "Mrpc") | .pid') +ARG_SID=$(cat "${LIST_OUTPUT}" | jq '.[] | select(.service == "Mrpc") | .sid') +echo $ARG_SID + +sleep 1 + +cargo run --bin addonctl -- --config ./attach.toml --pid ${ARG_PID} --sid ${ARG_SID} # Need to specifiy PHOENIX_PREFIX + +wait diff --git a/eval/policy/hello-acl/attach.toml b/eval/policy/hello-acl/attach.toml new file mode 100644 index 00000000..8f6511bb --- /dev/null +++ b/eval/policy/hello-acl/attach.toml @@ -0,0 +1,33 @@ +addon_engine = "HelloAclEngine" +tx_channels_replacements = [ + [ + "MrpcEngine", + "HelloAclEngine", + 0, + 0, + ], + [ + "HelloAclEngine", + "TcpRpcAdapterEngine", + 0, + 0, + ], +] +rx_channels_replacements = [ + [ + "TcpRpcAdapterEngine", + "HelloAclEngine", + 0, + 0, + ], + [ + "HelloAclEngine", + "MrpcEngine", + 0, + 0, + ], +] +group = ["MrpcEngine", "TcpRpcAdapterEngine"] +op = "attach" +config_string = ''' +''' diff --git a/eval/policy/hello-acl/collect.py b/eval/policy/hello-acl/collect.py new file mode 100755 index 00000000..900a8b4b --- /dev/null +++ b/eval/policy/hello-acl/collect.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +from typing import List +import glob +import sys + +OD = "/tmp/mrpc-eval" +if len(sys.argv) >= 2: + OD = sys.argv[1] + + +def convert_msg_size(s: str) -> int: + if s.endswith('gb'): + return int(s[:-2]) * 1024 * 1024 * 1024 + if s.endswith('mb'): + return int(s[:-2]) * 1024 * 1024 + if s.endswith('kb'): + return int(s[:-2]) * 1024 + if s.endswith('b'): + return int(s[:-1]) + + raise ValueError(f"unknown input: {s}") + + +def get_rate(path: str) -> List[float]: + rates = [] + with open(path, 'r') as fin: + for line in fin: + words = line.strip().split(' ') + if words[-3] == 'rps,': + rate = float(words[-4]) + rates.append(rate) + return rates[1:] + + +def load_result(sol_before, sol_after, f: str): + # print(f) + rates = get_rate(f) + before = rates[5:25] + after = rates[-25:-5] + for r in before: + print(f'{round(r/1000,2)},{sol_before},w/o ACL') + for r in after: + print(f'{round(r/1000,2)},{sol_after},w/ ACL') + + +for f in glob.glob(OD+"/policy/hello-acl/rpc_bench_tput_32b/rpc_bench_client2_danyang-04.stdout"): + load_result('mRPC', 'ADN+mRPC', f) diff --git a/eval/policy/hello-acl/config.toml b/eval/policy/hello-acl/config.toml new file mode 100644 index 00000000..d6ed4a5b --- /dev/null +++ b/eval/policy/hello-acl/config.toml @@ -0,0 +1,9 @@ +workdir = "~/nfs/Developing/livingshade/phoenix/experimental/mrpc" + +[env] +RUST_BACKTRACE = "1" +RUST_LOG_STYLE = "never" +CARGO_TERM_COLOR = "never" +PHOENIX_LOG = "info" +PROTOC = "/usr/bin/protoc" +PHOENIX_PREFIX = "/tmp/phoenix" diff --git a/eval/policy/hello-acl/detach.toml b/eval/policy/hello-acl/detach.toml new file mode 100644 index 00000000..04a673e5 --- /dev/null +++ b/eval/policy/hello-acl/detach.toml @@ -0,0 +1,4 @@ +addon_engine = "HelloAclEngine" +tx_channels_replacements = [["MrpcEngine", "TcpRpcAdapterEngine", 0, 0]] +rx_channels_replacements = [["TcpRpcAdapterEngine", "MrpcEngine", 0, 0]] +op = "detach" diff --git a/eval/policy/hello-acl/rpc_bench_tput_32b.toml b/eval/policy/hello-acl/rpc_bench_tput_32b.toml new file mode 100644 index 00000000..bfa29edd --- /dev/null +++ b/eval/policy/hello-acl/rpc_bench_tput_32b.toml @@ -0,0 +1,15 @@ +name = "policy/hello-acl/rpc_bench_tput_32b" +description = "Run rpc_bench benchmark" +group = "hello-acl" +timeout_secs = 70 + +[[worker]] +host = "danyang-06" +bin = "rpc_bench_server" +args = "--port 5002 -l info --transport tcp" + +[[worker]] +host = "danyang-04" +bin = "rpc_bench_client2" +args = "--transport tcp -c rdma0.danyang-06 --concurrency 128 --req-size 32 --duration 65 -i 1 --port 5002 -l error" +dependencies = [0] diff --git a/eval/policy/hello-acl/start_traffic.sh b/eval/policy/hello-acl/start_traffic.sh new file mode 100755 index 00000000..f0c5c3d1 --- /dev/null +++ b/eval/policy/hello-acl/start_traffic.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM SIGHUP EXIT + +OD=/tmp/mrpc-eval +if [[ $# -ge 1 ]]; then + OD=$1 +fi + +WORKDIR=$(dirname $(realpath $0)) +cd $WORKDIR + +# concurrency = 128 +cargo rr --bin launcher -- --output-dir ${OD} --timeout=120 --benchmark ./rpc_bench_tput_32b.toml --configfile ./config.toml & + +sleep 30 + +LIST_OUTPUT="${OD}"/policy/list.json +cargo rr --bin list -- --dump "${LIST_OUTPUT}" # Need to specifiy PHOENIX_PREFIX +cat "${LIST_OUTPUT}" +ARG_PID=$(cat "${LIST_OUTPUT}" | jq '.[] | select(.service == "Mrpc") | .pid') +ARG_SID=$(cat "${LIST_OUTPUT}" | jq '.[] | select(.service == "Mrpc") | .sid') +echo $ARG_SID + +sleep 1 + +cargo run --bin addonctl -- --config ./attach.toml --pid ${ARG_PID} --sid ${ARG_SID} # Need to specifiy PHOENIX_PREFIX + +wait diff --git a/eval/policy/logging/README.md b/eval/policy/logging/README.md new file mode 100644 index 00000000..2e14d4dc --- /dev/null +++ b/eval/policy/logging/README.md @@ -0,0 +1,4 @@ +If you want to measure latency + +change `--concurrency 128` to `--concurrency 1` and append +`--log-latency` to the end of the args in `rpc_bench_tput_32b.toml`. diff --git a/eval/policy/logging/collect.py b/eval/policy/logging/collect.py new file mode 100755 index 00000000..5aeb37f3 --- /dev/null +++ b/eval/policy/logging/collect.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +from typing import List +import glob +import sys + +OD = "/tmp/mrpc-eval" +if len(sys.argv) >= 2: + OD = sys.argv[1] + + +def get_rate(path: str) -> List[float]: + rates = [] + with open(path, 'r') as fin: + for line in fin: + words = line.strip().split(' ') + if words[-3] == 'rps,': + rate = float(words[-4]) + rates.append(rate) + return rates[1:] + + +def load_result(sol_before, sol_after, f: str): + rates = get_rate(f) + before = rates[5:25] + after = rates[-25:-5] + for r in before: + print(f'{round(r/1000,2)},{sol_before},w/o Logging') + for r in after: + print(f'{round(r/1000,2)},{sol_after},w/ Logging') + + +for f in glob.glob(OD+"/policy/logging/rpc_bench_tput_32b/rpc_bench_client_danyang-04.stdout"): + load_result('mRPC', 'Native mRPC', f) diff --git a/eval/policy/logging/config.toml b/eval/policy/logging/config.toml new file mode 100644 index 00000000..d6ed4a5b --- /dev/null +++ b/eval/policy/logging/config.toml @@ -0,0 +1,9 @@ +workdir = "~/nfs/Developing/livingshade/phoenix/experimental/mrpc" + +[env] +RUST_BACKTRACE = "1" +RUST_LOG_STYLE = "never" +CARGO_TERM_COLOR = "never" +PHOENIX_LOG = "info" +PROTOC = "/usr/bin/protoc" +PHOENIX_PREFIX = "/tmp/phoenix" diff --git a/eval/policy/logging/detach.toml b/eval/policy/logging/detach.toml index 465cc3e0..da29f3fb 100644 --- a/eval/policy/logging/detach.toml +++ b/eval/policy/logging/detach.toml @@ -1,6 +1,4 @@ addon_engine = "LoggingEngine" -tx_channels_replacements = [ - ["MrpcEngine", "RpcAdapterEngine", 0, 0], -] -rx_channels_replacements = [] +tx_channels_replacements = [["MrpcEngine", "TcpRpcAdapterEngine", 0, 0]] +rx_channels_replacements = [["TcpRpcAdapterEngine", "MrpcEngine", 0, 0]] op = "detach" diff --git a/eval/policy/logging/rpc_bench_tput_32b.toml b/eval/policy/logging/rpc_bench_tput_32b.toml new file mode 100644 index 00000000..b8aae34e --- /dev/null +++ b/eval/policy/logging/rpc_bench_tput_32b.toml @@ -0,0 +1,15 @@ +name = "policy/logging/rpc_bench_tput_32b" +description = "Run rpc_bench benchmark" +group = "logging" +timeout_secs = 70 + +[[worker]] +host = "danyang-06" +bin = "rpc_bench_server" +args = "--port 5002 -l info --transport tcp" + +[[worker]] +host = "danyang-04" +bin = "rpc_bench_client" +args = "--transport tcp -c rdma0.danyang-06 --concurrency 128 --req-size 32 --duration 65 -i 1 --port 5002 -l info" +dependencies = [0] diff --git a/eval/policy/logging/run_policy.py b/eval/policy/logging/run_policy.py new file mode 100755 index 00000000..02baaf74 --- /dev/null +++ b/eval/policy/logging/run_policy.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +import json +import subprocess +import pathlib +import os +from os.path import dirname +import time +import toml +import sys + +OD = "/tmp/mrpc-eval" +if len(sys.argv) >= 2: + OD = sys.argv[1] + +SCRIPTDIR = pathlib.Path(__file__).parent.resolve() +CONFIG_PATH = os.path.join(SCRIPTDIR, "config.toml") + +config = toml.load(CONFIG_PATH) +workdir = config["workdir"] +workdir = dirname(dirname(os.path.expanduser(workdir))) +env = {**os.environ, **config['env']} + +os.chdir(workdir) +os.makedirs(f"{OD}/policy/null", exist_ok=True) + +cmd = f'''cargo run --release -p benchmark --bin launcher -- -o {OD} --timeout=120 +--benchmark {os.path.join(SCRIPTDIR, 'rpc_bench_tput_32b.toml')} +--configfile { os.path.join(SCRIPTDIR, 'config.toml')}''' +workload = subprocess.Popen(cmd.split()) +time.sleep(30) + +list_cmd = f"cargo run --release --bin list -- --dump {OD}/policy/list.json" +subprocess.run(list_cmd.split(), env=env) + +with open(f"{OD}/policy/list.json", "r") as fin: + content = fin.read() + print(content) + data = json.loads(content) +mrpc_pid = None +mrpc_sid = None +for subscription in data: + pid = subscription["pid"] + sid = subscription["sid"] + engines = [x[1] for x in subscription["engines"]] + if "MrpcEngine" in engines: + mrpc_pid = pid + mrpc_sid = sid + +print("Start to attach policy") +attach_config = os.path.join(SCRIPTDIR, "attach.toml") +attach_cmd = f"cargo run --release --bin addonctl -- --config {attach_config} --pid {mrpc_pid} --sid {mrpc_sid}" +subprocess.run(attach_cmd.split(), env=env) + +subprocess.run(list_cmd.split(), env=env) +with open(f"{OD}/policy/list.json", "r") as fin: + print(fin.read()) + +workload.wait() diff --git a/eval/policy/nofile-logging/attach.toml b/eval/policy/nofile-logging/attach.toml new file mode 100644 index 00000000..7f86f8c9 --- /dev/null +++ b/eval/policy/nofile-logging/attach.toml @@ -0,0 +1,31 @@ +addon_engine = "NofileLoggingEngine" +tx_channels_replacements = [ + [ + "MrpcEngine", + "NofileLoggingEngine", + 0, + 0, + ], + [ + "NofileLoggingEngine", + "TcpRpcAdapterEngine", + 0, + 0, + ], +] +rx_channels_replacements = [ + [ + "TcpRpcAdapterEngine", + "NofileLoggingEngine", + 0, + 0, + ], + [ + "NofileLoggingEngine", + "MrpcEngine", + 0, + 0, + ], +] +group = ["MrpcEngine", "TcpRpcAdapterEngine"] +op = "attach" diff --git a/eval/policy/nofile-logging/collect.py b/eval/policy/nofile-logging/collect.py new file mode 100755 index 00000000..4694884b --- /dev/null +++ b/eval/policy/nofile-logging/collect.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +from typing import List +import glob +import sys + +OD = "/tmp/mrpc-eval" +if len(sys.argv) >= 2: + OD = sys.argv[1] + + +def get_rate(path: str) -> List[float]: + rates = [] + with open(path, 'r') as fin: + for line in fin: + words = line.strip().split(' ') + if words[-3] == 'rps,': + rate = float(words[-4]) + rates.append(rate) + return rates[1:] + + +def load_result(sol_before, sol_after, f: str): + rates = get_rate(f) + before = rates[5:25] + after = rates[-25:-5] + for r in before: + print(f'{round(r/1000,2)},{sol_before},w/o Logging') + for r in after: + print(f'{round(r/1000,2)},{sol_after},w/ Logging') + + +for f in glob.glob(OD+"/policy/nofile-logging/rpc_bench_tput_32b/rpc_bench_client_danyang-04.stdout"): + load_result('mRPC', 'ADN+mRPC', f) diff --git a/eval/policy/nofile-logging/config.toml b/eval/policy/nofile-logging/config.toml new file mode 100644 index 00000000..d6ed4a5b --- /dev/null +++ b/eval/policy/nofile-logging/config.toml @@ -0,0 +1,9 @@ +workdir = "~/nfs/Developing/livingshade/phoenix/experimental/mrpc" + +[env] +RUST_BACKTRACE = "1" +RUST_LOG_STYLE = "never" +CARGO_TERM_COLOR = "never" +PHOENIX_LOG = "info" +PROTOC = "/usr/bin/protoc" +PHOENIX_PREFIX = "/tmp/phoenix" diff --git a/eval/policy/nofile-logging/detach.toml b/eval/policy/nofile-logging/detach.toml new file mode 100644 index 00000000..6541d04f --- /dev/null +++ b/eval/policy/nofile-logging/detach.toml @@ -0,0 +1,4 @@ +addon_engine = "NofileLoggingEngine" +tx_channels_replacements = [["MrpcEngine", "TcpRpcAdapterEngine", 0, 0]] +rx_channels_replacements = [["TcpRpcAdapterEngine", "MrpcEngine", 0, 0]] +op = "detach" diff --git a/eval/policy/nofile-logging/rpc_bench_tput_32b.toml b/eval/policy/nofile-logging/rpc_bench_tput_32b.toml new file mode 100644 index 00000000..414d7fc4 --- /dev/null +++ b/eval/policy/nofile-logging/rpc_bench_tput_32b.toml @@ -0,0 +1,15 @@ +name = "policy/nofile-logging/rpc_bench_tput_32b" +description = "Run rpc_bench benchmark" +group = "nofile-logging" +timeout_secs = 70 + +[[worker]] +host = "danyang-06" +bin = "rpc_bench_server" +args = "--port 5002 -l info --transport tcp" + +[[worker]] +host = "danyang-04" +bin = "rpc_bench_client" +args = "--transport tcp -c rdma0.danyang-06 --concurrency 128 --req-size 32 --duration 65 -i 1 --port 5002 -l info" +dependencies = [0] diff --git a/eval/policy/nofile-logging/run_policy.py b/eval/policy/nofile-logging/run_policy.py new file mode 100755 index 00000000..02baaf74 --- /dev/null +++ b/eval/policy/nofile-logging/run_policy.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +import json +import subprocess +import pathlib +import os +from os.path import dirname +import time +import toml +import sys + +OD = "/tmp/mrpc-eval" +if len(sys.argv) >= 2: + OD = sys.argv[1] + +SCRIPTDIR = pathlib.Path(__file__).parent.resolve() +CONFIG_PATH = os.path.join(SCRIPTDIR, "config.toml") + +config = toml.load(CONFIG_PATH) +workdir = config["workdir"] +workdir = dirname(dirname(os.path.expanduser(workdir))) +env = {**os.environ, **config['env']} + +os.chdir(workdir) +os.makedirs(f"{OD}/policy/null", exist_ok=True) + +cmd = f'''cargo run --release -p benchmark --bin launcher -- -o {OD} --timeout=120 +--benchmark {os.path.join(SCRIPTDIR, 'rpc_bench_tput_32b.toml')} +--configfile { os.path.join(SCRIPTDIR, 'config.toml')}''' +workload = subprocess.Popen(cmd.split()) +time.sleep(30) + +list_cmd = f"cargo run --release --bin list -- --dump {OD}/policy/list.json" +subprocess.run(list_cmd.split(), env=env) + +with open(f"{OD}/policy/list.json", "r") as fin: + content = fin.read() + print(content) + data = json.loads(content) +mrpc_pid = None +mrpc_sid = None +for subscription in data: + pid = subscription["pid"] + sid = subscription["sid"] + engines = [x[1] for x in subscription["engines"]] + if "MrpcEngine" in engines: + mrpc_pid = pid + mrpc_sid = sid + +print("Start to attach policy") +attach_config = os.path.join(SCRIPTDIR, "attach.toml") +attach_cmd = f"cargo run --release --bin addonctl -- --config {attach_config} --pid {mrpc_pid} --sid {mrpc_sid}" +subprocess.run(attach_cmd.split(), env=env) + +subprocess.run(list_cmd.split(), env=env) +with open(f"{OD}/policy/list.json", "r") as fin: + print(fin.read()) + +workload.wait() diff --git a/eval/policy/ratelimit/config.toml b/eval/policy/ratelimit/config.toml index 4a2b8746..41c5ca06 100644 --- a/eval/policy/ratelimit/config.toml +++ b/eval/policy/ratelimit/config.toml @@ -1,4 +1,4 @@ -workdir = "~/nfs/phoenix" +workdir = "~/nfs/phoenix/experimental/mrpc" [env] RUST_BACKTRACE = "1" diff --git a/eval/policy/ratelimit/rpc_bench_tput_32b.toml b/eval/policy/ratelimit/rpc_bench_tput_32b.toml index 3b2f106d..b8d1e75d 100644 --- a/eval/policy/ratelimit/rpc_bench_tput_32b.toml +++ b/eval/policy/ratelimit/rpc_bench_tput_32b.toml @@ -6,10 +6,10 @@ timeout_secs = 70 [[worker]] host = "danyang-06" bin = "rpc_bench_server" -args = "--port 5002 -l info" +args = "--port 5002 -l info --transport tcp" [[worker]] -host = "danyang-05" +host = "danyang-04" bin = "rpc_bench_client" -args = "-c rdma0.danyang-06 --concurrency 128 --req-size 32 --duration 65 -i 1 --port 5002 -l info" +args = "--transport tcp -c rdma0.danyang-06 --concurrency 128 --req-size 32 --duration 65 -i 1 --port 5002 -l info" dependencies = [0] diff --git a/eval/policy/ratelimit/run_policy.py b/eval/policy/ratelimit/run_policy.py index 315387d3..02baaf74 100644 --- a/eval/policy/ratelimit/run_policy.py +++ b/eval/policy/ratelimit/run_policy.py @@ -3,6 +3,7 @@ import subprocess import pathlib import os +from os.path import dirname import time import toml import sys @@ -16,13 +17,13 @@ config = toml.load(CONFIG_PATH) workdir = config["workdir"] -workdir = os.path.expanduser(workdir) +workdir = dirname(dirname(os.path.expanduser(workdir))) env = {**os.environ, **config['env']} os.chdir(workdir) os.makedirs(f"{OD}/policy/null", exist_ok=True) -cmd = f'''cargo run --release --bin launcher -- -o {OD} --timeout=120 +cmd = f'''cargo run --release -p benchmark --bin launcher -- -o {OD} --timeout=120 --benchmark {os.path.join(SCRIPTDIR, 'rpc_bench_tput_32b.toml')} --configfile { os.path.join(SCRIPTDIR, 'config.toml')}''' workload = subprocess.Popen(cmd.split()) diff --git a/experimental/mrpc/Cargo.lock b/experimental/mrpc/Cargo.lock index 826584cc..63272e7e 100644 --- a/experimental/mrpc/Cargo.lock +++ b/experimental/mrpc/Cargo.lock @@ -1623,6 +1623,44 @@ dependencies = [ "static_assertions", ] +name = "phoenix-api-policy-delay" +version = "0.1.0" +dependencies = [ + "itertools", + "phoenix-api", + "rand 0.8.5", + "serde", +] + +[[package]] +name = "phoenix-api-policy-fault" +version = "0.1.0" +dependencies = [ + "itertools", + "phoenix-api", + "rand 0.8.5", + "serde", +] + +[[package]] +name = "phoenix-api-policy-fault2" +version = "0.1.0" +dependencies = [ + "itertools", + "phoenix-api", + "rand 0.8.5", + "serde", +] + +[[package]] +name = "phoenix-api-policy-hello-acl" +version = "0.1.0" +dependencies = [ + "itertools", + "phoenix-api", + "serde", +] + [[package]] name = "phoenix-api-policy-hello-acl-receiver" version = "0.1.0" @@ -1655,6 +1693,16 @@ dependencies = [ "serde", ] +[[package]] +name = "phoenix-api-policy-nofile-logging" +version = "0.1.0" +dependencies = [ + "itertools", + "phoenix-api", + "rand 0.8.5", + "serde", +] + [[package]] name = "phoenix-api-policy-null" version = "0.1.0" @@ -1733,6 +1781,101 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "phoenix-delay" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "chrono", + "futures", + "itertools", + "minstant", + "mrpc-derive", + "mrpc-marshal", + "nix", + "phoenix-api", + "phoenix-api-policy-delay", + "phoenix_common", + "rand 0.8.5", + "serde", + "serde_json", + "shm", + "thiserror", + "toml", +] + +[[package]] +name = "phoenix-fault" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "chrono", + "futures", + "itertools", + "minstant", + "mrpc-derive", + "mrpc-marshal", + "nix", + "phoenix-api", + "phoenix-api-policy-fault", + "phoenix_common", + "rand 0.8.5", + "serde", + "serde_json", + "shm", + "thiserror", + "toml", +] + +[[package]] +name = "phoenix-fault2" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "chrono", + "futures", + "itertools", + "minstant", + "mrpc-derive", + "mrpc-marshal", + "nix", + "phoenix-api", + "phoenix-api-policy-fault2", + "phoenix_common", + "rand 0.8.5", + "serde", + "serde_json", + "shm", + "thiserror", + "toml", +] + +[[package]] +name = "phoenix-hello-acl" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "chrono", + "futures", + "itertools", + "minstant", + "mrpc-derive", + "mrpc-marshal", + "nix", + "phoenix-api", + "phoenix-api-policy-hello-acl", + "phoenix_common", + "serde", + "serde_json", + "shm", + "thiserror", + "toml", +] + [[package]] name = "phoenix-hello-acl-receiver" version = "0.1.0" @@ -1839,12 +1982,18 @@ dependencies = [ "bincode", "chrono", "futures", + "itertools", "minstant", + "mrpc-derive", + "mrpc-marshal", "nix", + "phoenix-api", "phoenix-api-policy-logging", "phoenix_common", + "rand 0.8.5", "serde", "serde_json", + "shm", "thiserror", "toml", ] @@ -1913,6 +2062,29 @@ dependencies = [ "uuid", ] +name = "phoenix-nofile-logging" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "chrono", + "futures", + "itertools", + "minstant", + "mrpc-derive", + "mrpc-marshal", + "nix", + "phoenix-api", + "phoenix-api-policy-nofile-logging", + "phoenix_common", + "rand 0.8.5", + "serde", + "serde_json", + "shm", + "thiserror", + "toml", +] + [[package]] name = "phoenix-null" version = "0.1.0" @@ -2295,11 +2467,22 @@ checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ "getrandom 0.1.16", "libc", - "rand_chacha", + "rand_chacha 0.2.2", "rand_core 0.5.1", "rand_hc", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand_chacha" version = "0.2.2" @@ -2310,6 +2493,16 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + [[package]] name = "rand_core" version = "0.3.1" @@ -2334,6 +2527,15 @@ dependencies = [ "getrandom 0.1.16", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.8", +] + [[package]] name = "rand_hc" version = "0.2.0" diff --git a/experimental/mrpc/Cargo.toml b/experimental/mrpc/Cargo.toml index 8979240a..b7f39c6c 100644 --- a/experimental/mrpc/Cargo.toml +++ b/experimental/mrpc/Cargo.toml @@ -56,6 +56,11 @@ members = [ "phoenix-api/policy/logging", "phoenix-api/policy/hello-acl-receiver", "phoenix-api/policy/hello-acl-sender", + "phoenix-api/policy/hello-acl", + "phoenix-api/policy/nofile-logging", + "phoenix-api/policy/fault", + "phoenix-api/policy/fault2", + "phoenix-api/policy/delay", # the pheonix plugins "plugin/mrpc", "plugin/mrpclb", @@ -70,6 +75,11 @@ members = [ "plugin/policy/hotel-acl", "plugin/policy/hello-acl-receiver", "plugin/policy/hello-acl-sender", + "plugin/policy/hello-acl", + "plugin/policy/nofile-logging", + "plugin/policy/fault", + "plugin/policy/fault2", + "plugin/policy/delay", # examples "examples/rpc_echo", "examples/rpc_bench", @@ -96,6 +106,11 @@ phoenix-api-policy-hotel-acl = { path = "phoenix-api/policy/hotel-acl" } phoenix-api-policy-logging = { path = "phoenix-api/policy/logging" } phoenix-api-policy-hello-acl-receiver = { path = "phoenix-api/policy/hello-acl-receiver" } phoenix-api-policy-hello-acl-sender = { path = "phoenix-api/policy/hello-acl-sender" } +phoenix-api-policy-hello-acl = { path = "phoenix-api/policy/hello-acl" } +phoenix-api-policy-nofile-logging = { path = "phoenix-api/policy/nofile-logging" } +phoenix-api-policy-fault = { path = "phoenix-api/policy/fault" } +phoenix-api-policy-fault2 = { path = "phoenix-api/policy/fault2" } +phoenix-api-policy-delay = { path = "phoenix-api/policy/delay" } mrpc-build = { path = "mrpc-build" } mrpc-derive = { path = "mrpc-derive" } @@ -169,6 +184,7 @@ fasthash = "0.4.0" link-cplusplus = "1.0" arc-swap = "1.5.0" crossbeam-utils = "0.8.12" +rand = "0.8" [profile.release] debug = true diff --git a/experimental/mrpc/examples/rpc_bench/Cargo.toml b/experimental/mrpc/examples/rpc_bench/Cargo.toml index ac17867b..2ce4d3ab 100644 --- a/experimental/mrpc/examples/rpc_bench/Cargo.toml +++ b/experimental/mrpc/examples/rpc_bench/Cargo.toml @@ -28,6 +28,10 @@ libnuma.workspace = true name = "rpc_bench_client" path = "src/client.rs" +[[bin]] +name = "rpc_bench_client2" +path = "src/client2.rs" + [[bin]] name = "rpc_bench_server" path = "src/server.rs" diff --git a/experimental/mrpc/examples/rpc_bench/src/client.rs b/experimental/mrpc/examples/rpc_bench/src/client.rs index 7a19ac5a..e86fd38e 100644 --- a/experimental/mrpc/examples/rpc_bench/src/client.rs +++ b/experimental/mrpc/examples/rpc_bench/src/client.rs @@ -196,10 +196,12 @@ async fn run_bench( if rcnt>args.warmup{ if args.log_latency { my_print!( - "Thread {}, {} rps, {} Gb/s, p95: {:?}, p99: {:?}", + "Thread {}, {} rps, {} Gb/s, avg: {:?}, median: {:?}, p95: {:?}, p99: {:?}", tid, rps, bw, + Duration::from_nanos(hist.mean() as u64), + Duration::from_nanos(hist.value_at_percentile(50.0)), Duration::from_nanos(hist.value_at_percentile(95.0)), Duration::from_nanos(hist.value_at_percentile(99.0)), ); @@ -226,10 +228,12 @@ async fn run_bench( if rcnt>args.warmup{ if args.log_latency { my_print!( - "Thread {}, {} rps, {} Gb/s, p95: {:?}, p99: {:?}", + "Thread {}, {} rps, {} Gb/s, avg: {:?}, median: {:?}, p95: {:?}, p99: {:?}", tid, rps, bw, + Duration::from_nanos(hist.mean() as u64), + Duration::from_nanos(hist.value_at_percentile(50.0)), Duration::from_nanos(hist.value_at_percentile(95.0)), Duration::from_nanos(hist.value_at_percentile(99.0)), ); diff --git a/experimental/mrpc/examples/rpc_bench/src/client2.rs b/experimental/mrpc/examples/rpc_bench/src/client2.rs new file mode 100644 index 00000000..35fcaccc --- /dev/null +++ b/experimental/mrpc/examples/rpc_bench/src/client2.rs @@ -0,0 +1,396 @@ +use std::future::Future; +use std::path::PathBuf; +use std::time::Duration; + +use futures::select; +use futures::stream::FuturesUnordered; +use futures::stream::StreamExt; +use hdrhistogram::Histogram; +use minstant::Instant; +use structopt::StructOpt; + +use mrpc::stub::TransportType; +use mrpc::WRef; + +pub mod rpc_hello { + // The string specified here must match the proto package name + mrpc::include_proto!("rpc_hello"); + // include!("../../../mrpc/src/codegen.rs"); +} +use rpc_hello::greeter_client::GreeterClient; +use rpc_hello::{HelloReply, HelloRequest}; + +#[derive(StructOpt, Debug)] +#[structopt(about = "mRPC benchmark client")] +pub struct Args { + /// The address to connect, can be an IP address or domain name. + /// When multiple addresses are specified, each client thread connects to one of them. + #[structopt(short = "c", long = "connect", default_value = "192.168.211.66")] + pub connects: Vec, + + /// The port number to use. + #[structopt(short, long, default_value = "5000")] + pub port: u16, + + /// Log level for tracing. + #[structopt(short = "l", long, default_value = "error")] + pub log_level: String, + + /// Log directory. + #[structopt(long)] + pub log_dir: Option, + + #[structopt(long)] + pub log_latency: bool, + + /// Request size. + #[structopt(short, long, default_value = "1000000")] + pub req_size: usize, + + /// The maximal number of concurrenty outstanding requests. + #[structopt(long, default_value = "32")] + pub concurrency: usize, + + /// Total number of iterations. + #[structopt(short, long, default_value = "16384")] + pub total_iters: usize, + + /// Number of warmup iterations. + #[structopt(short, long, default_value = "1000")] + pub warmup: usize, + + /// Run test for a customized period of seconds. + #[structopt(short = "D", long)] + pub duration: Option, + + /// Seconds between periodic throughput reports. + #[structopt(short, long)] + pub interval: Option, + + /// The number of messages to provision. Must be a positive number. The test will repeatedly + /// sending the provisioned message. + #[structopt(long, default_value = "1")] + pub provision_count: usize, + + /// Number of client threads. Each client thread is mapped to one server threads. + #[structopt(long, default_value = "1")] + pub num_client_threads: usize, + + /// Number of server threads. + #[structopt(long, default_value = "1")] + pub num_server_threads: usize, + + /// Which transport to use, rdma or tcp + #[structopt(long, default_value = "rdma")] + pub transport: TransportType, +} + +// mod bench_app; +// include!("./bench_app.rs"); + +#[derive(Debug)] +struct Call { + ts: Instant, + req_size: usize, + result: Result, mrpc::Status>, +} + +fn make_rpc_call<'c>( + client: &'c GreeterClient, + workload: &'c Workload, + scnt: usize, +) -> impl Future + 'c { + let ts = Instant::now(); + let (req, req_size) = workload.next_request(scnt); + let fut = client.say_hello(req); + async move { + Call { + ts, + req_size, + result: fut.await, + } + } +} + +#[allow(unused)] +async fn run_bench( + args: &Args, + client: &GreeterClient, + workload: &Workload, + tid: usize, +) -> Result<(Duration, usize, usize, Histogram), mrpc::Status> { + macro_rules! my_print { + ($($arg:tt)*) => { + if args.log_level == "info" { + tracing::info!($($arg)*); + } else { + println!($($arg)*); + } + } + } + + let mut hist = hdrhistogram::Histogram::::new_with_max(60_000_000_000, 5).unwrap(); + + let mut reply_futures = FuturesUnordered::new(); + + let (total_iters, timeout) = if let Some(dura) = args.duration { + (usize::MAX / 2, Duration::from_secs_f64(dura)) + } else { + (args.total_iters, Duration::from_millis(u64::MAX)) + }; + + // report the rps every several milliseconds + let tput_interval = args.interval.map(Duration::from_secs_f64); + + // start sending + let mut last_ts = Instant::now(); + let start = Instant::now(); + + let mut warmup_end = Instant::now(); + let mut nbytes = 0; + let mut last_nbytes = 0; + + let mut scnt = 0; + let mut rcnt = 0; + let mut last_rcnt = 0; + + while scnt < args.concurrency && scnt < total_iters + args.warmup { + let fut = make_rpc_call(client, workload, scnt); + reply_futures.push(fut); + scnt += 1; + } + + loop { + select! { + resp = reply_futures.next() => { + if rcnt >= total_iters + args.warmup || start.elapsed() > timeout { + break; + } + + let Call { ts, req_size, result: resp } = resp.unwrap(); + if let Err(status) = resp { + tracing::warn!("failed request with: {}", status); + } + + if rcnt >= args.warmup { + let dura = ts.elapsed(); + let _ = hist.record(dura.as_nanos() as u64); + } + + nbytes += req_size; + rcnt += 1; + if rcnt == args.warmup { + warmup_end = Instant::now(); + } + + if scnt < total_iters + args.warmup { + let fut = make_rpc_call(client, workload, scnt); + reply_futures.push(fut); + scnt += 1; + } + + let last_dura = last_ts.elapsed(); + if tput_interval.is_some() && last_dura > tput_interval.unwrap() { + let rps = (rcnt - last_rcnt) as f64 / last_dura.as_secs_f64(); + let bw = 8e-9 * (nbytes - last_nbytes) as f64 / last_dura.as_secs_f64(); + if rcnt>args.warmup{ + if args.log_latency { + my_print!( + "Thread {}, {} rps, {} Gb/s, avg: {:?}, median: {:?}, p95: {:?}, p99: {:?}", + tid, + rps, + bw, + Duration::from_nanos(hist.mean() as u64), + Duration::from_nanos(hist.value_at_percentile(50.0)), + Duration::from_nanos(hist.value_at_percentile(95.0)), + Duration::from_nanos(hist.value_at_percentile(99.0)), + ); + hist.clear(); + } else { + my_print!("Thread {}, {} rps, {} Gb/s", tid, rps, bw); + } + } + last_ts = Instant::now(); + last_rcnt = rcnt; + last_nbytes = nbytes; + } + } + complete => break, + default => { + // no futures is ready + if rcnt >= total_iters + args.warmup || start.elapsed() > timeout { + break; + } + let last_dura = last_ts.elapsed(); + if tput_interval.is_some() && last_dura > tput_interval.unwrap() { + let rps = (rcnt - last_rcnt) as f64 / last_dura.as_secs_f64(); + let bw = 8e-9 * (nbytes - last_nbytes) as f64 / last_dura.as_secs_f64(); + if rcnt>args.warmup{ + if args.log_latency { + my_print!( + "Thread {}, {} rps, {} Gb/s, avg: {:?}, median: {:?}, p95: {:?}, p99: {:?}", + tid, + rps, + bw, + Duration::from_nanos(hist.mean() as u64), + Duration::from_nanos(hist.value_at_percentile(50.0)), + Duration::from_nanos(hist.value_at_percentile(95.0)), + Duration::from_nanos(hist.value_at_percentile(99.0)), + ); + hist.clear(); + } else { + my_print!("Thread {}, {} rps, {} Gb/s", tid, rps, bw); + } + } + last_ts = Instant::now(); + last_rcnt = rcnt; + last_nbytes = nbytes; + } + } + } + } + + let dura = warmup_end.elapsed(); + Ok((dura, nbytes, rcnt, hist)) +} + +struct Workload { + reqs: Vec>, + req_sizes: Vec, +} + +impl Workload { + fn new(_args: &Args) -> Self { + let req1 = HelloRequest { + name: "Apple".into(), + }; + let req2 = HelloRequest { + name: "Banana".into(), + }; + let req_sizes = vec![req1.name.len(), req2.name.len()]; + + Self { + reqs: vec![WRef::new(req1), WRef::new(req2)], + req_sizes, + } + } + + fn next_request(&self, scnt: usize) -> (WRef, usize) { + // 1% Apple, 99% Banana + let index = (scnt % 100 >= 1) as usize; + // let index = scnt % self.reqs.len(); + (WRef::clone(&self.reqs[index]), self.req_sizes[index]) + } +} + +fn run_client_thread(tid: usize, args: &Args) -> Result<(), Box> { + macro_rules! my_print { + ($($arg:tt)*) => { + if args.log_level == "info" { + tracing::info!($($arg)*); + } else { + println!($($arg)*); + } + } + } + + // Set transport type + let mut setting = mrpc::current_setting(); + setting.transport = args.transport; + mrpc::set(&setting); + + // bind to NUMA node (tid % num_nodes) + mrpc::bind_to_node((tid % mrpc::num_numa_nodes()) as u8); + + // choose a server + let host = args.connects[tid % args.connects.len()].as_str(); + let port = args.port + (tid % args.num_server_threads) as u16; + + let client = GreeterClient::connect((host, port))?; + eprintln!("connection setup for thread {tid}"); + + smol::block_on(async { + // initialize workload + let workload = Workload::new(args); + + let (dura, total_bytes, rcnt, hist) = run_bench(args, &client, &workload, tid).await?; + + my_print!( + "Thread {tid}, duration: {:?}, bandwidth: {:?} Gb/s, rate: {:.5} Mrps", + dura, + 8e-9 * (total_bytes - args.warmup * args.req_size) as f64 / dura.as_secs_f64(), + 1e-6 * (rcnt - args.warmup) as f64 / dura.as_secs_f64(), + ); + // print latencies + my_print!( + "Thread {tid}, duration: {:?}, avg: {:?}, min: {:?}, median: {:?}, p95: {:?}, p99: {:?}, max: {:?}", + dura, + Duration::from_nanos(hist.mean() as u64), + Duration::from_nanos(hist.min()), + Duration::from_nanos(hist.value_at_percentile(50.0)), + Duration::from_nanos(hist.value_at_percentile(95.0)), + Duration::from_nanos(hist.value_at_percentile(99.0)), + Duration::from_nanos(hist.max()), + ); + + Result::<(), mrpc::Status>::Ok(()) + })?; + + Ok(()) +} + +fn main() -> Result<(), Box> { + let args = Args::from_args(); + eprintln!("args: {:?}", args); + + assert!(args.num_client_threads % args.num_server_threads == 0); + + let _guard = init_tokio_tracing(&args.log_level, &args.log_dir); + + std::thread::scope(|s| { + let mut handles = Vec::new(); + for tid in 1..args.num_client_threads { + let args = &args; + handles.push(s.spawn(move || { + run_client_thread(tid, args).unwrap(); + })); + } + run_client_thread(0, &args).unwrap(); + }); + + Ok(()) +} + +fn init_tokio_tracing( + level: &str, + log_directory: &Option, +) -> tracing_appender::non_blocking::WorkerGuard { + let format = tracing_subscriber::fmt::format() + .with_level(true) + .with_target(true) + .with_thread_ids(false) + .with_thread_names(false) + .with_ansi(false) + .compact(); + + let env_filter = tracing_subscriber::filter::EnvFilter::builder() + .parse(level) + .expect("invalid tracing level"); + + let (non_blocking, appender_guard) = if let Some(log_dir) = log_directory { + let file_appender = tracing_appender::rolling::minutely(log_dir, "rpc-client.log"); + tracing_appender::non_blocking(file_appender) + } else { + tracing_appender::non_blocking(std::io::stdout()) + }; + + tracing_subscriber::fmt::fmt() + .event_format(format) + .with_writer(non_blocking) + .with_env_filter(env_filter) + .init(); + + tracing::info!("tokio_tracing initialized"); + + appender_guard +} diff --git a/experimental/mrpc/load-mrpc-plugins.toml b/experimental/mrpc/load-mrpc-plugins.toml index 1841c039..6aeafef5 100644 --- a/experimental/mrpc/load-mrpc-plugins.toml +++ b/experimental/mrpc/load-mrpc-plugins.toml @@ -80,3 +80,35 @@ name = "HelloAclSender" lib_path = "plugins/libphoenix_hello_acl_sender.rlib" config_string = ''' ''' + +[[addons]] +name = "HelloAcl" +lib_path = "plugins/libphoenix_hello_acl.rlib" +config_string = ''' +''' + +[[addons]] +name = "NofileLogging" +lib_path = "plugins/libphoenix_nofile_logging.rlib" +config_string = ''' +''' + +[[addons]] +name = "Fault" +lib_path = "plugins/libphoenix_fault.rlib" +config_string = ''' +''' + +[[addons]] +name = "Fault2" +lib_path = "plugins/libphoenix_fault2.rlib" +config_string = ''' +''' + +[[addons]] +name = "Delay" +lib_path = "plugins/libphoenix_delay.rlib" +config_string = ''' +delay_probability = 0.2 +delay_ms = 100 +''' diff --git a/experimental/mrpc/phoenix-api/policy/delay/Cargo.toml b/experimental/mrpc/phoenix-api/policy/delay/Cargo.toml new file mode 100644 index 00000000..dedb9488 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/delay/Cargo.toml @@ -0,0 +1,14 @@ + +[package] +name = "phoenix-api-policy-delay" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix-api.workspace = true + +serde.workspace = true +itertools.workspace = true +rand.workspace = true diff --git a/experimental/mrpc/phoenix-api/policy/delay/src/control_plane.rs b/experimental/mrpc/phoenix-api/policy/delay/src/control_plane.rs new file mode 100644 index 00000000..5820428f --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/delay/src/control_plane.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +type IResult = Result; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Request { + NewConfig(f32, u64), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ResponseKind {} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Response(pub IResult); diff --git a/experimental/mrpc/phoenix-api/policy/delay/src/lib.rs b/experimental/mrpc/phoenix-api/policy/delay/src/lib.rs new file mode 100644 index 00000000..412c5a55 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/delay/src/lib.rs @@ -0,0 +1 @@ +pub mod control_plane; diff --git a/experimental/mrpc/phoenix-api/policy/fault/Cargo.toml b/experimental/mrpc/phoenix-api/policy/fault/Cargo.toml new file mode 100644 index 00000000..f4337096 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/fault/Cargo.toml @@ -0,0 +1,14 @@ + +[package] +name = "phoenix-api-policy-fault" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix-api.workspace = true + +serde.workspace = true +itertools.workspace = true +rand.workspace = true diff --git a/experimental/mrpc/phoenix-api/policy/fault/src/control_plane.rs b/experimental/mrpc/phoenix-api/policy/fault/src/control_plane.rs new file mode 100644 index 00000000..4778cd75 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/fault/src/control_plane.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +type IResult = Result; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Request { + NewConfig(), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ResponseKind {} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Response(pub IResult); diff --git a/experimental/mrpc/phoenix-api/policy/fault/src/lib.rs b/experimental/mrpc/phoenix-api/policy/fault/src/lib.rs new file mode 100644 index 00000000..412c5a55 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/fault/src/lib.rs @@ -0,0 +1 @@ +pub mod control_plane; diff --git a/experimental/mrpc/phoenix-api/policy/fault2/Cargo.toml b/experimental/mrpc/phoenix-api/policy/fault2/Cargo.toml new file mode 100644 index 00000000..01c851cc --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/fault2/Cargo.toml @@ -0,0 +1,14 @@ + +[package] +name = "phoenix-api-policy-fault2" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix-api.workspace = true + +serde.workspace = true +itertools.workspace = true +rand.workspace = true diff --git a/experimental/mrpc/phoenix-api/policy/fault2/src/control_plane.rs b/experimental/mrpc/phoenix-api/policy/fault2/src/control_plane.rs new file mode 100644 index 00000000..4778cd75 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/fault2/src/control_plane.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +type IResult = Result; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Request { + NewConfig(), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ResponseKind {} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Response(pub IResult); diff --git a/experimental/mrpc/phoenix-api/policy/fault2/src/lib.rs b/experimental/mrpc/phoenix-api/policy/fault2/src/lib.rs new file mode 100644 index 00000000..412c5a55 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/fault2/src/lib.rs @@ -0,0 +1 @@ +pub mod control_plane; diff --git a/experimental/mrpc/phoenix-api/policy/hello-acl/Cargo.toml b/experimental/mrpc/phoenix-api/policy/hello-acl/Cargo.toml new file mode 100644 index 00000000..ff6d943d --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/hello-acl/Cargo.toml @@ -0,0 +1,13 @@ + +[package] +name = "phoenix-api-policy-hello-acl" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix-api.workspace = true + +serde.workspace = true +itertools.workspace = true diff --git a/experimental/mrpc/phoenix-api/policy/hello-acl/src/control_plane.rs b/experimental/mrpc/phoenix-api/policy/hello-acl/src/control_plane.rs new file mode 100644 index 00000000..4778cd75 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/hello-acl/src/control_plane.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +type IResult = Result; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Request { + NewConfig(), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ResponseKind {} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Response(pub IResult); diff --git a/experimental/mrpc/phoenix-api/policy/hello-acl/src/lib.rs b/experimental/mrpc/phoenix-api/policy/hello-acl/src/lib.rs new file mode 100644 index 00000000..412c5a55 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/hello-acl/src/lib.rs @@ -0,0 +1 @@ +pub mod control_plane; diff --git a/experimental/mrpc/phoenix-api/policy/nofile-logging/Cargo.toml b/experimental/mrpc/phoenix-api/policy/nofile-logging/Cargo.toml new file mode 100644 index 00000000..9dffc571 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/nofile-logging/Cargo.toml @@ -0,0 +1,14 @@ + +[package] +name = "phoenix-api-policy-nofile-logging" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix-api.workspace = true + +serde.workspace = true +itertools.workspace = true +rand.workspace = true diff --git a/experimental/mrpc/phoenix-api/policy/nofile-logging/src/control_plane.rs b/experimental/mrpc/phoenix-api/policy/nofile-logging/src/control_plane.rs new file mode 100644 index 00000000..4778cd75 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/nofile-logging/src/control_plane.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +type IResult = Result; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Request { + NewConfig(), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ResponseKind {} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Response(pub IResult); diff --git a/experimental/mrpc/phoenix-api/policy/nofile-logging/src/lib.rs b/experimental/mrpc/phoenix-api/policy/nofile-logging/src/lib.rs new file mode 100644 index 00000000..412c5a55 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/nofile-logging/src/lib.rs @@ -0,0 +1 @@ +pub mod control_plane; diff --git a/experimental/mrpc/plugin/policy/delay/Cargo.toml b/experimental/mrpc/plugin/policy/delay/Cargo.toml new file mode 100644 index 00000000..dd57021e --- /dev/null +++ b/experimental/mrpc/plugin/policy/delay/Cargo.toml @@ -0,0 +1,28 @@ + +[package] +name = "phoenix-delay" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix_common.workspace = true +phoenix-api-policy-delay.workspace = true +mrpc-marshal.workspace = true +mrpc-derive.workspace = true +shm.workspace = true +phoenix-api = { workspace = true, features = ["mrpc"] } + +futures.workspace = true +minstant.workspace = true +thiserror.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +nix.workspace = true +toml = { workspace = true, features = ["preserve_order"] } +bincode.workspace = true +chrono.workspace = true +itertools.workspace = true +rand.workspace = true diff --git a/experimental/mrpc/plugin/policy/delay/src/config.rs b/experimental/mrpc/plugin/policy/delay/src/config.rs new file mode 100644 index 00000000..87d08086 --- /dev/null +++ b/experimental/mrpc/plugin/policy/delay/src/config.rs @@ -0,0 +1,25 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DelayConfig { + pub delay_probability: f32, + pub delay_ms: u64, +} + +impl Default for DelayConfig { + fn default() -> Self { + DelayConfig { + delay_probability: 0.2, + delay_ms: 100, + } + } +} + +impl DelayConfig { + /// Get config from toml file + pub fn new(config: Option<&str>) -> anyhow::Result { + let config = toml::from_str(config.unwrap_or(""))?; + Ok(config) + } +} diff --git a/experimental/mrpc/plugin/policy/delay/src/engine.rs b/experimental/mrpc/plugin/policy/delay/src/engine.rs new file mode 100644 index 00000000..652a8ad4 --- /dev/null +++ b/experimental/mrpc/plugin/policy/delay/src/engine.rs @@ -0,0 +1,212 @@ +use std::collections::VecDeque; +use std::os::unix::ucred::UCred; +use std::pin::Pin; + +use anyhow::{anyhow, Result}; +use futures::future::BoxFuture; +use minstant::Instant; + +use phoenix_api_policy_delay::control_plane; + +use phoenix_common::engine::datapath::message::{EngineTxMessage, RpcMessageTx}; +use phoenix_common::engine::datapath::node::DataPathNode; +use phoenix_common::engine::{future, Decompose, Engine, EngineResult, Indicator, Vertex}; +use phoenix_common::envelop::ResourceDowncast; +use phoenix_common::impl_vertex_for_engine; +use phoenix_common::module::Version; +use phoenix_common::storage::{ResourceCollection, SharedStorage}; + +use super::DatapathError; +use crate::config::DelayConfig; + +pub(crate) struct DelayRpcInfo { + pub(crate) msg: RpcMessageTx, + pub(crate) timestamp: Instant, +} + +pub(crate) struct DelayEngine { + pub(crate) node: DataPathNode, + pub(crate) indicator: Indicator, + pub(crate) config: DelayConfig, + // The probability of delaying an RPC. + pub(crate) delay_probability: f32, + // Delaying time (in ms). + pub(crate) delay_ms: u64, + // The queue to buffer delayed requests. + pub(crate) queue: VecDeque, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + Progress(usize), + Disconnected, +} + +use Status::Progress; + +impl Engine for DelayEngine { + fn activate<'a>(self: Pin<&'a mut Self>) -> BoxFuture<'a, EngineResult> { + Box::pin(async move { self.get_mut().mainloop().await }) + } + + fn description(self: Pin<&Self>) -> String { + "DelayEngine".to_owned() + } + + #[inline] + fn tracker(self: Pin<&mut Self>) -> &mut Indicator { + &mut self.get_mut().indicator + } + + fn handle_request(&mut self, request: Vec, _cred: UCred) -> Result<()> { + let request: control_plane::Request = bincode::deserialize(&request[..])?; + + match request { + control_plane::Request::NewConfig(delay_probability, delay_ms) => { + self.config = DelayConfig { + delay_probability, + delay_ms, + }; + } + } + Ok(()) + } +} + +impl_vertex_for_engine!(DelayEngine, node); + +impl Decompose for DelayEngine { + fn flush(&mut self) -> Result { + let mut work = 0; + while !self.tx_inputs()[0].is_empty() { + if let Progress(n) = self.check_input_queue()? { + work += n; + } + } + while !self.queue.is_empty() { + let DelayRpcInfo { msg, .. } = self.queue.pop_front().unwrap(); + self.tx_outputs()[0].send(EngineTxMessage::RpcMessage(msg))?; + work += 1; + } + Ok(work) + } + + fn decompose( + self: Box, + _shared: &mut SharedStorage, + _global: &mut ResourceCollection, + ) -> (ResourceCollection, DataPathNode) { + let engine = *self; + let mut collections = ResourceCollection::with_capacity(4); + collections.insert("config".to_string(), Box::new(engine.config)); + collections.insert( + "delay_probability".to_string(), + Box::new(engine.delay_probability), + ); + collections.insert("delay_ms".to_string(), Box::new(engine.delay_ms)); + collections.insert("queue".to_string(), Box::new(engine.queue)); + (collections, engine.node) + } +} + +impl DelayEngine { + pub(crate) fn restore( + mut local: ResourceCollection, + node: DataPathNode, + _prev_version: Version, + ) -> Result { + let config = *local + .remove("config") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let delay_probability = *local + .remove("delay_probability") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let delay_ms = *local + .remove("delay_ms") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let queue = *local + .remove("queue") + .unwrap() + .downcast::>() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + + let engine = DelayEngine { + node, + indicator: Default::default(), + config, + delay_probability, + delay_ms, + queue, + }; + Ok(engine) + } +} + +impl DelayEngine { + async fn mainloop(&mut self) -> EngineResult { + loop { + let mut work = 0; + loop { + match self.check_input_queue()? { + Progress(0) => break, + Progress(n) => work += n, + Status::Disconnected => return Ok(()), + } + } + self.check_delay_buffer()?; + self.indicator.set_nwork(work); + future::yield_now().await; + } + } +} + +impl DelayEngine { + fn check_delay_buffer(&mut self) -> Result<(), DatapathError> { + while !self.queue.is_empty() { + let oldest_msg = self.queue.pop_front().unwrap(); + if oldest_msg.timestamp.elapsed().as_millis() as u64 > self.delay_ms { + self.tx_outputs()[0].send(EngineTxMessage::RpcMessage(oldest_msg.msg))?; + } else { + self.queue.push_front(oldest_msg); + break; + } + } + Ok(()) + } + + fn check_input_queue(&mut self) -> Result { + use phoenix_common::engine::datapath::TryRecvError; + + match self.tx_inputs()[0].try_recv() { + Ok(msg) => { + match msg { + EngineTxMessage::RpcMessage(msg) => { + if rand::random::() < self.delay_probability { + let delay_msg = DelayRpcInfo { + msg: msg, + timestamp: Instant::now(), + }; + self.queue.push_back(delay_msg); + } else { + self.tx_outputs()[0].send(EngineTxMessage::RpcMessage(msg))?; + } + } + m => self.tx_outputs()[0].send(m)?, + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + return Ok(Status::Disconnected); + } + } + + Ok(Progress(0)) + } +} diff --git a/experimental/mrpc/plugin/policy/delay/src/lib.rs b/experimental/mrpc/plugin/policy/delay/src/lib.rs new file mode 100644 index 00000000..901438ea --- /dev/null +++ b/experimental/mrpc/plugin/policy/delay/src/lib.rs @@ -0,0 +1,33 @@ +#![feature(peer_credentials_unix_socket)] +#![feature(ptr_internals)] +#![feature(strict_provenance)] +use thiserror::Error; + +pub use phoenix_common::{InitFnResult, PhoenixAddon}; + +pub mod config; +pub(crate) mod engine; +pub mod module; + +#[derive(Error, Debug)] +pub(crate) enum DatapathError { + #[error("Internal queue send error")] + InternalQueueSend, +} + +use phoenix_common::engine::datapath::SendError; +impl From> for DatapathError { + fn from(_other: SendError) -> Self { + DatapathError::InternalQueueSend + } +} + +use crate::config::DelayConfig; +use crate::module::DelayAddon; + +#[no_mangle] +pub fn init_addon(config_string: Option<&str>) -> InitFnResult> { + let config = DelayConfig::new(config_string)?; + let addon = DelayAddon::new(config); + Ok(Box::new(addon)) +} diff --git a/experimental/mrpc/plugin/policy/delay/src/module.rs b/experimental/mrpc/plugin/policy/delay/src/module.rs new file mode 100644 index 00000000..ec08477a --- /dev/null +++ b/experimental/mrpc/plugin/policy/delay/src/module.rs @@ -0,0 +1,104 @@ +use std::collections::VecDeque; + +use anyhow::{bail, Result}; +use nix::unistd::Pid; + +use phoenix_common::addon::{PhoenixAddon, Version}; +use phoenix_common::engine::datapath::DataPathNode; +use phoenix_common::engine::{Engine, EngineType}; +use phoenix_common::storage::ResourceCollection; + +use super::engine::DelayEngine; +use crate::config::DelayConfig; + +pub(crate) struct DelayEngineBuilder { + node: DataPathNode, + config: DelayConfig, +} + +impl DelayEngineBuilder { + fn new(node: DataPathNode, config: DelayConfig) -> Self { + DelayEngineBuilder { node, config } + } + // TODO! LogFile + fn build(self) -> Result { + Ok(DelayEngine { + node: self.node, + indicator: Default::default(), + config: self.config, + delay_probability: self.config.delay_probability as _, + delay_ms: self.config.delay_ms as _, + queue: VecDeque::new(), + }) + } +} + +pub struct DelayAddon { + config: DelayConfig, +} + +impl DelayAddon { + pub const DELAY_ENGINE: EngineType = EngineType("DelayEngine"); + pub const ENGINES: &'static [EngineType] = &[DelayAddon::DELAY_ENGINE]; +} + +impl DelayAddon { + pub fn new(config: DelayConfig) -> Self { + DelayAddon { config } + } +} + +impl PhoenixAddon for DelayAddon { + fn check_compatibility(&self, _prev: Option<&Version>) -> bool { + true + } + + fn decompose(self: Box) -> ResourceCollection { + let addon = *self; + let mut collections = ResourceCollection::new(); + collections.insert("config".to_string(), Box::new(addon.config)); + collections + } + + #[inline] + fn migrate(&mut self, _prev_addon: Box) {} + + fn engines(&self) -> &[EngineType] { + DelayAddon::ENGINES + } + + fn update_config(&mut self, config: &str) -> Result<()> { + self.config = toml::from_str(config)?; + Ok(()) + } + + fn create_engine( + &mut self, + ty: EngineType, + _pid: Pid, + node: DataPathNode, + ) -> Result> { + if ty != DelayAddon::DELAY_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let builder = DelayEngineBuilder::new(node, self.config); + let engine = builder.build()?; + Ok(Box::new(engine)) + } + + fn restore_engine( + &mut self, + ty: EngineType, + local: ResourceCollection, + node: DataPathNode, + prev_version: Version, + ) -> Result> { + if ty != DelayAddon::DELAY_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let engine = DelayEngine::restore(local, node, prev_version)?; + Ok(Box::new(engine)) + } +} diff --git a/experimental/mrpc/plugin/policy/example-hello-acl/Cargo.toml b/experimental/mrpc/plugin/policy/example-hello-acl/Cargo.toml new file mode 100644 index 00000000..3f762fd6 --- /dev/null +++ b/experimental/mrpc/plugin/policy/example-hello-acl/Cargo.toml @@ -0,0 +1,27 @@ + +[package] +name = "phoenix-hello-acl" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix_common.workspace = true +phoenix-api-policy-hello-acl.workspace = true +mrpc-marshal.workspace = true +mrpc-derive.workspace = true +shm.workspace = true +phoenix-api = { workspace = true, features = ["mrpc"] } + +futures.workspace = true +minstant.workspace = true +thiserror.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +nix.workspace = true +toml = { workspace = true, features = ["preserve_order"] } +bincode.workspace = true +chrono.workspace = true +itertools.workspace = true diff --git a/experimental/mrpc/plugin/policy/example-hello-acl/src/config.rs b/experimental/mrpc/plugin/policy/example-hello-acl/src/config.rs new file mode 100644 index 00000000..661ec63b --- /dev/null +++ b/experimental/mrpc/plugin/policy/example-hello-acl/src/config.rs @@ -0,0 +1,39 @@ +use chrono::{Datelike, Timelike, Utc}; +use phoenix_common::log; +use serde::{Deserialize, Serialize}; + +use chrono::prelude::*; +use itertools::iproduct; +use phoenix_common::engine::datapath::RpcMessageTx; + +use crate::engine::struct_acl; + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HelloAclConfig {} + +impl HelloAclConfig { + /// Get config from toml file + pub fn new(config: Option<&str>) -> anyhow::Result { + let config = toml::from_str(config.unwrap_or(""))?; + Ok(config) + } +} + +pub fn create_log_file() -> std::fs::File { + std::fs::create_dir_all("/tmp/phoenix/log").expect("mkdir failed"); + let now = Utc::now(); + let date_string = format!( + "{}-{}-{}-{}-{}-{}", + now.year(), + now.month(), + now.day(), + now.hour(), + now.minute(), + now.second() + ); + let file_name = format!("/tmp/phoenix/log/logging_engine_{}.log", date_string); + ///log::info!("create log file {}", file_name); + let log_file = std::fs::File::create(file_name).expect("create file failed"); + log_file +} diff --git a/experimental/mrpc/plugin/policy/example-hello-acl/src/engine.rs b/experimental/mrpc/plugin/policy/example-hello-acl/src/engine.rs new file mode 100644 index 00000000..1f5fe76b --- /dev/null +++ b/experimental/mrpc/plugin/policy/example-hello-acl/src/engine.rs @@ -0,0 +1,277 @@ +use anyhow::{anyhow, Result}; +use futures::future::BoxFuture; +use phoenix_api::rpc::{RpcId, TransportStatus}; +use std::fmt; +use std::fs::File; +use std::io::Write; +use std::num::NonZeroU32; +use std::os::unix::ucred::UCred; +use std::pin::Pin; + +use phoenix_api_policy_hello_acl::control_plane; + +use phoenix_common::engine::datapath::message::{ + EngineRxMessage, EngineTxMessage, RpcMessageGeneral, +}; + +use phoenix_common::engine::datapath::node::DataPathNode; +use phoenix_common::engine::{future, Decompose, Engine, EngineResult, Indicator, Vertex}; +use phoenix_common::envelop::ResourceDowncast; +use phoenix_common::impl_vertex_for_engine; +use phoenix_common::log; +use phoenix_common::module::Version; + +use phoenix_common::storage::{ResourceCollection, SharedStorage}; + +use super::DatapathError; +use crate::config::{create_log_file, HelloAclConfig}; + +use chrono::prelude::*; +use itertools::iproduct; +use phoenix_common::engine::datapath::RpcMessageTx; + +pub mod hello { + // The string specified here must match the proto package name + include!("proto.rs"); +} + +pub struct struct_acl { + pub name: String, + pub permission: String, +} +impl struct_acl { + pub fn new(name: String, permission: String) -> struct_acl { + struct_acl { + name: name, + permission: permission, + } + } +} + +pub(crate) struct HelloAclEngine { + pub(crate) node: DataPathNode, + pub(crate) indicator: Indicator, + pub(crate) config: HelloAclConfig, + pub(crate) table_acl: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + Progress(usize), + Disconnected, +} + +use Status::Progress; + +impl Engine for HelloAclEngine { + fn activate<'a>(self: Pin<&'a mut Self>) -> BoxFuture<'a, EngineResult> { + Box::pin(async move { self.get_mut().mainloop().await }) + } + + fn description(self: Pin<&Self>) -> String { + "HelloAclEngine".to_owned() + } + + #[inline] + fn tracker(self: Pin<&mut Self>) -> &mut Indicator { + &mut self.get_mut().indicator + } + + fn handle_request(&mut self, request: Vec, _cred: UCred) -> Result<()> { + let request: control_plane::Request = bincode::deserialize(&request[..])?; + + match request { + control_plane::Request::NewConfig() => { + self.config = HelloAclConfig {}; + } + } + Ok(()) + } +} + +impl_vertex_for_engine!(HelloAclEngine, node); + +impl Decompose for HelloAclEngine { + fn flush(&mut self) -> Result { + let mut work = 0; + while !self.tx_inputs()[0].is_empty() || !self.rx_inputs()[0].is_empty() { + if let Progress(n) = self.check_input_queue()? { + work += n; + } + } + Ok(work) + } + + fn decompose( + self: Box, + _shared: &mut SharedStorage, + _global: &mut ResourceCollection, + ) -> (ResourceCollection, DataPathNode) { + let engine = *self; + let mut collections = ResourceCollection::with_capacity(4); + collections.insert("config".to_string(), Box::new(engine.config)); + (collections, engine.node) + } +} + +impl HelloAclEngine { + pub(crate) fn restore( + mut local: ResourceCollection, + node: DataPathNode, + _prev_version: Version, + ) -> Result { + let config = *local + .remove("config") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let mut table_acl = Vec::new(); + table_acl.push(struct_acl { + name: "Apple".to_string(), + permission: "N".to_string(), + }); + table_acl.push(struct_acl { + name: "Banana".to_string(), + permission: "Y".to_string(), + }); + + let engine = HelloAclEngine { + node, + indicator: Default::default(), + config, + table_acl, + }; + Ok(engine) + } +} + +impl HelloAclEngine { + async fn mainloop(&mut self) -> EngineResult { + loop { + let mut work = 0; + loop { + match self.check_input_queue()? { + Progress(0) => break, + Progress(n) => work += n, + Status::Disconnected => return Ok(()), + } + } + self.indicator.set_nwork(work); + future::yield_now().await; + } + } +} + +fn hello_request_name_readonly(req: &hello::HelloRequest) -> String { + let buf = &req.name as &[u8]; + String::from_utf8_lossy(buf).to_string().clone() +} + +/// Copy the RPC request to a private heap and returns the request. +// #[inline] +// fn materialize(msg: &RpcMessageTx) -> Box { +// let req_ptr = Unique::new(msg.addr_backend as *mut hello::HelloRequest).unwrap(); +// let req = unsafe { req_ptr.as_ref() }; +// // returns a private_req +// Box::new(req.clone()) +// } + +#[inline] +fn materialize_nocopy(msg: &RpcMessageTx) -> &hello::HelloRequest { + let req_ptr = msg.addr_backend as *mut hello::HelloRequest; + let req = unsafe { req_ptr.as_ref().unwrap() }; + // returns a private_req + return req; +} + +impl HelloAclEngine { + fn check_input_queue(&mut self) -> Result { + use phoenix_common::engine::datapath::TryRecvError; + + match self.tx_inputs()[0].try_recv() { + Ok(msg) => { + match msg { + EngineTxMessage::RpcMessage(msg) => { + let mut input = Vec::new(); + input.push(msg); + let output: Vec<_> = iproduct!(input.iter(), self.table_acl.iter()) + .map(|(msg, acl)| { + let rpc_message = materialize_nocopy(&msg); + let conn_id = unsafe { &*msg.meta_buf_ptr.as_meta_ptr() }.conn_id; + let call_id = unsafe { &*msg.meta_buf_ptr.as_meta_ptr() }.call_id; + let rpc_id = RpcId::new(conn_id, call_id); + + if hello_request_name_readonly(rpc_message) == acl.name { + if acl.permission != "Y" { + // reject + let error = EngineRxMessage::Ack( + rpc_id, + TransportStatus::Error(unsafe { + NonZeroU32::new_unchecked(403) + }), + ); + log::debug!("tx input reject msg:{:?}", error); + RpcMessageGeneral::RxMessage(error) + } else { + let raw_ptr: *const hello::HelloRequest = rpc_message; + let new_msg = RpcMessageTx { + meta_buf_ptr: msg.meta_buf_ptr.clone(), + addr_backend: raw_ptr.addr(), + }; + log::debug!("tx input accept msg:{:?}", new_msg); + RpcMessageGeneral::TxMessage(EngineTxMessage::RpcMessage( + new_msg, + )) + } + } else { + RpcMessageGeneral::Pass + } + }) + .collect(); + + for msg in output { + log::info!("sent msg:{:?}", msg); + match msg { + RpcMessageGeneral::TxMessage(msg) => { + self.tx_outputs()[0].send(msg).unwrap(); + } + RpcMessageGeneral::RxMessage(msg) => { + self.rx_outputs()[0].send(msg).unwrap(); + } + _ => {} + } + } + } + m => self.tx_outputs()[0].send(m)?, + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + return Ok(Status::Disconnected); + } + } + + match self.rx_inputs()[0].try_recv() { + Ok(msg) => { + match msg { + EngineRxMessage::Ack(rpc_id, status) => { + log::debug!("rx input ack msg:{:?}", msg); + self.rx_outputs()[0].send(EngineRxMessage::Ack(rpc_id, status))?; + } + EngineRxMessage::RpcMessage(msg) => { + log::debug!("rx input rpc msg:{:?}", msg); + self.rx_outputs()[0].send(EngineRxMessage::RpcMessage(msg))?; + } + m => self.rx_outputs()[0].send(m)?, + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + return Ok(Status::Disconnected); + } + } + Ok(Progress(0)) + } +} diff --git a/experimental/mrpc/plugin/policy/example-hello-acl/src/lib.rs b/experimental/mrpc/plugin/policy/example-hello-acl/src/lib.rs new file mode 100644 index 00000000..242f6294 --- /dev/null +++ b/experimental/mrpc/plugin/policy/example-hello-acl/src/lib.rs @@ -0,0 +1,37 @@ +#![feature(peer_credentials_unix_socket)] +#![feature(strict_provenance)] +use crate::engine::struct_acl; +use thiserror::Error; + +use chrono::prelude::*; +use itertools::iproduct; +use phoenix_common::engine::datapath::RpcMessageTx; + +pub use phoenix_common::{InitFnResult, PhoenixAddon}; + +pub mod config; +pub(crate) mod engine; +pub mod module; + +#[derive(Error, Debug)] +pub(crate) enum DatapathError { + #[error("Internal queue send error")] + InternalQueueSend, +} + +use phoenix_common::engine::datapath::SendError; +impl From> for DatapathError { + fn from(_other: SendError) -> Self { + DatapathError::InternalQueueSend + } +} + +use crate::config::HelloAclConfig; +use crate::module::HelloAclAddon; + +#[no_mangle] +pub fn init_addon(config_string: Option<&str>) -> InitFnResult> { + let config = HelloAclConfig::new(config_string)?; + let addon = HelloAclAddon::new(config); + Ok(Box::new(addon)) +} diff --git a/experimental/mrpc/plugin/policy/example-hello-acl/src/module.rs b/experimental/mrpc/plugin/policy/example-hello-acl/src/module.rs new file mode 100644 index 00000000..382eb340 --- /dev/null +++ b/experimental/mrpc/plugin/policy/example-hello-acl/src/module.rs @@ -0,0 +1,115 @@ +use anyhow::{bail, Result}; +use nix::unistd::Pid; + +use phoenix_common::addon::{PhoenixAddon, Version}; +use phoenix_common::engine::datapath::DataPathNode; +use phoenix_common::engine::{Engine, EngineType}; +use phoenix_common::storage::ResourceCollection; + +use super::engine::HelloAclEngine; +use crate::config::{create_log_file, HelloAclConfig}; +use crate::engine::struct_acl; + +use chrono::prelude::*; +use itertools::iproduct; +use phoenix_common::engine::datapath::RpcMessageTx; + +pub(crate) struct HelloAclEngineBuilder { + node: DataPathNode, + config: HelloAclConfig, +} + +impl HelloAclEngineBuilder { + fn new(node: DataPathNode, config: HelloAclConfig) -> Self { + HelloAclEngineBuilder { node, config } + } + // TODO! LogFile + fn build(self) -> Result { + let mut table_acl = Vec::new(); + table_acl.push(struct_acl { + name: "Apple".to_string(), + permission: "N".to_string(), + }); + table_acl.push(struct_acl { + name: "Banana".to_string(), + permission: "Y".to_string(), + }); + + Ok(HelloAclEngine { + node: self.node, + indicator: Default::default(), + config: self.config, + table_acl, + }) + } +} + +pub struct HelloAclAddon { + config: HelloAclConfig, +} + +impl HelloAclAddon { + pub const HELLO_ACL_ENGINE: EngineType = EngineType("HelloAclEngine"); + pub const ENGINES: &'static [EngineType] = &[HelloAclAddon::HELLO_ACL_ENGINE]; +} + +impl HelloAclAddon { + pub fn new(config: HelloAclConfig) -> Self { + HelloAclAddon { config } + } +} + +impl PhoenixAddon for HelloAclAddon { + fn check_compatibility(&self, _prev: Option<&Version>) -> bool { + true + } + + fn decompose(self: Box) -> ResourceCollection { + let addon = *self; + let mut collections = ResourceCollection::new(); + collections.insert("config".to_string(), Box::new(addon.config)); + collections + } + + #[inline] + fn migrate(&mut self, _prev_addon: Box) {} + + fn engines(&self) -> &[EngineType] { + HelloAclAddon::ENGINES + } + + fn update_config(&mut self, config: &str) -> Result<()> { + self.config = toml::from_str(config)?; + Ok(()) + } + + fn create_engine( + &mut self, + ty: EngineType, + _pid: Pid, + node: DataPathNode, + ) -> Result> { + if ty != HelloAclAddon::HELLO_ACL_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let builder = HelloAclEngineBuilder::new(node, self.config); + let engine = builder.build()?; + Ok(Box::new(engine)) + } + + fn restore_engine( + &mut self, + ty: EngineType, + local: ResourceCollection, + node: DataPathNode, + prev_version: Version, + ) -> Result> { + if ty != HelloAclAddon::HELLO_ACL_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let engine = HelloAclEngine::restore(local, node, prev_version)?; + Ok(Box::new(engine)) + } +} diff --git a/experimental/mrpc/plugin/policy/example-hello-acl/src/proto.rs b/experimental/mrpc/plugin/policy/example-hello-acl/src/proto.rs new file mode 100644 index 00000000..83ee8ab0 --- /dev/null +++ b/experimental/mrpc/plugin/policy/example-hello-acl/src/proto.rs @@ -0,0 +1,29 @@ +/// The request message containing the user's name. +#[repr(C)] +#[derive(Debug, Clone, ::mrpc_derive::Message)] +pub struct HelloRequest { + #[prost(bytes = "vec", tag = "1")] + pub name: ::mrpc_marshal::shadow::Vec, +} +/// The response message containing the greetings +#[repr(C)] +#[derive(Debug, ::mrpc_derive::Message)] +pub struct HelloReply { + #[prost(bytes = "vec", tag = "1")] + pub message: ::mrpc_marshal::shadow::Vec, +} + +// /// The request message containing the user's name. +// #[repr(C)] +// #[derive(Debug, Clone, ::mrpc_derive::Message)] +// pub struct HelloRequest { +// #[prost(bytes = "vec", tag = "1")] +// pub name: ::mrpc::alloc::Vec, +// } +// /// The response message containing the greetings +// #[repr(C)] +// #[derive(Debug, ::mrpc_derive::Message)] +// pub struct HelloReply { +// #[prost(bytes = "vec", tag = "1")] +// pub message: ::mrpc::alloc::Vec, +// } diff --git a/experimental/mrpc/plugin/policy/fault/Cargo.toml b/experimental/mrpc/plugin/policy/fault/Cargo.toml new file mode 100644 index 00000000..ba70e092 --- /dev/null +++ b/experimental/mrpc/plugin/policy/fault/Cargo.toml @@ -0,0 +1,28 @@ + +[package] +name = "phoenix-fault" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix_common.workspace = true +phoenix-api-policy-fault.workspace = true +mrpc-marshal.workspace = true +mrpc-derive.workspace = true +shm.workspace = true +phoenix-api = { workspace = true, features = ["mrpc"] } + +futures.workspace = true +minstant.workspace = true +thiserror.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +nix.workspace = true +toml = { workspace = true, features = ["preserve_order"] } +bincode.workspace = true +chrono.workspace = true +itertools.workspace = true +rand.workspace = true diff --git a/experimental/mrpc/plugin/policy/fault/src/config.rs b/experimental/mrpc/plugin/policy/fault/src/config.rs new file mode 100644 index 00000000..265cf886 --- /dev/null +++ b/experimental/mrpc/plugin/policy/fault/src/config.rs @@ -0,0 +1,37 @@ +use chrono::{Datelike, Timelike, Utc}; +use phoenix_common::log; +use serde::{Deserialize, Serialize}; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FaultConfig {} + +impl FaultConfig { + /// Get config from toml file + pub fn new(config: Option<&str>) -> anyhow::Result { + let config = toml::from_str(config.unwrap_or(""))?; + Ok(config) + } +} + +pub fn create_log_file() -> std::fs::File { + std::fs::create_dir_all("/tmp/phoenix/log").expect("mkdir failed"); + let now = Utc::now(); + let date_string = format!( + "{}-{}-{}-{}-{}-{}", + now.year(), + now.month(), + now.day(), + now.hour(), + now.minute(), + now.second() + ); + let file_name = format!("/tmp/phoenix/log/logging_engine_{}.log", date_string); + ///log::info!("create log file {}", file_name); + let log_file = std::fs::File::create(file_name).expect("create file failed"); + log_file +} diff --git a/experimental/mrpc/plugin/policy/fault/src/engine.rs b/experimental/mrpc/plugin/policy/fault/src/engine.rs new file mode 100644 index 00000000..3a8f46e7 --- /dev/null +++ b/experimental/mrpc/plugin/policy/fault/src/engine.rs @@ -0,0 +1,238 @@ +use anyhow::{anyhow, Result}; +use futures::future::BoxFuture; +use phoenix_api::rpc::{RpcId, TransportStatus}; +use std::fmt; +use std::fs::File; +use std::io::Write; +use std::num::NonZeroU32; +use std::os::unix::ucred::UCred; +use std::pin::Pin; + +use phoenix_api_policy_fault::control_plane; + +use phoenix_common::engine::datapath::message::{ + EngineRxMessage, EngineTxMessage, RpcMessageGeneral, +}; + +use phoenix_common::engine::datapath::node::DataPathNode; +use phoenix_common::engine::{future, Decompose, Engine, EngineResult, Indicator, Vertex}; +use phoenix_common::envelop::ResourceDowncast; +use phoenix_common::impl_vertex_for_engine; +use phoenix_common::log; +use phoenix_common::module::Version; + +use phoenix_common::engine::datapath::RpcMessageTx; +use phoenix_common::storage::{ResourceCollection, SharedStorage}; + +use super::DatapathError; +use crate::config::{create_log_file, FaultConfig}; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +pub mod hello { + include!("proto.rs"); +} + +fn hello_request_name_readonly(req: &hello::HelloRequest) -> String { + let buf = &req.name as &[u8]; + String::from_utf8_lossy(buf).to_string().clone() +} + +pub(crate) struct FaultEngine { + pub(crate) node: DataPathNode, + pub(crate) indicator: Indicator, + pub(crate) config: FaultConfig, + pub(crate) var_probability: f32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + Progress(usize), + Disconnected, +} + +use Status::Progress; + +impl Engine for FaultEngine { + fn activate<'a>(self: Pin<&'a mut Self>) -> BoxFuture<'a, EngineResult> { + Box::pin(async move { self.get_mut().mainloop().await }) + } + + fn description(self: Pin<&Self>) -> String { + "FaultEngine".to_owned() + } + + #[inline] + fn tracker(self: Pin<&mut Self>) -> &mut Indicator { + &mut self.get_mut().indicator + } + + fn handle_request(&mut self, request: Vec, _cred: UCred) -> Result<()> { + let request: control_plane::Request = bincode::deserialize(&request[..])?; + + match request { + control_plane::Request::NewConfig() => { + self.config = FaultConfig {}; + } + } + Ok(()) + } +} + +impl_vertex_for_engine!(FaultEngine, node); + +impl Decompose for FaultEngine { + fn flush(&mut self) -> Result { + let mut work = 0; + while !self.tx_inputs()[0].is_empty() || !self.rx_inputs()[0].is_empty() { + if let Progress(n) = self.check_input_queue()? { + work += n; + } + } + Ok(work) + } + + fn decompose( + self: Box, + _shared: &mut SharedStorage, + _global: &mut ResourceCollection, + ) -> (ResourceCollection, DataPathNode) { + let engine = *self; + let mut collections = ResourceCollection::with_capacity(4); + collections.insert("config".to_string(), Box::new(engine.config)); + (collections, engine.node) + } +} + +impl FaultEngine { + pub(crate) fn restore( + mut local: ResourceCollection, + node: DataPathNode, + _prev_version: Version, + ) -> Result { + let config = *local + .remove("config") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let var_probability = 0.01; + + let engine = FaultEngine { + node, + indicator: Default::default(), + config, + var_probability, + }; + Ok(engine) + } +} + +impl FaultEngine { + async fn mainloop(&mut self) -> EngineResult { + loop { + let mut work = 0; + loop { + match self.check_input_queue()? { + Progress(0) => break, + Progress(n) => work += n, + Status::Disconnected => return Ok(()), + } + } + self.indicator.set_nwork(work); + future::yield_now().await; + } + } +} + +#[inline] +fn materialize_nocopy(msg: &RpcMessageTx) -> &hello::HelloRequest { + let req_ptr = msg.addr_backend as *mut hello::HelloRequest; + let req = unsafe { req_ptr.as_ref().unwrap() }; + return req; +} + +impl FaultEngine { + fn check_input_queue(&mut self) -> Result { + use phoenix_common::engine::datapath::TryRecvError; + + match self.tx_inputs()[0].try_recv() { + Ok(msg) => { + match msg { + EngineTxMessage::RpcMessage(msg) => { + let meta_ref = unsafe { &*msg.meta_buf_ptr.as_meta_ptr() }; + let mut input = Vec::new(); + input.push(msg); + let output: Vec<_> = input + .iter() + .map(|msg| { + let rpc_message = materialize_nocopy(&msg); + let conn_id = unsafe { &*msg.meta_buf_ptr.as_meta_ptr() }.conn_id; + let call_id = unsafe { &*msg.meta_buf_ptr.as_meta_ptr() }.call_id; + let rpc_id = RpcId::new(conn_id, call_id); + if rand::random::() < self.var_probability { + let error = EngineRxMessage::Ack( + rpc_id, + TransportStatus::Error(unsafe { + NonZeroU32::new_unchecked(403) + }), + ); + RpcMessageGeneral::RxMessage(error) + } else { + let raw_ptr: *const hello::HelloRequest = rpc_message; + let new_msg = RpcMessageTx { + meta_buf_ptr: msg.meta_buf_ptr.clone(), + addr_backend: raw_ptr.addr(), + }; + RpcMessageGeneral::TxMessage(EngineTxMessage::RpcMessage( + new_msg, + )) + } + }) + .collect(); + + for msg in output { + match msg { + RpcMessageGeneral::TxMessage(msg) => { + self.tx_outputs()[0].send(msg)?; + } + RpcMessageGeneral::RxMessage(msg) => { + self.rx_outputs()[0].send(msg)?; + } + _ => {} + } + } + } + m => self.tx_outputs()[0].send(m)?, + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + return Ok(Status::Disconnected); + } + } + + match self.rx_inputs()[0].try_recv() { + Ok(msg) => { + match msg { + EngineRxMessage::Ack(rpc_id, status) => { + // todo + self.rx_outputs()[0].send(EngineRxMessage::Ack(rpc_id, status))?; + } + EngineRxMessage::RpcMessage(msg) => { + self.rx_outputs()[0].send(EngineRxMessage::RpcMessage(msg))?; + } + m => self.rx_outputs()[0].send(m)?, + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + return Ok(Status::Disconnected); + } + } + Ok(Progress(0)) + } +} diff --git a/experimental/mrpc/plugin/policy/fault/src/lib.rs b/experimental/mrpc/plugin/policy/fault/src/lib.rs new file mode 100644 index 00000000..0e61aa10 --- /dev/null +++ b/experimental/mrpc/plugin/policy/fault/src/lib.rs @@ -0,0 +1,37 @@ +#![feature(peer_credentials_unix_socket)] +#![feature(ptr_internals)] +#![feature(strict_provenance)] +use thiserror::Error; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +pub use phoenix_common::{InitFnResult, PhoenixAddon}; + +pub mod config; +pub(crate) mod engine; +pub mod module; + +#[derive(Error, Debug)] +pub(crate) enum DatapathError { + #[error("Internal queue send error")] + InternalQueueSend, +} + +use phoenix_common::engine::datapath::SendError; +impl From> for DatapathError { + fn from(_other: SendError) -> Self { + DatapathError::InternalQueueSend + } +} + +use crate::config::FaultConfig; +use crate::module::FaultAddon; + +#[no_mangle] +pub fn init_addon(config_string: Option<&str>) -> InitFnResult> { + let config = FaultConfig::new(config_string)?; + let addon = FaultAddon::new(config); + Ok(Box::new(addon)) +} diff --git a/experimental/mrpc/plugin/policy/fault/src/module.rs b/experimental/mrpc/plugin/policy/fault/src/module.rs new file mode 100644 index 00000000..c1512125 --- /dev/null +++ b/experimental/mrpc/plugin/policy/fault/src/module.rs @@ -0,0 +1,106 @@ +use anyhow::{bail, Result}; +use nix::unistd::Pid; + +use phoenix_common::addon::{PhoenixAddon, Version}; +use phoenix_common::engine::datapath::DataPathNode; +use phoenix_common::engine::{Engine, EngineType}; +use phoenix_common::storage::ResourceCollection; + +use super::engine::FaultEngine; +use crate::config::{create_log_file, FaultConfig}; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +pub(crate) struct FaultEngineBuilder { + node: DataPathNode, + config: FaultConfig, +} + +impl FaultEngineBuilder { + fn new(node: DataPathNode, config: FaultConfig) -> Self { + FaultEngineBuilder { node, config } + } + // TODO! LogFile + fn build(self) -> Result { + let var_probability = 0.01; + + Ok(FaultEngine { + node: self.node, + indicator: Default::default(), + config: self.config, + var_probability, + }) + } +} + +pub struct FaultAddon { + config: FaultConfig, +} + +impl FaultAddon { + pub const FAULT_ENGINE: EngineType = EngineType("FaultEngine"); + pub const ENGINES: &'static [EngineType] = &[FaultAddon::FAULT_ENGINE]; +} + +impl FaultAddon { + pub fn new(config: FaultConfig) -> Self { + FaultAddon { config } + } +} + +impl PhoenixAddon for FaultAddon { + fn check_compatibility(&self, _prev: Option<&Version>) -> bool { + true + } + + fn decompose(self: Box) -> ResourceCollection { + let addon = *self; + let mut collections = ResourceCollection::new(); + collections.insert("config".to_string(), Box::new(addon.config)); + collections + } + + #[inline] + fn migrate(&mut self, _prev_addon: Box) {} + + fn engines(&self) -> &[EngineType] { + FaultAddon::ENGINES + } + + fn update_config(&mut self, config: &str) -> Result<()> { + self.config = toml::from_str(config)?; + Ok(()) + } + + fn create_engine( + &mut self, + ty: EngineType, + _pid: Pid, + node: DataPathNode, + ) -> Result> { + if ty != FaultAddon::FAULT_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let builder = FaultEngineBuilder::new(node, self.config); + let engine = builder.build()?; + Ok(Box::new(engine)) + } + + fn restore_engine( + &mut self, + ty: EngineType, + local: ResourceCollection, + node: DataPathNode, + prev_version: Version, + ) -> Result> { + if ty != FaultAddon::FAULT_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let engine = FaultEngine::restore(local, node, prev_version)?; + Ok(Box::new(engine)) + } +} diff --git a/experimental/mrpc/plugin/policy/fault/src/proto.rs b/experimental/mrpc/plugin/policy/fault/src/proto.rs new file mode 100644 index 00000000..83ee8ab0 --- /dev/null +++ b/experimental/mrpc/plugin/policy/fault/src/proto.rs @@ -0,0 +1,29 @@ +/// The request message containing the user's name. +#[repr(C)] +#[derive(Debug, Clone, ::mrpc_derive::Message)] +pub struct HelloRequest { + #[prost(bytes = "vec", tag = "1")] + pub name: ::mrpc_marshal::shadow::Vec, +} +/// The response message containing the greetings +#[repr(C)] +#[derive(Debug, ::mrpc_derive::Message)] +pub struct HelloReply { + #[prost(bytes = "vec", tag = "1")] + pub message: ::mrpc_marshal::shadow::Vec, +} + +// /// The request message containing the user's name. +// #[repr(C)] +// #[derive(Debug, Clone, ::mrpc_derive::Message)] +// pub struct HelloRequest { +// #[prost(bytes = "vec", tag = "1")] +// pub name: ::mrpc::alloc::Vec, +// } +// /// The response message containing the greetings +// #[repr(C)] +// #[derive(Debug, ::mrpc_derive::Message)] +// pub struct HelloReply { +// #[prost(bytes = "vec", tag = "1")] +// pub message: ::mrpc::alloc::Vec, +// } diff --git a/experimental/mrpc/plugin/policy/fault2/Cargo.toml b/experimental/mrpc/plugin/policy/fault2/Cargo.toml new file mode 100644 index 00000000..4d6a2106 --- /dev/null +++ b/experimental/mrpc/plugin/policy/fault2/Cargo.toml @@ -0,0 +1,28 @@ + +[package] +name = "phoenix-fault2" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix_common.workspace = true +phoenix-api-policy-fault2.workspace = true +mrpc-marshal.workspace = true +mrpc-derive.workspace = true +shm.workspace = true +phoenix-api = { workspace = true, features = ["mrpc"] } + +futures.workspace = true +minstant.workspace = true +thiserror.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +nix.workspace = true +toml = { workspace = true, features = ["preserve_order"] } +bincode.workspace = true +chrono.workspace = true +itertools.workspace = true +rand.workspace = true diff --git a/experimental/mrpc/plugin/policy/fault2/src/config.rs b/experimental/mrpc/plugin/policy/fault2/src/config.rs new file mode 100644 index 00000000..6dd1bf70 --- /dev/null +++ b/experimental/mrpc/plugin/policy/fault2/src/config.rs @@ -0,0 +1,37 @@ +use chrono::{Datelike, Timelike, Utc}; +use phoenix_common::log; +use serde::{Deserialize, Serialize}; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Fault2Config {} + +impl Fault2Config { + /// Get config from toml file + pub fn new(config: Option<&str>) -> anyhow::Result { + let config = toml::from_str(config.unwrap_or(""))?; + Ok(config) + } +} + +pub fn create_log_file() -> std::fs::File { + std::fs::create_dir_all("/tmp/phoenix/log").expect("mkdir failed"); + let now = Utc::now(); + let date_string = format!( + "{}-{}-{}-{}-{}-{}", + now.year(), + now.month(), + now.day(), + now.hour(), + now.minute(), + now.second() + ); + let file_name = format!("/tmp/phoenix/log/logging_engine_{}.log", date_string); + ///log::info!("create log file {}", file_name); + let log_file = std::fs::File::create(file_name).expect("create file failed"); + log_file +} diff --git a/experimental/mrpc/plugin/policy/fault2/src/engine.rs b/experimental/mrpc/plugin/policy/fault2/src/engine.rs new file mode 100644 index 00000000..db8831c9 --- /dev/null +++ b/experimental/mrpc/plugin/policy/fault2/src/engine.rs @@ -0,0 +1,214 @@ +use anyhow::{anyhow, Result}; +use futures::future::BoxFuture; +use phoenix_api::rpc::{RpcId, TransportStatus}; +use std::fmt; +use std::fs::File; +use std::io::Write; +use std::num::NonZeroU32; +use std::os::unix::ucred::UCred; +use std::pin::Pin; + +use phoenix_api_policy_fault2::control_plane; + +use phoenix_common::engine::datapath::message::{ + EngineRxMessage, EngineTxMessage, RpcMessageGeneral, +}; + +use phoenix_common::engine::datapath::node::DataPathNode; +use phoenix_common::engine::{future, Decompose, Engine, EngineResult, Indicator, Vertex}; +use phoenix_common::envelop::ResourceDowncast; +use phoenix_common::impl_vertex_for_engine; +use phoenix_common::log; +use phoenix_common::module::Version; + +use phoenix_common::engine::datapath::RpcMessageTx; +use phoenix_common::storage::{ResourceCollection, SharedStorage}; + +use super::DatapathError; +use crate::config::{create_log_file, Fault2Config}; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +pub mod hello { + include!("proto.rs"); +} + +fn hello_request_name_readonly(req: &hello::HelloRequest) -> String { + let buf = &req.name as &[u8]; + String::from_utf8_lossy(buf).to_string().clone() +} + +pub(crate) struct Fault2Engine { + pub(crate) node: DataPathNode, + pub(crate) indicator: Indicator, + pub(crate) config: Fault2Config, + pub(crate) var_probability: f32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + Progress(usize), + Disconnected, +} + +use Status::Progress; + +impl Engine for Fault2Engine { + fn activate<'a>(self: Pin<&'a mut Self>) -> BoxFuture<'a, EngineResult> { + Box::pin(async move { self.get_mut().mainloop().await }) + } + + fn description(self: Pin<&Self>) -> String { + "Fault2Engine".to_owned() + } + + #[inline] + fn tracker(self: Pin<&mut Self>) -> &mut Indicator { + &mut self.get_mut().indicator + } + + fn handle_request(&mut self, request: Vec, _cred: UCred) -> Result<()> { + let request: control_plane::Request = bincode::deserialize(&request[..])?; + + match request { + control_plane::Request::NewConfig() => { + self.config = Fault2Config {}; + } + } + Ok(()) + } +} + +impl_vertex_for_engine!(Fault2Engine, node); + +impl Decompose for Fault2Engine { + fn flush(&mut self) -> Result { + let mut work = 0; + while !self.tx_inputs()[0].is_empty() || !self.rx_inputs()[0].is_empty() { + if let Progress(n) = self.check_input_queue()? { + work += n; + } + } + Ok(work) + } + + fn decompose( + self: Box, + _shared: &mut SharedStorage, + _global: &mut ResourceCollection, + ) -> (ResourceCollection, DataPathNode) { + let engine = *self; + let mut collections = ResourceCollection::with_capacity(4); + collections.insert("config".to_string(), Box::new(engine.config)); + (collections, engine.node) + } +} + +impl Fault2Engine { + pub(crate) fn restore( + mut local: ResourceCollection, + node: DataPathNode, + _prev_version: Version, + ) -> Result { + let config = *local + .remove("config") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let var_probability = 0.01; + + let engine = Fault2Engine { + node, + indicator: Default::default(), + config, + var_probability, + }; + Ok(engine) + } +} + +impl Fault2Engine { + async fn mainloop(&mut self) -> EngineResult { + loop { + let mut work = 0; + loop { + match self.check_input_queue()? { + Progress(0) => break, + Progress(n) => work += n, + Status::Disconnected => return Ok(()), + } + } + self.indicator.set_nwork(work); + future::yield_now().await; + } + } +} + +#[inline] +fn materialize_nocopy(msg: &RpcMessageTx) -> &hello::HelloRequest { + let req_ptr = msg.addr_backend as *mut hello::HelloRequest; + let req = unsafe { req_ptr.as_ref().unwrap() }; + return req; +} + +impl Fault2Engine { + fn check_input_queue(&mut self) -> Result { + use phoenix_common::engine::datapath::TryRecvError; + + match self.tx_inputs()[0].try_recv() { + Ok(msg) => { + match msg { + EngineTxMessage::RpcMessage(msg) => { + if rand::random::() < self.var_probability { + let error = EngineRxMessage::Ack( + RpcId::new( + unsafe { &*msg.meta_buf_ptr.as_meta_ptr() }.conn_id, + unsafe { &*msg.meta_buf_ptr.as_meta_ptr() }.call_id, + ), + TransportStatus::Error(unsafe { NonZeroU32::new_unchecked(403) }), + ); + self.rx_outputs()[0].send(error)? + } else { + let raw_ptr: *const hello::HelloRequest = msg.addr_backend as *const _; + let new_msg = RpcMessageTx { + meta_buf_ptr: msg.meta_buf_ptr.clone(), + addr_backend: raw_ptr.addr(), + }; + self.tx_outputs()[0].send(EngineTxMessage::RpcMessage(new_msg))? + } + } + + m => self.tx_outputs()[0].send(m)?, + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + return Ok(Status::Disconnected); + } + } + + match self.rx_inputs()[0].try_recv() { + Ok(msg) => { + match msg { + EngineRxMessage::Ack(rpc_id, status) => { + // todo + self.rx_outputs()[0].send(EngineRxMessage::Ack(rpc_id, status))?; + } + EngineRxMessage::RpcMessage(msg) => { + self.rx_outputs()[0].send(EngineRxMessage::RpcMessage(msg))?; + } + m => self.rx_outputs()[0].send(m)?, + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + return Ok(Status::Disconnected); + } + } + Ok(Progress(0)) + } +} diff --git a/experimental/mrpc/plugin/policy/fault2/src/lib.rs b/experimental/mrpc/plugin/policy/fault2/src/lib.rs new file mode 100644 index 00000000..d9e53d48 --- /dev/null +++ b/experimental/mrpc/plugin/policy/fault2/src/lib.rs @@ -0,0 +1,37 @@ +#![feature(peer_credentials_unix_socket)] +#![feature(ptr_internals)] +#![feature(strict_provenance)] +use thiserror::Error; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +pub use phoenix_common::{InitFnResult, PhoenixAddon}; + +pub mod config; +pub(crate) mod engine; +pub mod module; + +#[derive(Error, Debug)] +pub(crate) enum DatapathError { + #[error("Internal queue send error")] + InternalQueueSend, +} + +use phoenix_common::engine::datapath::SendError; +impl From> for DatapathError { + fn from(_other: SendError) -> Self { + DatapathError::InternalQueueSend + } +} + +use crate::config::Fault2Config; +use crate::module::Fault2Addon; + +#[no_mangle] +pub fn init_addon(config_string: Option<&str>) -> InitFnResult> { + let config = Fault2Config::new(config_string)?; + let addon = Fault2Addon::new(config); + Ok(Box::new(addon)) +} diff --git a/experimental/mrpc/plugin/policy/fault2/src/module.rs b/experimental/mrpc/plugin/policy/fault2/src/module.rs new file mode 100644 index 00000000..d57ddbdd --- /dev/null +++ b/experimental/mrpc/plugin/policy/fault2/src/module.rs @@ -0,0 +1,106 @@ +use anyhow::{bail, Result}; +use nix::unistd::Pid; + +use phoenix_common::addon::{PhoenixAddon, Version}; +use phoenix_common::engine::datapath::DataPathNode; +use phoenix_common::engine::{Engine, EngineType}; +use phoenix_common::storage::ResourceCollection; + +use super::engine::Fault2Engine; +use crate::config::{create_log_file, Fault2Config}; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +pub(crate) struct Fault2EngineBuilder { + node: DataPathNode, + config: Fault2Config, +} + +impl Fault2EngineBuilder { + fn new(node: DataPathNode, config: Fault2Config) -> Self { + Fault2EngineBuilder { node, config } + } + // TODO! LogFile + fn build(self) -> Result { + let var_probability = 0.01; + + Ok(Fault2Engine { + node: self.node, + indicator: Default::default(), + config: self.config, + var_probability, + }) + } +} + +pub struct Fault2Addon { + config: Fault2Config, +} + +impl Fault2Addon { + pub const FAULT2_ENGINE: EngineType = EngineType("Fault2Engine"); + pub const ENGINES: &'static [EngineType] = &[Fault2Addon::FAULT2_ENGINE]; +} + +impl Fault2Addon { + pub fn new(config: Fault2Config) -> Self { + Fault2Addon { config } + } +} + +impl PhoenixAddon for Fault2Addon { + fn check_compatibility(&self, _prev: Option<&Version>) -> bool { + true + } + + fn decompose(self: Box) -> ResourceCollection { + let addon = *self; + let mut collections = ResourceCollection::new(); + collections.insert("config".to_string(), Box::new(addon.config)); + collections + } + + #[inline] + fn migrate(&mut self, _prev_addon: Box) {} + + fn engines(&self) -> &[EngineType] { + Fault2Addon::ENGINES + } + + fn update_config(&mut self, config: &str) -> Result<()> { + self.config = toml::from_str(config)?; + Ok(()) + } + + fn create_engine( + &mut self, + ty: EngineType, + _pid: Pid, + node: DataPathNode, + ) -> Result> { + if ty != Fault2Addon::FAULT2_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let builder = Fault2EngineBuilder::new(node, self.config); + let engine = builder.build()?; + Ok(Box::new(engine)) + } + + fn restore_engine( + &mut self, + ty: EngineType, + local: ResourceCollection, + node: DataPathNode, + prev_version: Version, + ) -> Result> { + if ty != Fault2Addon::FAULT2_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let engine = Fault2Engine::restore(local, node, prev_version)?; + Ok(Box::new(engine)) + } +} diff --git a/experimental/mrpc/plugin/policy/fault2/src/proto.rs b/experimental/mrpc/plugin/policy/fault2/src/proto.rs new file mode 100644 index 00000000..83ee8ab0 --- /dev/null +++ b/experimental/mrpc/plugin/policy/fault2/src/proto.rs @@ -0,0 +1,29 @@ +/// The request message containing the user's name. +#[repr(C)] +#[derive(Debug, Clone, ::mrpc_derive::Message)] +pub struct HelloRequest { + #[prost(bytes = "vec", tag = "1")] + pub name: ::mrpc_marshal::shadow::Vec, +} +/// The response message containing the greetings +#[repr(C)] +#[derive(Debug, ::mrpc_derive::Message)] +pub struct HelloReply { + #[prost(bytes = "vec", tag = "1")] + pub message: ::mrpc_marshal::shadow::Vec, +} + +// /// The request message containing the user's name. +// #[repr(C)] +// #[derive(Debug, Clone, ::mrpc_derive::Message)] +// pub struct HelloRequest { +// #[prost(bytes = "vec", tag = "1")] +// pub name: ::mrpc::alloc::Vec, +// } +// /// The response message containing the greetings +// #[repr(C)] +// #[derive(Debug, ::mrpc_derive::Message)] +// pub struct HelloReply { +// #[prost(bytes = "vec", tag = "1")] +// pub message: ::mrpc::alloc::Vec, +// } diff --git a/experimental/mrpc/plugin/policy/hello-acl-sender/src/engine.rs b/experimental/mrpc/plugin/policy/hello-acl-sender/src/engine.rs index caf3ac13..5201dc76 100644 --- a/experimental/mrpc/plugin/policy/hello-acl-sender/src/engine.rs +++ b/experimental/mrpc/plugin/policy/hello-acl-sender/src/engine.rs @@ -1,4 +1,5 @@ //! This engine can only be placed at the sender side for now. +use std::f32::consts::E; use std::num::NonZeroU32; use std::os::unix::ucred::UCred; use std::pin::Pin; @@ -28,6 +29,11 @@ pub mod hello { include!("rpc_hello.rs"); } +pub struct AclTable { + pub name: String, + pub permission: String, +} + pub(crate) struct HelloAclSenderEngine { pub(crate) node: DataPathNode, @@ -40,6 +46,7 @@ pub(crate) struct HelloAclSenderEngine { // pub(crate) filter: FnvHashSet, // Number of tokens to add for each seconds. pub(crate) config: HelloAclSenderConfig, + pub(crate) acl_table: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -124,11 +131,22 @@ impl HelloAclSenderEngine { .downcast::() .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let acl_table = vec![ + AclTable { + name: "Apple".to_string(), + permission: "N".to_string(), + }, + AclTable { + name: "Banana".to_string(), + permission: "Y".to_string(), + }, + ]; let engine = HelloAclSenderEngine { node, indicator: Default::default(), outstanding_req_pool, config, + acl_table, }; Ok(engine) } @@ -165,12 +183,18 @@ fn materialize(msg: &RpcMessageTx) -> Box { } #[inline] -fn should_block(req: &hello::HelloRequest) -> bool { +fn nocopy_materialize(msg: &RpcMessageTx) -> &hello::HelloRequest { + let req_ptr = msg.addr_backend as *mut hello::HelloRequest; + let req = unsafe { req_ptr.as_ref().unwrap() }; + // returns a private_req + return req; +} + +fn hello_request_name_readonly(req: &hello::HelloRequest) -> &[u8] { let buf = &req.name as &[u8]; - //let name = String::from_utf8_lossy(buf); - //log::info!("raw: {:?}, req.name: {:?}", buf, name); - buf == b"Apple" + buf } + impl HelloAclSenderEngine { fn check_input_queue(&mut self) -> Result { use phoenix_common::engine::datapath::TryRecvError; @@ -179,34 +203,31 @@ impl HelloAclSenderEngine { Ok(msg) => { match msg { EngineTxMessage::RpcMessage(msg) => { - // 1 clone - let private_req = materialize(&msg); - // 2 check should block - // yes: ACK with error, drop the data - // no: pass the cloned msg to the next engine, who drops the data? - // Should we Ack right after clone? + let req = nocopy_materialize(&msg); let conn_id = unsafe { &*msg.meta_buf_ptr.as_meta_ptr() }.conn_id; let call_id = unsafe { &*msg.meta_buf_ptr.as_meta_ptr() }.call_id; let rpc_id = RpcId::new(conn_id, call_id); - if should_block(&private_req) { - let error = EngineRxMessage::Ack( - rpc_id, - TransportStatus::Error(unsafe { NonZeroU32::new_unchecked(403) }), - ); - self.rx_outputs()[0].send(error).unwrap_or_else(|e| { - log::warn!("error when bubbling up the error, send failed e: {}", e) - }); - drop(private_req); - } else { - // We will release the request on private heap after the RPC adapter - // passes us an Ack. - let raw_ptr: *const hello::HelloRequest = &*private_req; - self.outstanding_req_pool.insert(rpc_id, private_req); - let new_msg = RpcMessageTx { - meta_buf_ptr: msg.meta_buf_ptr, - addr_backend: raw_ptr.addr(), - }; - self.tx_outputs()[0].send(EngineTxMessage::RpcMessage(new_msg))?; + for acl_rec in &self.acl_table { + if String::from_utf8_lossy(&req.name) == acl_rec.name { + if acl_rec.permission == "N" { + let error = EngineRxMessage::Ack( + rpc_id, + TransportStatus::Error(unsafe { + NonZeroU32::new_unchecked(403) + }), + ); + self.rx_outputs()[0].send(error)?; + } else { + let raw_ptr: *const hello::HelloRequest = req; + let new_msg = RpcMessageTx { + meta_buf_ptr: msg.meta_buf_ptr, + addr_backend: raw_ptr.addr(), + }; + self.tx_outputs()[0] + .send(EngineTxMessage::RpcMessage(new_msg))?; + } + break; + } } } // XXX TODO(cjr): it is best not to reorder the message @@ -221,10 +242,6 @@ impl HelloAclSenderEngine { // forward all rx msgs match self.rx_inputs()[0].try_recv() { Ok(m) => { - if let EngineRxMessage::Ack(rpc_id, _status) = m { - // remove private_req - self.outstanding_req_pool.remove(&rpc_id); - } self.rx_outputs()[0].send(m)?; return Ok(Progress(1)); } diff --git a/experimental/mrpc/plugin/policy/hello-acl-sender/src/module.rs b/experimental/mrpc/plugin/policy/hello-acl-sender/src/module.rs index 5b195b18..0a70b25c 100644 --- a/experimental/mrpc/plugin/policy/hello-acl-sender/src/module.rs +++ b/experimental/mrpc/plugin/policy/hello-acl-sender/src/module.rs @@ -9,6 +9,7 @@ use phoenix_common::storage::ResourceCollection; use super::engine::HelloAclSenderEngine; use crate::config::HelloAclSenderConfig; +use crate::engine::AclTable; pub(crate) struct HelloAclSenderEngineBuilder { node: DataPathNode, @@ -21,11 +22,22 @@ impl HelloAclSenderEngineBuilder { } fn build(self) -> Result { + let acl_table = vec![ + AclTable { + name: "Apple".to_string(), + permission: "N".to_string(), + }, + AclTable { + name: "Banana".to_string(), + permission: "Y".to_string(), + }, + ]; Ok(HelloAclSenderEngine { node: self.node, indicator: Default::default(), outstanding_req_pool: HashMap::default(), config: self.config, + acl_table, }) } } diff --git a/experimental/mrpc/plugin/policy/hello-acl/Cargo.toml b/experimental/mrpc/plugin/policy/hello-acl/Cargo.toml new file mode 100644 index 00000000..3f762fd6 --- /dev/null +++ b/experimental/mrpc/plugin/policy/hello-acl/Cargo.toml @@ -0,0 +1,27 @@ + +[package] +name = "phoenix-hello-acl" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix_common.workspace = true +phoenix-api-policy-hello-acl.workspace = true +mrpc-marshal.workspace = true +mrpc-derive.workspace = true +shm.workspace = true +phoenix-api = { workspace = true, features = ["mrpc"] } + +futures.workspace = true +minstant.workspace = true +thiserror.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +nix.workspace = true +toml = { workspace = true, features = ["preserve_order"] } +bincode.workspace = true +chrono.workspace = true +itertools.workspace = true diff --git a/experimental/mrpc/plugin/policy/hello-acl/src/config.rs b/experimental/mrpc/plugin/policy/hello-acl/src/config.rs new file mode 100644 index 00000000..5790ce71 --- /dev/null +++ b/experimental/mrpc/plugin/policy/hello-acl/src/config.rs @@ -0,0 +1,38 @@ +use chrono::{Datelike, Timelike, Utc}; +use phoenix_common::log; +use serde::{Deserialize, Serialize}; + +use chrono::prelude::*; +use itertools::iproduct; + +use crate::engine::struct_acl; + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HelloAclConfig {} + +impl HelloAclConfig { + /// Get config from toml file + pub fn new(config: Option<&str>) -> anyhow::Result { + let config = toml::from_str(config.unwrap_or(""))?; + Ok(config) + } +} + +pub fn create_log_file() -> std::fs::File { + std::fs::create_dir_all("/tmp/phoenix/log").expect("mkdir failed"); + let now = Utc::now(); + let date_string = format!( + "{}-{}-{}-{}-{}-{}", + now.year(), + now.month(), + now.day(), + now.hour(), + now.minute(), + now.second() + ); + let file_name = format!("/tmp/phoenix/log/logging_engine_{}.log", date_string); + ///log::info!("create log file {}", file_name); + let log_file = std::fs::File::create(file_name).expect("create file failed"); + log_file +} diff --git a/experimental/mrpc/plugin/policy/hello-acl/src/engine.rs b/experimental/mrpc/plugin/policy/hello-acl/src/engine.rs new file mode 100644 index 00000000..b1edc4da --- /dev/null +++ b/experimental/mrpc/plugin/policy/hello-acl/src/engine.rs @@ -0,0 +1,261 @@ +use anyhow::{anyhow, Result}; +use futures::future::BoxFuture; +use phoenix_api::rpc::{RpcId, TransportStatus}; +use std::fmt; +use std::fs::File; +use std::io::Write; +use std::num::NonZeroU32; +use std::os::unix::ucred::UCred; +use std::pin::Pin; + +use phoenix_api_policy_hello_acl::control_plane; + +use phoenix_common::engine::datapath::message::{ + EngineRxMessage, EngineTxMessage, RpcMessageGeneral, +}; + +use phoenix_common::engine::datapath::node::DataPathNode; +use phoenix_common::engine::{future, Decompose, Engine, EngineResult, Indicator, Vertex}; +use phoenix_common::envelop::ResourceDowncast; +use phoenix_common::impl_vertex_for_engine; +use phoenix_common::log; +use phoenix_common::module::Version; + +use phoenix_common::engine::datapath::RpcMessageTx; +use phoenix_common::storage::{ResourceCollection, SharedStorage}; + +use super::DatapathError; +use crate::config::{create_log_file, HelloAclConfig}; + +use chrono::prelude::*; +use itertools::iproduct; + +pub mod hello { + include!("proto.rs"); +} + +fn hello_request_name_readonly(req: &hello::HelloRequest) -> String { + let buf = &req.name as &[u8]; + String::from_utf8_lossy(buf).to_string().clone() +} + +pub struct struct_acl { + pub name: String, + pub permission: String, +} +impl struct_acl { + pub fn new(name: String, permission: String) -> struct_acl { + struct_acl { + name: name, + permission: permission, + } + } +} + +pub(crate) struct HelloAclEngine { + pub(crate) node: DataPathNode, + pub(crate) indicator: Indicator, + pub(crate) config: HelloAclConfig, + pub(crate) table_acl: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + Progress(usize), + Disconnected, +} + +use Status::Progress; + +impl Engine for HelloAclEngine { + fn activate<'a>(self: Pin<&'a mut Self>) -> BoxFuture<'a, EngineResult> { + Box::pin(async move { self.get_mut().mainloop().await }) + } + + fn description(self: Pin<&Self>) -> String { + "HelloAclEngine".to_owned() + } + + #[inline] + fn tracker(self: Pin<&mut Self>) -> &mut Indicator { + &mut self.get_mut().indicator + } + + fn handle_request(&mut self, request: Vec, _cred: UCred) -> Result<()> { + let request: control_plane::Request = bincode::deserialize(&request[..])?; + + match request { + control_plane::Request::NewConfig() => { + self.config = HelloAclConfig {}; + } + } + Ok(()) + } +} + +impl_vertex_for_engine!(HelloAclEngine, node); + +impl Decompose for HelloAclEngine { + fn flush(&mut self) -> Result { + let mut work = 0; + while !self.tx_inputs()[0].is_empty() || !self.rx_inputs()[0].is_empty() { + if let Progress(n) = self.check_input_queue()? { + work += n; + } + } + Ok(work) + } + + fn decompose( + self: Box, + _shared: &mut SharedStorage, + _global: &mut ResourceCollection, + ) -> (ResourceCollection, DataPathNode) { + let engine = *self; + let mut collections = ResourceCollection::with_capacity(4); + collections.insert("config".to_string(), Box::new(engine.config)); + (collections, engine.node) + } +} + +impl HelloAclEngine { + pub(crate) fn restore( + mut local: ResourceCollection, + node: DataPathNode, + _prev_version: Version, + ) -> Result { + let config = *local + .remove("config") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let mut table_acl = Vec::new(); + table_acl.push(struct_acl { + name: "Apple".to_string(), + permission: "N".to_string(), + }); + table_acl.push(struct_acl { + name: "Banana".to_string(), + permission: "Y".to_string(), + }); + + let engine = HelloAclEngine { + node, + indicator: Default::default(), + config, + table_acl, + }; + Ok(engine) + } +} + +impl HelloAclEngine { + async fn mainloop(&mut self) -> EngineResult { + loop { + let mut work = 0; + loop { + match self.check_input_queue()? { + Progress(0) => break, + Progress(n) => work += n, + Status::Disconnected => return Ok(()), + } + } + self.indicator.set_nwork(work); + future::yield_now().await; + } + } +} + +#[inline] +fn materialize_nocopy(msg: &RpcMessageTx) -> &hello::HelloRequest { + let req_ptr = msg.addr_backend as *mut hello::HelloRequest; + let req = unsafe { req_ptr.as_ref().unwrap() }; + return req; +} + +impl HelloAclEngine { + fn check_input_queue(&mut self) -> Result { + use phoenix_common::engine::datapath::TryRecvError; + + match self.tx_inputs()[0].try_recv() { + Ok(msg) => { + match msg { + EngineTxMessage::RpcMessage(msg) => { + let meta_ref = unsafe { &*msg.meta_buf_ptr.as_meta_ptr() }; + let mut input = Vec::new(); + input.push(msg); + let output: Vec<_> = iproduct!(input.iter(), self.table_acl.iter()) + .map(|(msg, join)| { + let rpc_message = materialize_nocopy(&msg); + let conn_id = unsafe { &*msg.meta_buf_ptr.as_meta_ptr() }.conn_id; + let call_id = unsafe { &*msg.meta_buf_ptr.as_meta_ptr() }.call_id; + let rpc_id = RpcId::new(conn_id, call_id); + if hello_request_name_readonly(rpc_message) == join.name { + if join.permission == "N" { + let error = EngineRxMessage::Ack( + rpc_id, + TransportStatus::Error(unsafe { + NonZeroU32::new_unchecked(403) + }), + ); + RpcMessageGeneral::RxMessage(error) + } else { + let raw_ptr: *const hello::HelloRequest = rpc_message; + let new_msg = RpcMessageTx { + meta_buf_ptr: msg.meta_buf_ptr.clone(), + addr_backend: raw_ptr.addr(), + }; + RpcMessageGeneral::TxMessage(EngineTxMessage::RpcMessage( + new_msg, + )) + } + } else { + RpcMessageGeneral::Pass + } + }) + .collect(); + + for msg in output { + match msg { + RpcMessageGeneral::TxMessage(msg) => { + self.tx_outputs()[0].send(msg)?; + } + RpcMessageGeneral::RxMessage(msg) => { + self.rx_outputs()[0].send(msg)?; + } + _ => {} + } + } + } + m => self.tx_outputs()[0].send(m)?, + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + return Ok(Status::Disconnected); + } + } + + match self.rx_inputs()[0].try_recv() { + Ok(msg) => { + match msg { + EngineRxMessage::Ack(rpc_id, status) => { + // todo + self.rx_outputs()[0].send(EngineRxMessage::Ack(rpc_id, status))?; + } + EngineRxMessage::RpcMessage(msg) => { + self.rx_outputs()[0].send(EngineRxMessage::RpcMessage(msg))?; + } + m => self.rx_outputs()[0].send(m)?, + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + return Ok(Status::Disconnected); + } + } + Ok(Progress(0)) + } +} diff --git a/experimental/mrpc/plugin/policy/hello-acl/src/lib.rs b/experimental/mrpc/plugin/policy/hello-acl/src/lib.rs new file mode 100644 index 00000000..fccedb63 --- /dev/null +++ b/experimental/mrpc/plugin/policy/hello-acl/src/lib.rs @@ -0,0 +1,38 @@ +#![feature(peer_credentials_unix_socket)] +#![feature(ptr_internals)] +#![feature(strict_provenance)] +use thiserror::Error; + +use crate::engine::struct_acl; + +use chrono::prelude::*; +use itertools::iproduct; + +pub use phoenix_common::{InitFnResult, PhoenixAddon}; + +pub mod config; +pub(crate) mod engine; +pub mod module; + +#[derive(Error, Debug)] +pub(crate) enum DatapathError { + #[error("Internal queue send error")] + InternalQueueSend, +} + +use phoenix_common::engine::datapath::SendError; +impl From> for DatapathError { + fn from(_other: SendError) -> Self { + DatapathError::InternalQueueSend + } +} + +use crate::config::HelloAclConfig; +use crate::module::HelloAclAddon; + +#[no_mangle] +pub fn init_addon(config_string: Option<&str>) -> InitFnResult> { + let config = HelloAclConfig::new(config_string)?; + let addon = HelloAclAddon::new(config); + Ok(Box::new(addon)) +} diff --git a/experimental/mrpc/plugin/policy/hello-acl/src/module.rs b/experimental/mrpc/plugin/policy/hello-acl/src/module.rs new file mode 100644 index 00000000..6290c56e --- /dev/null +++ b/experimental/mrpc/plugin/policy/hello-acl/src/module.rs @@ -0,0 +1,114 @@ +use anyhow::{bail, Result}; +use nix::unistd::Pid; + +use phoenix_common::addon::{PhoenixAddon, Version}; +use phoenix_common::engine::datapath::DataPathNode; +use phoenix_common::engine::{Engine, EngineType}; +use phoenix_common::storage::ResourceCollection; + +use super::engine::HelloAclEngine; +use crate::config::{create_log_file, HelloAclConfig}; +use crate::engine::struct_acl; + +use chrono::prelude::*; +use itertools::iproduct; + +pub(crate) struct HelloAclEngineBuilder { + node: DataPathNode, + config: HelloAclConfig, +} + +impl HelloAclEngineBuilder { + fn new(node: DataPathNode, config: HelloAclConfig) -> Self { + HelloAclEngineBuilder { node, config } + } + // TODO! LogFile + fn build(self) -> Result { + let mut table_acl = Vec::new(); + table_acl.push(struct_acl { + name: "Apple".to_string(), + permission: "N".to_string(), + }); + table_acl.push(struct_acl { + name: "Banana".to_string(), + permission: "Y".to_string(), + }); + + Ok(HelloAclEngine { + node: self.node, + indicator: Default::default(), + config: self.config, + table_acl, + }) + } +} + +pub struct HelloAclAddon { + config: HelloAclConfig, +} + +impl HelloAclAddon { + pub const HELLO_ACL_ENGINE: EngineType = EngineType("HelloAclEngine"); + pub const ENGINES: &'static [EngineType] = &[HelloAclAddon::HELLO_ACL_ENGINE]; +} + +impl HelloAclAddon { + pub fn new(config: HelloAclConfig) -> Self { + HelloAclAddon { config } + } +} + +impl PhoenixAddon for HelloAclAddon { + fn check_compatibility(&self, _prev: Option<&Version>) -> bool { + true + } + + fn decompose(self: Box) -> ResourceCollection { + let addon = *self; + let mut collections = ResourceCollection::new(); + collections.insert("config".to_string(), Box::new(addon.config)); + collections + } + + #[inline] + fn migrate(&mut self, _prev_addon: Box) {} + + fn engines(&self) -> &[EngineType] { + HelloAclAddon::ENGINES + } + + fn update_config(&mut self, config: &str) -> Result<()> { + self.config = toml::from_str(config)?; + Ok(()) + } + + fn create_engine( + &mut self, + ty: EngineType, + _pid: Pid, + node: DataPathNode, + ) -> Result> { + if ty != HelloAclAddon::HELLO_ACL_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let builder = HelloAclEngineBuilder::new(node, self.config); + let engine = builder.build()?; + Ok(Box::new(engine)) + } + + fn restore_engine( + &mut self, + ty: EngineType, + local: ResourceCollection, + node: DataPathNode, + prev_version: Version, + ) -> Result> { + if ty != HelloAclAddon::HELLO_ACL_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let engine = HelloAclEngine::restore(local, node, prev_version)?; + Ok(Box::new(engine)) + } +} diff --git a/experimental/mrpc/plugin/policy/hello-acl/src/proto.rs b/experimental/mrpc/plugin/policy/hello-acl/src/proto.rs new file mode 100644 index 00000000..83ee8ab0 --- /dev/null +++ b/experimental/mrpc/plugin/policy/hello-acl/src/proto.rs @@ -0,0 +1,29 @@ +/// The request message containing the user's name. +#[repr(C)] +#[derive(Debug, Clone, ::mrpc_derive::Message)] +pub struct HelloRequest { + #[prost(bytes = "vec", tag = "1")] + pub name: ::mrpc_marshal::shadow::Vec, +} +/// The response message containing the greetings +#[repr(C)] +#[derive(Debug, ::mrpc_derive::Message)] +pub struct HelloReply { + #[prost(bytes = "vec", tag = "1")] + pub message: ::mrpc_marshal::shadow::Vec, +} + +// /// The request message containing the user's name. +// #[repr(C)] +// #[derive(Debug, Clone, ::mrpc_derive::Message)] +// pub struct HelloRequest { +// #[prost(bytes = "vec", tag = "1")] +// pub name: ::mrpc::alloc::Vec, +// } +// /// The response message containing the greetings +// #[repr(C)] +// #[derive(Debug, ::mrpc_derive::Message)] +// pub struct HelloReply { +// #[prost(bytes = "vec", tag = "1")] +// pub message: ::mrpc::alloc::Vec, +// } diff --git a/experimental/mrpc/plugin/policy/logging/Cargo.toml b/experimental/mrpc/plugin/policy/logging/Cargo.toml index 5df2751e..1ba4952d 100644 --- a/experimental/mrpc/plugin/policy/logging/Cargo.toml +++ b/experimental/mrpc/plugin/policy/logging/Cargo.toml @@ -8,6 +8,10 @@ edition = "2021" [dependencies] phoenix_common.workspace = true phoenix-api-policy-logging.workspace = true +mrpc-marshal.workspace = true +mrpc-derive.workspace = true +shm.workspace = true +phoenix-api = { workspace = true, features = ["mrpc"] } futures.workspace = true minstant.workspace = true @@ -19,3 +23,5 @@ nix.workspace = true toml = { workspace = true, features = ["preserve_order"] } bincode.workspace = true chrono.workspace = true +itertools.workspace = true +rand.workspace = true diff --git a/experimental/mrpc/plugin/policy/logging/src/config.rs b/experimental/mrpc/plugin/policy/logging/src/config.rs index ffecf8bc..4be66f50 100644 --- a/experimental/mrpc/plugin/policy/logging/src/config.rs +++ b/experimental/mrpc/plugin/policy/logging/src/config.rs @@ -1,18 +1,27 @@ +//! template file for engine config +//! we can add custome functions here +//! so that the whole crate can import the function + use chrono::{Datelike, Timelike, Utc}; use phoenix_common::log; use serde::{Deserialize, Serialize}; +/// currently, logging engine does not need a config #[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct LoggingConfig {} impl LoggingConfig { + /// Get config from toml file pub fn new(config: Option<&str>) -> anyhow::Result { let config = toml::from_str(config.unwrap_or(""))?; Ok(config) } } +/// Create a log file in `/tmp/phoenix/log` +/// This function will be called every time +/// a logging engine is started or restored pub fn create_log_file() -> std::fs::File { std::fs::create_dir_all("/tmp/phoenix/log").expect("mkdir failed"); let now = Utc::now(); diff --git a/experimental/mrpc/plugin/policy/logging/src/engine.rs b/experimental/mrpc/plugin/policy/logging/src/engine.rs index d35c45fb..dbef2796 100644 --- a/experimental/mrpc/plugin/policy/logging/src/engine.rs +++ b/experimental/mrpc/plugin/policy/logging/src/engine.rs @@ -1,11 +1,13 @@ +//! main logic happens here use anyhow::{anyhow, Result}; +use chrono::Utc; use futures::future::BoxFuture; +use phoenix_api_policy_logging::control_plane; +use phoenix_common::engine::datapath::RpcMessageTx; use std::io::Write; use std::os::unix::ucred::UCred; use std::pin::Pin; -use phoenix_api_policy_logging::control_plane; - use phoenix_common::engine::datapath::message::{EngineRxMessage, EngineTxMessage}; use phoenix_common::engine::datapath::node::DataPathNode; @@ -18,11 +20,20 @@ use phoenix_common::storage::{ResourceCollection, SharedStorage}; use super::DatapathError; use crate::config::{create_log_file, LoggingConfig}; +pub mod hello { + include!("proto.rs"); +} + +/// The internal state of an logging engine, +/// it contains some template fields like `node`, `indicator`, +/// a config field, in that case `LoggingConfig` +/// and other custome fields like `log_file pub(crate) struct LoggingEngine { pub(crate) node: DataPathNode, - pub(crate) indicator: Indicator, pub(crate) config: LoggingConfig, + /// log_file is where the log will be written into + /// it is temperoray, i.e. we don't store it when restart pub(crate) log_file: std::fs::File, } @@ -34,6 +45,7 @@ enum Status { use Status::Progress; +/// template impl Engine for LoggingEngine { fn activate<'a>(self: Pin<&'a mut Self>) -> BoxFuture<'a, EngineResult> { Box::pin(async move { self.get_mut().mainloop().await }) @@ -63,8 +75,12 @@ impl Engine for LoggingEngine { impl_vertex_for_engine!(LoggingEngine, node); impl Decompose for LoggingEngine { + /// flush will be called when we need to clean the transient state before decompose + /// # return + /// * `Result` - number of work drained from tx & rx queue fn flush(&mut self) -> Result { let mut work = 0; + /// drain the rx & tx queue while !self.tx_inputs()[0].is_empty() || !self.rx_inputs()[0].is_empty() { if let Progress(n) = self.check_input_queue()? { work += n; @@ -75,6 +91,7 @@ impl Decompose for LoggingEngine { Ok(work) } + /// template fn decompose( self: Box, _shared: &mut SharedStorage, @@ -131,60 +148,88 @@ impl LoggingEngine { } } +#[inline] +fn materialize_nocopy(msg: &RpcMessageTx) -> &hello::HelloRequest { + let req_ptr = msg.addr_backend as *mut hello::HelloRequest; + let req = unsafe { req_ptr.as_ref().unwrap() }; + return req; +} + impl LoggingEngine { + /// main logic about handling rx & tx input messages + /// note that a logging engine can be deployed in client-side or server-side fn check_input_queue(&mut self) -> Result { use phoenix_common::engine::datapath::TryRecvError; + // tx logic + // For server it is `On-Response` logic, when sending response to network + // For client it is `On-Request` logic, when sending request to network match self.tx_inputs()[0].try_recv() { Ok(msg) => { match msg { + // we care only log RPCs + // other types like ACK should not be logged since they are not + // ACKs between Client/Server, but communication between engines + // "Real" ACKs are logged in rx logic EngineTxMessage::RpcMessage(msg) => { + // we get the metadata of RPC from the shared memory let meta_ref = unsafe { &*msg.meta_buf_ptr.as_meta_ptr() }; - //log::info!("Got message on tx queue: {:?}", meta_ref); - self.log_file - .write(format!("Got message on tx queue: {:?}", meta_ref).as_bytes()) - .expect("error writing to log file"); + let rpc_message = materialize_nocopy(&msg); + // write the metadata into the file + // since meta_ref implements Debug, we can use {:?} + // rather than manully parse the metadata struct + write!( + self.log_file, + "{}{}{}{}{}\n", + Utc::now(), + format!("{:?}", meta_ref.msg_type), + format!("{:?}", meta_ref.conn_id), + format!("{:?}", meta_ref.conn_id), + format!("{}", String::from_utf8_lossy(&rpc_message.name)), + ) + .unwrap(); + + // after logging, we forward the message to the next engine self.tx_outputs()[0].send(EngineTxMessage::RpcMessage(msg))?; } + // if received message is not RPC, we simple forward it m => self.tx_outputs()[0].send(m)?, } return Ok(Progress(1)); } Err(TryRecvError::Empty) => {} Err(TryRecvError::Disconnected) => { - // log::info!("Disconnected!"); return Ok(Status::Disconnected); } } + // tx logic + // For server it is `On-Request` logic, when recving request from network + // For client it is `On-Response` logic, when recving response from network match self.rx_inputs()[0].try_recv() { Ok(msg) => { match msg { + // ACK means that + // If I am client: server received my request + // If I am server: client recevied my response EngineRxMessage::Ack(rpc_id, status) => { - self.log_file - .write( - format!( - "Got ack on rx queue, rpc_id {:?}, status: {:?}", - rpc_id, status - ) - .as_bytes(), - ) - .expect("error writing to log file"); + // log the info to the file + // forward the message self.rx_outputs()[0].send(EngineRxMessage::Ack(rpc_id, status))?; } EngineRxMessage::RpcMessage(msg) => { - //log::info!("Got msg on rx queue: {:?}", msg); + // forward the message + // again, this RpcMessage is not the application-level rpc + // so we don log them self.rx_outputs()[0].send(EngineRxMessage::RpcMessage(msg))?; } + // forward other unknown msg m => self.rx_outputs()[0].send(m)?, } - //log::info!("Send msg in rx queue"); - //self.rx_outputs()[0].send(msg)?; return Ok(Progress(1)); } Err(TryRecvError::Empty) => {} Err(TryRecvError::Disconnected) => { - // log::info!("Disconnected!"); return Ok(Status::Disconnected); } } diff --git a/experimental/mrpc/plugin/policy/logging/src/lib.rs b/experimental/mrpc/plugin/policy/logging/src/lib.rs index a2eb236a..2a8608a8 100644 --- a/experimental/mrpc/plugin/policy/logging/src/lib.rs +++ b/experimental/mrpc/plugin/policy/logging/src/lib.rs @@ -1,5 +1,6 @@ -#![feature(peer_credentials_unix_socket)] +//! template file for export +#![feature(peer_credentials_unix_socket)] use thiserror::Error; pub use phoenix_common::{InitFnResult, PhoenixAddon}; diff --git a/experimental/mrpc/plugin/policy/logging/src/module.rs b/experimental/mrpc/plugin/policy/logging/src/module.rs index 70c052a6..521f92e2 100644 --- a/experimental/mrpc/plugin/policy/logging/src/module.rs +++ b/experimental/mrpc/plugin/policy/logging/src/module.rs @@ -1,3 +1,6 @@ +//! template file for an engine +//! we only need to change the args in config and engine constructor + use anyhow::{bail, Result}; use nix::unistd::Pid; diff --git a/experimental/mrpc/plugin/policy/logging/src/proto.rs b/experimental/mrpc/plugin/policy/logging/src/proto.rs new file mode 100644 index 00000000..83ee8ab0 --- /dev/null +++ b/experimental/mrpc/plugin/policy/logging/src/proto.rs @@ -0,0 +1,29 @@ +/// The request message containing the user's name. +#[repr(C)] +#[derive(Debug, Clone, ::mrpc_derive::Message)] +pub struct HelloRequest { + #[prost(bytes = "vec", tag = "1")] + pub name: ::mrpc_marshal::shadow::Vec, +} +/// The response message containing the greetings +#[repr(C)] +#[derive(Debug, ::mrpc_derive::Message)] +pub struct HelloReply { + #[prost(bytes = "vec", tag = "1")] + pub message: ::mrpc_marshal::shadow::Vec, +} + +// /// The request message containing the user's name. +// #[repr(C)] +// #[derive(Debug, Clone, ::mrpc_derive::Message)] +// pub struct HelloRequest { +// #[prost(bytes = "vec", tag = "1")] +// pub name: ::mrpc::alloc::Vec, +// } +// /// The response message containing the greetings +// #[repr(C)] +// #[derive(Debug, ::mrpc_derive::Message)] +// pub struct HelloReply { +// #[prost(bytes = "vec", tag = "1")] +// pub message: ::mrpc::alloc::Vec, +// } diff --git a/experimental/mrpc/plugin/policy/nofile-logging/Cargo.toml b/experimental/mrpc/plugin/policy/nofile-logging/Cargo.toml new file mode 100644 index 00000000..71b7dd57 --- /dev/null +++ b/experimental/mrpc/plugin/policy/nofile-logging/Cargo.toml @@ -0,0 +1,28 @@ + +[package] +name = "phoenix-nofile-logging" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix_common.workspace = true +phoenix-api-policy-nofile-logging.workspace = true +mrpc-marshal.workspace = true +mrpc-derive.workspace = true +shm.workspace = true +phoenix-api = { workspace = true, features = ["mrpc"] } + +futures.workspace = true +minstant.workspace = true +thiserror.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +nix.workspace = true +toml = { workspace = true, features = ["preserve_order"] } +bincode.workspace = true +chrono.workspace = true +itertools.workspace = true +rand.workspace = true diff --git a/experimental/mrpc/plugin/policy/nofile-logging/src/config.rs b/experimental/mrpc/plugin/policy/nofile-logging/src/config.rs new file mode 100644 index 00000000..542f8e79 --- /dev/null +++ b/experimental/mrpc/plugin/policy/nofile-logging/src/config.rs @@ -0,0 +1,39 @@ +use chrono::{Datelike, Timelike, Utc}; +use phoenix_common::log; +use serde::{Deserialize, Serialize}; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +use crate::engine::struct_rpc_events_file; + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct NofileLoggingConfig {} + +impl NofileLoggingConfig { + /// Get config from toml file + pub fn new(config: Option<&str>) -> anyhow::Result { + let config = toml::from_str(config.unwrap_or(""))?; + Ok(config) + } +} + +pub fn create_log_file() -> std::fs::File { + std::fs::create_dir_all("/tmp/phoenix/log").expect("mkdir failed"); + let now = Utc::now(); + let date_string = format!( + "{}-{}-{}-{}-{}-{}", + now.year(), + now.month(), + now.day(), + now.hour(), + now.minute(), + now.second() + ); + let file_name = format!("/tmp/phoenix/log/logging_engine_{}.log", date_string); + ///log::info!("create log file {}", file_name); + let log_file = std::fs::File::create(file_name).expect("create file failed"); + log_file +} diff --git a/experimental/mrpc/plugin/policy/nofile-logging/src/engine.rs b/experimental/mrpc/plugin/policy/nofile-logging/src/engine.rs new file mode 100644 index 00000000..074f59d1 --- /dev/null +++ b/experimental/mrpc/plugin/policy/nofile-logging/src/engine.rs @@ -0,0 +1,273 @@ +use anyhow::{anyhow, Result}; +use futures::future::BoxFuture; +use phoenix_api::rpc::{RpcId, TransportStatus}; +use std::fmt; +use std::fs::File; +use std::io::Write; +use std::num::NonZeroU32; +use std::os::unix::ucred::UCred; +use std::pin::Pin; + +use phoenix_api_policy_nofile_logging::control_plane; + +use phoenix_common::engine::datapath::message::{ + EngineRxMessage, EngineTxMessage, RpcMessageGeneral, +}; + +use phoenix_common::engine::datapath::node::DataPathNode; +use phoenix_common::engine::{future, Decompose, Engine, EngineResult, Indicator, Vertex}; +use phoenix_common::envelop::ResourceDowncast; +use phoenix_common::impl_vertex_for_engine; +use phoenix_common::log; +use phoenix_common::module::Version; + +use phoenix_common::engine::datapath::RpcMessageTx; +use phoenix_common::storage::{ResourceCollection, SharedStorage}; + +use super::DatapathError; +use crate::config::{create_log_file, NofileLoggingConfig}; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +pub mod hello { + include!("proto.rs"); +} + +fn hello_request_name_readonly(req: &hello::HelloRequest) -> String { + let buf = &req.name as &[u8]; + String::from_utf8_lossy(buf).to_string().clone() +} + +pub struct struct_rpc_events_file { + pub timestamp: DateTime, + pub event_type: String, + pub source: String, + pub destination: String, + pub rpc: String, +} +impl struct_rpc_events_file { + pub fn new( + timestamp: DateTime, + event_type: String, + source: String, + destination: String, + rpc: String, + ) -> struct_rpc_events_file { + struct_rpc_events_file { + timestamp: timestamp, + event_type: event_type, + source: source, + destination: destination, + rpc: rpc, + } + } +} +impl fmt::Display for struct_rpc_events_file { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{},", self.timestamp); + write!(f, "{},", self.event_type); + write!(f, "{},", self.source); + write!(f, "{},", self.destination); + write!(f, "{},", self.rpc); + write!(f, "\n") + } +} + +pub(crate) struct NofileLoggingEngine { + pub(crate) node: DataPathNode, + pub(crate) indicator: Indicator, + pub(crate) config: NofileLoggingConfig, + pub(crate) log_file: File, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + Progress(usize), + Disconnected, +} + +use Status::Progress; + +impl Engine for NofileLoggingEngine { + fn activate<'a>(self: Pin<&'a mut Self>) -> BoxFuture<'a, EngineResult> { + Box::pin(async move { self.get_mut().mainloop().await }) + } + + fn description(self: Pin<&Self>) -> String { + "NofileLoggingEngine".to_owned() + } + + #[inline] + fn tracker(self: Pin<&mut Self>) -> &mut Indicator { + &mut self.get_mut().indicator + } + + fn handle_request(&mut self, request: Vec, _cred: UCred) -> Result<()> { + let request: control_plane::Request = bincode::deserialize(&request[..])?; + + match request { + control_plane::Request::NewConfig() => { + self.config = NofileLoggingConfig {}; + } + } + Ok(()) + } +} + +impl_vertex_for_engine!(NofileLoggingEngine, node); + +impl Decompose for NofileLoggingEngine { + fn flush(&mut self) -> Result { + let mut work = 0; + while !self.tx_inputs()[0].is_empty() || !self.rx_inputs()[0].is_empty() { + if let Progress(n) = self.check_input_queue()? { + work += n; + } + } + Ok(work) + } + + fn decompose( + self: Box, + _shared: &mut SharedStorage, + _global: &mut ResourceCollection, + ) -> (ResourceCollection, DataPathNode) { + let engine = *self; + let mut collections = ResourceCollection::with_capacity(4); + collections.insert("config".to_string(), Box::new(engine.config)); + (collections, engine.node) + } +} + +impl NofileLoggingEngine { + pub(crate) fn restore( + mut local: ResourceCollection, + node: DataPathNode, + _prev_version: Version, + ) -> Result { + let config = *local + .remove("config") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let mut log_file = create_log_file(); + + let engine = NofileLoggingEngine { + node, + indicator: Default::default(), + config, + log_file, + }; + Ok(engine) + } +} + +impl NofileLoggingEngine { + async fn mainloop(&mut self) -> EngineResult { + loop { + let mut work = 0; + loop { + match self.check_input_queue()? { + Progress(0) => break, + Progress(n) => work += n, + Status::Disconnected => return Ok(()), + } + } + self.indicator.set_nwork(work); + future::yield_now().await; + } + } +} + +#[inline] +fn materialize_nocopy(msg: &RpcMessageTx) -> &hello::HelloRequest { + let req_ptr = msg.addr_backend as *mut hello::HelloRequest; + let req = unsafe { req_ptr.as_ref().unwrap() }; + return req; +} + +impl NofileLoggingEngine { + fn check_input_queue(&mut self) -> Result { + use phoenix_common::engine::datapath::TryRecvError; + + match self.tx_inputs()[0].try_recv() { + Ok(msg) => { + match msg { + EngineTxMessage::RpcMessage(msg) => { + let meta_ref = unsafe { &*msg.meta_buf_ptr.as_meta_ptr() }; + let mut input = Vec::new(); + input.push(msg); + for event in input + .iter() + .map(|req| { + let rpc_message = materialize_nocopy(req); + struct_rpc_events_file::new( + Utc::now(), + format!("{:?}", meta_ref.msg_type), + format!("{:?}", meta_ref.conn_id), + format!("{:?}", meta_ref.conn_id), + format!("{}", hello_request_name_readonly(rpc_message)), + ) + }) + .collect::>() + { + write!(self.log_file, "{}", event); + } + let output: Vec<_> = input + .iter() + .map(|req| { + RpcMessageGeneral::TxMessage(EngineTxMessage::RpcMessage( + RpcMessageTx::new( + req.meta_buf_ptr.clone(), + req.addr_backend.clone(), + ), + )) + }) + .collect::>(); + + for msg in output { + match msg { + RpcMessageGeneral::TxMessage(msg) => { + self.tx_outputs()[0].send(msg)?; + } + RpcMessageGeneral::RxMessage(msg) => { + self.rx_outputs()[0].send(msg)?; + } + _ => {} + } + } + } + m => self.tx_outputs()[0].send(m)?, + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + return Ok(Status::Disconnected); + } + } + + match self.rx_inputs()[0].try_recv() { + Ok(msg) => { + match msg { + EngineRxMessage::Ack(rpc_id, status) => { + // todo + self.rx_outputs()[0].send(EngineRxMessage::Ack(rpc_id, status))?; + } + EngineRxMessage::RpcMessage(msg) => { + self.rx_outputs()[0].send(EngineRxMessage::RpcMessage(msg))?; + } + m => self.rx_outputs()[0].send(m)?, + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + return Ok(Status::Disconnected); + } + } + Ok(Progress(0)) + } +} diff --git a/experimental/mrpc/plugin/policy/nofile-logging/src/lib.rs b/experimental/mrpc/plugin/policy/nofile-logging/src/lib.rs new file mode 100644 index 00000000..0c2ddb5c --- /dev/null +++ b/experimental/mrpc/plugin/policy/nofile-logging/src/lib.rs @@ -0,0 +1,39 @@ +#![feature(peer_credentials_unix_socket)] +#![feature(ptr_internals)] +#![feature(strict_provenance)] +use thiserror::Error; + +use crate::engine::struct_rpc_events_file; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +pub use phoenix_common::{InitFnResult, PhoenixAddon}; + +pub mod config; +pub(crate) mod engine; +pub mod module; + +#[derive(Error, Debug)] +pub(crate) enum DatapathError { + #[error("Internal queue send error")] + InternalQueueSend, +} + +use phoenix_common::engine::datapath::SendError; +impl From> for DatapathError { + fn from(_other: SendError) -> Self { + DatapathError::InternalQueueSend + } +} + +use crate::config::NofileLoggingConfig; +use crate::module::NofileLoggingAddon; + +#[no_mangle] +pub fn init_addon(config_string: Option<&str>) -> InitFnResult> { + let config = NofileLoggingConfig::new(config_string)?; + let addon = NofileLoggingAddon::new(config); + Ok(Box::new(addon)) +} diff --git a/experimental/mrpc/plugin/policy/nofile-logging/src/module.rs b/experimental/mrpc/plugin/policy/nofile-logging/src/module.rs new file mode 100644 index 00000000..078754c2 --- /dev/null +++ b/experimental/mrpc/plugin/policy/nofile-logging/src/module.rs @@ -0,0 +1,107 @@ +use anyhow::{bail, Result}; +use nix::unistd::Pid; + +use phoenix_common::addon::{PhoenixAddon, Version}; +use phoenix_common::engine::datapath::DataPathNode; +use phoenix_common::engine::{Engine, EngineType}; +use phoenix_common::storage::ResourceCollection; + +use super::engine::NofileLoggingEngine; +use crate::config::{create_log_file, NofileLoggingConfig}; +use crate::engine::struct_rpc_events_file; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +pub(crate) struct NofileLoggingEngineBuilder { + node: DataPathNode, + config: NofileLoggingConfig, +} + +impl NofileLoggingEngineBuilder { + fn new(node: DataPathNode, config: NofileLoggingConfig) -> Self { + NofileLoggingEngineBuilder { node, config } + } + // TODO! LogFile + fn build(self) -> Result { + let mut log_file = create_log_file(); + + Ok(NofileLoggingEngine { + node: self.node, + indicator: Default::default(), + config: self.config, + log_file, + }) + } +} + +pub struct NofileLoggingAddon { + config: NofileLoggingConfig, +} + +impl NofileLoggingAddon { + pub const NOFILE_LOGGING_ENGINE: EngineType = EngineType("NofileLoggingEngine"); + pub const ENGINES: &'static [EngineType] = &[NofileLoggingAddon::NOFILE_LOGGING_ENGINE]; +} + +impl NofileLoggingAddon { + pub fn new(config: NofileLoggingConfig) -> Self { + NofileLoggingAddon { config } + } +} + +impl PhoenixAddon for NofileLoggingAddon { + fn check_compatibility(&self, _prev: Option<&Version>) -> bool { + true + } + + fn decompose(self: Box) -> ResourceCollection { + let addon = *self; + let mut collections = ResourceCollection::new(); + collections.insert("config".to_string(), Box::new(addon.config)); + collections + } + + #[inline] + fn migrate(&mut self, _prev_addon: Box) {} + + fn engines(&self) -> &[EngineType] { + NofileLoggingAddon::ENGINES + } + + fn update_config(&mut self, config: &str) -> Result<()> { + self.config = toml::from_str(config)?; + Ok(()) + } + + fn create_engine( + &mut self, + ty: EngineType, + _pid: Pid, + node: DataPathNode, + ) -> Result> { + if ty != NofileLoggingAddon::NOFILE_LOGGING_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let builder = NofileLoggingEngineBuilder::new(node, self.config); + let engine = builder.build()?; + Ok(Box::new(engine)) + } + + fn restore_engine( + &mut self, + ty: EngineType, + local: ResourceCollection, + node: DataPathNode, + prev_version: Version, + ) -> Result> { + if ty != NofileLoggingAddon::NOFILE_LOGGING_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let engine = NofileLoggingEngine::restore(local, node, prev_version)?; + Ok(Box::new(engine)) + } +} diff --git a/experimental/mrpc/plugin/policy/nofile-logging/src/proto.rs b/experimental/mrpc/plugin/policy/nofile-logging/src/proto.rs new file mode 100644 index 00000000..83ee8ab0 --- /dev/null +++ b/experimental/mrpc/plugin/policy/nofile-logging/src/proto.rs @@ -0,0 +1,29 @@ +/// The request message containing the user's name. +#[repr(C)] +#[derive(Debug, Clone, ::mrpc_derive::Message)] +pub struct HelloRequest { + #[prost(bytes = "vec", tag = "1")] + pub name: ::mrpc_marshal::shadow::Vec, +} +/// The response message containing the greetings +#[repr(C)] +#[derive(Debug, ::mrpc_derive::Message)] +pub struct HelloReply { + #[prost(bytes = "vec", tag = "1")] + pub message: ::mrpc_marshal::shadow::Vec, +} + +// /// The request message containing the user's name. +// #[repr(C)] +// #[derive(Debug, Clone, ::mrpc_derive::Message)] +// pub struct HelloRequest { +// #[prost(bytes = "vec", tag = "1")] +// pub name: ::mrpc::alloc::Vec, +// } +// /// The response message containing the greetings +// #[repr(C)] +// #[derive(Debug, ::mrpc_derive::Message)] +// pub struct HelloReply { +// #[prost(bytes = "vec", tag = "1")] +// pub message: ::mrpc::alloc::Vec, +// } diff --git a/experimental/mrpc/plugin/policy/null/src/config.rs b/experimental/mrpc/plugin/policy/null/src/config.rs index a31cfae3..5a93ab2f 100644 --- a/experimental/mrpc/plugin/policy/null/src/config.rs +++ b/experimental/mrpc/plugin/policy/null/src/config.rs @@ -1,5 +1,6 @@ use serde::{Deserialize, Serialize}; +/// TODO Add your own config(parameter) here #[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct NullConfig {} diff --git a/experimental/mrpc/plugin/policy/null/src/engine.rs b/experimental/mrpc/plugin/policy/null/src/engine.rs index 1530a573..9bc8b1f3 100644 --- a/experimental/mrpc/plugin/policy/null/src/engine.rs +++ b/experimental/mrpc/plugin/policy/null/src/engine.rs @@ -22,6 +22,7 @@ pub(crate) struct NullEngine { pub(crate) indicator: Indicator, pub(crate) config: NullConfig, + // TODO Add you own internal state here } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -81,6 +82,8 @@ impl Decompose for NullEngine { let mut collections = ResourceCollection::with_capacity(4); collections.insert("config".to_string(), Box::new(engine.config)); (collections, engine.node) + // TODO if you have internal state that need to be stored + // add it to collections } } @@ -95,7 +98,8 @@ impl NullEngine { .unwrap() .downcast::() .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; - + // TODO if you have stored internal state + // load it from collection here let engine = NullEngine { node, indicator: Default::default(), @@ -128,11 +132,11 @@ impl NullEngine { impl NullEngine { fn check_input_queue(&mut self) -> Result { use phoenix_common::engine::datapath::TryRecvError; - match self.tx_inputs()[0].try_recv() { Ok(msg) => { match msg { EngineTxMessage::RpcMessage(msg) => { + // TODO Add logic for tx input here self.tx_outputs()[0].send(EngineTxMessage::RpcMessage(msg))?; } m => self.tx_outputs()[0].send(m)?, @@ -143,6 +147,17 @@ impl NullEngine { Err(TryRecvError::Disconnected) => return Ok(Status::Disconnected), } + match self.rx_inputs()[0].try_recv() { + Ok(m) => { + // TODO Add logic for rx input here + // use match statement like tx + self.rx_outputs()[0].send(m)?; + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => return Ok(Status::Disconnected), + } + Ok(Progress(0)) } } diff --git a/experimental/mrpc/plugin/policy/ratelimit/src/engine.rs b/experimental/mrpc/plugin/policy/ratelimit/src/engine.rs index 9ca1581d..46480663 100644 --- a/experimental/mrpc/plugin/policy/ratelimit/src/engine.rs +++ b/experimental/mrpc/plugin/policy/ratelimit/src/engine.rs @@ -34,6 +34,7 @@ pub(crate) struct RateLimitEngine { // The number of available tokens in the token bucket algorithm. pub(crate) num_tokens: f64, // The queue to buffer the requests that cannot be sent immediately. + // This is RPC buffer pub(crate) queue: VecDeque, } diff --git a/mdbook/src/tutorials/adn.md b/mdbook/src/tutorials/adn.md new file mode 100644 index 00000000..f37af661 --- /dev/null +++ b/mdbook/src/tutorials/adn.md @@ -0,0 +1,10 @@ +cargo run --release -p rpc_bench --bin rpc_bench_server -- --transport tcp + +cargo run --release -p rpc_bench --bin rpc_bench_client -- --transport tcp -D 600 -i 1 --req-size 64 -c + + +cargo run --release --bin addonctl -- --config eval/policy/nofile-logging/attach.toml --pid --sid 1 +cargo run --release --bin addonctl -- --config eval/policy/logging/attach.toml --pid --sid 1 + +cargo run --release --bin addonctl -- --config eval/policy/logging/detach.toml --pid --sid 1 +cargo run --release --bin addonctl -- --config eval/policy/nofile-logging/attach.toml --pid --sid 1 \ No newline at end of file diff --git a/src/phoenix_common/src/engine/datapath/message.rs b/src/phoenix_common/src/engine/datapath/message.rs index b8cb3579..60eac16a 100644 --- a/src/phoenix_common/src/engine/datapath/message.rs +++ b/src/phoenix_common/src/engine/datapath/message.rs @@ -9,6 +9,13 @@ use super::meta_pool::MetaBufferPtr; // TODO(cjr): Should be repr(C) +#[derive(Debug)] +pub enum RpcMessageGeneral { + TxMessage(EngineTxMessage), + RxMessage(EngineRxMessage), + Pass, +} + #[derive(Debug)] pub struct RpcMessageTx { // Each RPC message is assigned a buffer for meta and optionally for its data @@ -16,6 +23,15 @@ pub struct RpcMessageTx { pub addr_backend: usize, } +impl RpcMessageTx { + pub fn new(meta_buf_ptr: MetaBufferPtr, addr_backend: usize) -> RpcMessageTx { + RpcMessageTx { + meta_buf_ptr, + addr_backend, + } + } +} + #[derive(Debug)] pub enum EngineTxMessage { RpcMessage(RpcMessageTx),