From 3ead102cd63febef249409b01f743301b2520ae2 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 3 May 2024 11:54:44 +0800 Subject: [PATCH 1/7] debug ci --- .github/workflows/ci.yml | 197 +---- .../org/apache/fury/CrossLanguageTest.java | 739 ------------------ 2 files changed, 3 insertions(+), 933 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c22abdeae5..5ef81e1ce8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: MY_VAR: "PATH" strategy: matrix: - java-version: ["8", "11", "17", "21"] + java-version: ["17", "21"] steps: - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java-version }} @@ -54,198 +54,7 @@ jobs: with: java-version: ${{ matrix.java-version }} distribution: 'temurin' - - name: Install bazel - run: ./ci/run_ci.sh install_bazel - - name: Install python - run: ./ci/run_ci.sh install_python - - name: Install pyfury - run: ./ci/run_ci.sh install_pyfury + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 - name: Run CI with Maven run: ./ci/run_ci.sh java${{ matrix.java-version }} - openj9: - name: Openj9 Java CI - runs-on: ubuntu-latest - env: - MY_VAR: "PATH" - strategy: - matrix: - # String in openj9 1.8 share byte array by offset, fury doesn't allow it. - java-version: ["21"] - steps: - - uses: actions/checkout@v4 - - name: Set up JDK ${{ matrix.java-version }} - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'adopt-openj9' - - name: Install bazel - run: ./ci/run_ci.sh install_bazel - - name: Install python - run: ./ci/run_ci.sh install_python - - name: Install pyfury - run: ./ci/run_ci.sh install_pyfury - - name: Run CI with Maven - run: ./ci/run_ci.sh java${{ matrix.java-version }} - graalvm: - name: GraalVM CI - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: graalvm/setup-graalvm@v1 - with: - java-version: '21' - distribution: 'graalvm' - github-token: ${{ secrets.GITHUB_TOKEN }} - native-image-job-reports: 'true' - - name: Build native image and run - run: ./ci/run_ci.sh graalvm_test - - scala: - name: Scala CI - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up JDK8 - uses: actions/setup-java@v4 - with: - java-version: 8 - distribution: 'temurin' - - name: Install fury java - run: cd java && mvn -T10 --no-transfer-progress clean install -DskipTests && cd - - - name: Test - run: | - # Avoid sbt download jackson-databind error - rm -rf /home/runner/.m2/repository/com/fasterxml/jackson/ - cd scala && sbt +test && cd - - - integration_tests: - name: Integration Tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up JDK8 - uses: actions/setup-java@v4 - with: - java-version: 8 - distribution: 'temurin' - - name: Run CI - run: ./ci/run_ci.sh integration_tests - - javascript: - name: JavaScript CI - strategy: - matrix: - node-version: [14.x, 16.x, 18.x, 20.x] - os: [ubuntu-latest, macos-13, windows-2022] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - - name: Upgrade npm - run: npm install -g npm@8 - # node-gyp needs to use python and relies on the distutils module. - # The distutils module has been removed starting from python 3.12 - # (see https://docs.python.org/3.10/library/distutils.html). Some - # OS (such as macos -latest) uses python3.12 by default, so python 3.8 - # is used here to avoid this problem. - - name: Set up Python3.8 - uses: actions/setup-python@v5 - with: - python-version: 3.8 - - name: Run CI with NodeJS - run: python ./ci/run_ci.py javascript - - rust: - name: Rust CI - strategy: - matrix: - os: [ubuntu-latest, macos-12, macos-14] # macos-12: x86, macos-14: arm64 - runs-on: ${{ matrix.os }} - timeout-minutes: 45 - steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: 3.11 - - uses: dtolnay/rust-toolchain@nightly - - name: Run Rust CI - run: python ./ci/run_ci.py rust - - cpp: - name: C++ CI - strategy: - matrix: - os: [ubuntu-latest, macos-12, macos-14] # macos-12: x86, macos-14: arm64 - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: 3.11 - - name: Run C++ CI with Bazel - run: python ./ci/run_ci.py cpp - python: - name: Python CI - # Fix python 3.6 install issue, see - # https://github.com/rwth-i6/returnn/commit/38ecab17d781c4b74db6a174c8097187380b4ddc - runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: [3.7, 3.12] - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install bazel - run: ./ci/run_ci.sh install_bazel - - name: Run Python CI - run: ./ci/run_ci.sh python - - go: - name: Golang CI - runs-on: ubuntu-latest - strategy: - matrix: - go-version: [ '1.13', '1.18' ] - steps: - - uses: actions/checkout@v4 - - name: Setup Go ${{ matrix.go-version }} - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go-version }} - - name: Display Go version - run: go version - - name: Install bazel - run: ./ci/run_ci.sh install_bazel - - name: Install python - run: ./ci/run_ci.sh install_python - - name: Install pyfury - run: ./ci/run_ci.sh install_pyfury - - name: Run Golang CI - run: ./ci/run_ci.sh go - - lint: - name: Code Style Check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up JDK ${{ matrix.java-version }} - uses: actions/setup-java@v4 - with: - java-version: 21 - distribution: 'oracle' - - name: Check License Header - uses: korandoru/hawkeye@v3 - - name: Set up Python 3.8 - uses: actions/setup-python@v5 - with: - python-version: 3.8 - - name: Check code style - run: ./ci/run_ci.sh format diff --git a/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java b/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java index 8a85c678df..e69de29bb2 100644 --- a/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java @@ -1,739 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fury; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.hash.Hashing; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; -import java.time.Instant; -import java.time.LocalDate; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiConsumer; -import java.util.function.Consumer; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import lombok.Data; -import org.apache.fury.config.Language; -import org.apache.fury.logging.Logger; -import org.apache.fury.logging.LoggerFactory; -import org.apache.fury.memory.MemoryBuffer; -import org.apache.fury.memory.MemoryUtils; -import org.apache.fury.serializer.BufferObject; -import org.apache.fury.serializer.Serializer; -import org.apache.fury.util.MurmurHash3; -import org.testng.Assert; -import org.testng.annotations.Test; - -/** Tests in this class need fury python installed. */ -@Test -public class CrossLanguageTest { - private static final Logger LOG = LoggerFactory.getLogger(CrossLanguageTest.class); - private static final String PYTHON_MODULE = "pyfury.tests.test_cross_language"; - private static final String PYTHON_EXECUTABLE = "python"; - - /** - * Execute an external command. - * - * @return Whether the command succeeded. - */ - private boolean executeCommand(List command, int waitTimeoutSeconds) { - return executeCommand( - command, waitTimeoutSeconds, ImmutableMap.of("ENABLE_CROSS_LANGUAGE_TESTS", "true")); - } - - private boolean executeCommand( - List command, int waitTimeoutSeconds, Map env) { - try { - LOG.info("Executing command: {}", String.join(" ", command)); - ProcessBuilder processBuilder = - new ProcessBuilder(command) - .redirectOutput(ProcessBuilder.Redirect.INHERIT) - .redirectError(ProcessBuilder.Redirect.INHERIT); - for (Map.Entry entry : env.entrySet()) { - processBuilder.environment().put(entry.getKey(), entry.getValue()); - } - Process process = processBuilder.start(); - process.waitFor(waitTimeoutSeconds, TimeUnit.SECONDS); - return process.exitValue() == 0; - } catch (Exception e) { - throw new RuntimeException("Error executing command " + String.join(" ", command), e); - } - } - - @Data - public static class A { - public Integer f1; - public Map f2; - - public static A create() { - A a = new A(); - a.f1 = 1; - a.f2 = new HashMap<>(); - a.f2.put("pid", "12345"); - a.f2.put("ip", "0.0.0.0"); - a.f2.put("k1", "v1"); - return a; - } - } - - @Test - public void testBuffer() throws IOException { - MemoryBuffer buffer = MemoryUtils.buffer(32); - buffer.writeBoolean(true); - buffer.writeByte(Byte.MAX_VALUE); - buffer.writeInt16(Short.MAX_VALUE); - buffer.writeInt32(Integer.MAX_VALUE); - buffer.writeInt64(Long.MAX_VALUE); - buffer.writeFloat32(-1.1f); - buffer.writeFloat64(-1.1); - buffer.writeVarUint32(100); - byte[] bytes = {'a', 'b'}; - buffer.writeInt32(bytes.length); - buffer.writeBytes(bytes); - Path dataFile = Files.createTempFile("test_buffer", "data"); - Files.write(dataFile, buffer.getBytes(0, buffer.writerIndex())); - ImmutableList command = - ImmutableList.of( - PYTHON_EXECUTABLE, - "-m", - PYTHON_MODULE, - "test_buffer", - dataFile.toAbsolutePath().toString()); - Assert.assertTrue(executeCommand(command, 30)); - buffer = MemoryUtils.wrap(Files.readAllBytes(dataFile)); - Assert.assertTrue(buffer.readBoolean()); - Assert.assertEquals(buffer.readByte(), Byte.MAX_VALUE); - Assert.assertEquals(buffer.readInt16(), Short.MAX_VALUE); - Assert.assertEquals(buffer.readInt32(), Integer.MAX_VALUE); - Assert.assertEquals(buffer.readInt64(), Long.MAX_VALUE); - Assert.assertEquals(buffer.readFloat32(), -1.1f, 0.0001); - Assert.assertEquals(buffer.readFloat64(), -1.1, 0.0001); - Assert.assertEquals(buffer.readVarUint32(), 100); - Assert.assertTrue(Arrays.equals(buffer.readBytes(buffer.readInt32()), bytes)); - } - - @Test - public void testMurmurHash3() throws IOException { - MemoryBuffer buffer = MemoryUtils.buffer(32); - byte[] hash1 = Hashing.murmur3_128(47).hashBytes(new byte[] {1, 2, 8}).asBytes(); - buffer.writeBytes(hash1); - byte[] hash2 = - Hashing.murmur3_128(47) - .hashBytes("01234567890123456789".getBytes(StandardCharsets.UTF_8)) - .asBytes(); - buffer.writeBytes(hash2); - Path dataFile = Files.createTempFile("test_murmurhash3", "data"); - Files.write(dataFile, buffer.getBytes(0, buffer.writerIndex())); - ImmutableList command = - ImmutableList.of( - PYTHON_EXECUTABLE, - "-m", - PYTHON_MODULE, - "test_murmurhash3", - dataFile.toAbsolutePath().toString()); - Assert.assertTrue(executeCommand(command, 30)); - long[] longs = MurmurHash3.murmurhash3_x64_128(new byte[] {1, 2, 8}, 0, 3, 47); - buffer.writerIndex(0); - buffer.writeInt64(longs[0]); - buffer.writeInt64(longs[1]); - Files.write( - dataFile, buffer.getBytes(0, buffer.writerIndex()), StandardOpenOption.TRUNCATE_EXISTING); - Assert.assertTrue(executeCommand(command, 30)); - } - - /** Keep this in sync with `foo_schema` in test_cross_language.py */ - @Data - public static class Foo { - public Integer f1; - public String f2; - public List f3; - public Map f4; - public Bar f5; - - public static Foo create() { - Foo foo = new Foo(); - foo.f1 = 1; - foo.f2 = "str"; - foo.f3 = Arrays.asList("str1", null, "str2"); - foo.f4 = - new HashMap() { - { - put("k1", 1); - put("k2", 2); - put("k3", 3); - put("k4", 4); - put("k5", 5); - put("k6", 6); - } - }; - foo.f5 = Bar.create(); - return foo; - } - } - - /** Keep this in sync with `bar_schema` in test_cross_language.py */ - @Data - public static class Bar { - public Integer f1; - public String f2; - - public static Bar create() { - Bar bar = new Bar(); - bar.f1 = 1; - bar.f2 = "str"; - return bar; - } - } - - @Test - public void testCrossLanguageSerializer() throws Exception { - Fury fury = - Fury.builder() - .withLanguage(Language.XLANG) - .withRefTracking(true) - .requireClassRegistration(false) - .build(); - MemoryBuffer buffer = MemoryUtils.buffer(32); - fury.serialize(buffer, true); - fury.serialize(buffer, false); - fury.serialize(buffer, -1); - fury.serialize(buffer, Byte.MAX_VALUE); - fury.serialize(buffer, Byte.MIN_VALUE); - fury.serialize(buffer, Short.MAX_VALUE); - fury.serialize(buffer, Short.MIN_VALUE); - fury.serialize(buffer, Integer.MAX_VALUE); - fury.serialize(buffer, Integer.MIN_VALUE); - fury.serialize(buffer, Long.MAX_VALUE); - fury.serialize(buffer, Long.MIN_VALUE); - fury.serialize(buffer, -1.f); - fury.serialize(buffer, -1.d); - fury.serialize(buffer, "str"); - LocalDate day = LocalDate.of(2021, 11, 23); - fury.serialize(buffer, day); - Instant instant = Instant.ofEpochSecond(100); - fury.serialize(buffer, instant); - List list = Arrays.asList("a", 1, -1.0, instant, day); - fury.serialize(buffer, list); - Map map = new HashMap<>(); - for (int i = 0; i < list.size(); i++) { - map.put("k" + i, list.get(i)); - map.put(list.get(i), list.get(i)); - } - fury.serialize(buffer, map); - Set set = new HashSet<>(list); - fury.serialize(buffer, set); - - // test primitive arrays - fury.serialize(buffer, new boolean[] {true, false}); - fury.serialize(buffer, new short[] {1, Short.MAX_VALUE}); - fury.serialize(buffer, new int[] {1, Integer.MAX_VALUE}); - fury.serialize(buffer, new long[] {1, Long.MAX_VALUE}); - fury.serialize(buffer, new float[] {1.f, 2.f}); - fury.serialize(buffer, new double[] {1.0, 2.0}); - - BiConsumer function = - (MemoryBuffer buf, Boolean useToString) -> { - assertStringEquals(fury.deserialize(buf), true, useToString); - assertStringEquals(fury.deserialize(buf), false, useToString); - assertStringEquals(fury.deserialize(buf), -1, useToString); - assertStringEquals(fury.deserialize(buf), Byte.MAX_VALUE, useToString); - assertStringEquals(fury.deserialize(buf), Byte.MIN_VALUE, useToString); - assertStringEquals(fury.deserialize(buf), Short.MAX_VALUE, useToString); - assertStringEquals(fury.deserialize(buf), Short.MIN_VALUE, useToString); - assertStringEquals(fury.deserialize(buf), Integer.MAX_VALUE, useToString); - assertStringEquals(fury.deserialize(buf), Integer.MIN_VALUE, useToString); - assertStringEquals(fury.deserialize(buf), Long.MAX_VALUE, useToString); - assertStringEquals(fury.deserialize(buf), Long.MIN_VALUE, useToString); - assertStringEquals(fury.deserialize(buf), -1.f, useToString); - assertStringEquals(fury.deserialize(buf), -1.d, useToString); - assertStringEquals(fury.deserialize(buf), "str", useToString); - assertStringEquals(fury.deserialize(buf), day, useToString); - assertStringEquals(fury.deserialize(buf), instant, useToString); - assertStringEquals(fury.deserialize(buf), list, useToString); - assertStringEquals(fury.deserialize(buf), map, useToString); - assertStringEquals(fury.deserialize(buf), set, useToString); - assertStringEquals(fury.deserialize(buf), new boolean[] {true, false}, false); - assertStringEquals(fury.deserialize(buf), new short[] {1, Short.MAX_VALUE}, false); - assertStringEquals(fury.deserialize(buf), new int[] {1, Integer.MAX_VALUE}, false); - assertStringEquals(fury.deserialize(buf), new long[] {1, Long.MAX_VALUE}, false); - assertStringEquals(fury.deserialize(buf), new float[] {1.f, 2.f}, false); - assertStringEquals(fury.deserialize(buf), new double[] {1.0, 2.0}, false); - }; - function.accept(buffer, false); - - Path dataFile = Files.createTempFile("test_cross_language_serializer", "data"); - // Files.deleteIfExists(Paths.get("test_cross_language_serializer.data")); - // Path dataFile = Files.createFile(Paths.get("test_cross_language_serializer.data")); - Files.write(dataFile, buffer.getBytes(0, buffer.writerIndex())); - ImmutableList command = - ImmutableList.of( - PYTHON_EXECUTABLE, - "-m", - PYTHON_MODULE, - "test_cross_language_serializer", - dataFile.toAbsolutePath().toString()); - Assert.assertTrue(executeCommand(command, 30)); - MemoryBuffer buffer2 = MemoryUtils.wrap(Files.readAllBytes(dataFile)); - function.accept(buffer2, true); - } - - @Test - public void testCrossLanguagePreserveTypes() { - Fury fury = - Fury.builder() - .withLanguage(Language.XLANG) - .withRefTracking(true) - .requireClassRegistration(false) - .build(); - Assert.assertEquals( - new String[] {"str", "str"}, (Object[]) serDe(fury, new String[] {"str", "str"})); - Assert.assertEquals(new Object[] {"str", 1}, (Object[]) serDe(fury, new Object[] {"str", 1})); - Assert.assertTrue( - Arrays.deepEquals( - new Integer[][] {{1, 2}, {1, 2}}, - (Integer[][]) serDe(fury, new Integer[][] {{1, 2}, {1, 2}}))); - - Assert.assertEquals(Arrays.asList(1, 2), serDe(fury, Arrays.asList(1, 2))); - List arrayList = Arrays.asList("str", "str"); - Assert.assertEquals(arrayList, serDe(fury, arrayList)); - Assert.assertEquals(new LinkedList<>(arrayList), serDe(fury, new LinkedList<>(arrayList))); - Assert.assertEquals(new HashSet<>(arrayList), serDe(fury, new HashSet<>(arrayList))); - TreeSet treeSet = new TreeSet<>(Comparator.naturalOrder()); - treeSet.add("str1"); - treeSet.add("str2"); - Assert.assertEquals(treeSet, serDe(fury, treeSet)); - - HashMap hashMap = new HashMap<>(); - hashMap.put("k1", 1); - hashMap.put("k2", 2); - Assert.assertEquals(hashMap, serDe(fury, hashMap)); - Assert.assertEquals(new LinkedHashMap<>(hashMap), serDe(fury, new LinkedHashMap<>(hashMap))); - Assert.assertEquals(Collections.EMPTY_LIST, serDe(fury, Collections.EMPTY_LIST)); - Assert.assertEquals(Collections.EMPTY_SET, serDe(fury, Collections.EMPTY_SET)); - Assert.assertEquals(Collections.EMPTY_MAP, serDe(fury, Collections.EMPTY_MAP)); - Assert.assertEquals( - Collections.singletonList("str"), serDe(fury, Collections.singletonList("str"))); - Assert.assertEquals(Collections.singleton("str"), serDe(fury, Collections.singleton("str"))); - Assert.assertEquals( - Collections.singletonMap("k", 1), serDe(fury, Collections.singletonMap("k", 1))); - } - - @SuppressWarnings("unchecked") - private void assertStringEquals(Object actual, Object expected, boolean useToString) { - if (useToString) { - if (expected instanceof Map) { - Map actualMap = - (Map) - ((Map) actual) - .entrySet().stream() - .collect( - Collectors.toMap( - (Map.Entry e) -> e.getKey().toString(), - (Map.Entry e) -> e.getValue().toString())); - Map expectedMap = - (Map) - ((Map) expected) - .entrySet().stream() - .collect( - Collectors.toMap( - (Map.Entry e) -> e.getKey().toString(), - (Map.Entry e) -> e.getValue().toString())); - Assert.assertEquals(actualMap, expectedMap); - } else if (expected instanceof Set) { - Object actualSet = - ((Set) actual).stream().map(Object::toString).collect(Collectors.toSet()); - Object expectedSet = - ((Set) expected).stream().map(Object::toString).collect(Collectors.toSet()); - Assert.assertEquals(actualSet, expectedSet); - } else { - Assert.assertEquals(actual.toString(), expected.toString()); - } - } else { - Assert.assertEquals(actual, expected); - } - } - - private Object serDe(Fury fury, Object obj) { - byte[] bytes = fury.serialize(obj); - return fury.deserialize(bytes); - } - - @SuppressWarnings("unchecked") - @Test - public void testCrossLanguageReference() throws Exception { - Fury fury = - Fury.builder() - .withLanguage(Language.XLANG) - .withRefTracking(true) - .requireClassRegistration(false) - .build(); - List list = new ArrayList<>(); - Map map = new HashMap<>(); - list.add(list); - list.add(map); - map.put("k1", map); - map.put("k2", list); - MemoryBuffer buffer = MemoryUtils.buffer(32); - fury.serialize(buffer, list); - - Consumer function = - (MemoryBuffer buf) -> { - List newList = (List) fury.deserialize(buf); - Assert.assertNotNull(newList); - Assert.assertSame(newList, newList.get(0)); - Map newMap = (Map) newList.get(1); - Assert.assertSame(newMap.get("k1"), newMap); - Assert.assertSame(newMap.get("k2"), newList); - }; - - Path dataFile = Files.createTempFile("test_cross_language_reference", "data"); - // Files.deleteIfExists(Paths.get("test_cross_language_reference.data")); - // Path dataFile = Files.createFile(Paths.get("test_cross_language_reference.data")); - Files.write(dataFile, buffer.getBytes(0, buffer.writerIndex())); - ImmutableList command = - ImmutableList.of( - PYTHON_EXECUTABLE, - "-m", - PYTHON_MODULE, - "test_cross_language_reference", - dataFile.toAbsolutePath().toString()); - Assert.assertTrue(executeCommand(command, 30)); - MemoryBuffer buffer2 = MemoryUtils.wrap(Files.readAllBytes(dataFile)); - function.accept(buffer2); - } - - @Data - public static class ComplexObject1 { - Object f1; - String f2; - List f3; - Map f4; - Byte f5; - Short f6; - Integer f7; - Long f8; - Float f9; - Double f10; - short[] f11; - List f12; - } - - @Data - public static class ComplexObject2 { - Object f1; - Map f2; - } - - @Test - public void testSerializeSimpleStruct() throws Exception { - Fury fury = - Fury.builder() - .withLanguage(Language.XLANG) - .withRefTracking(true) - .requireClassRegistration(false) - .build(); - fury.register(ComplexObject2.class, "test.ComplexObject2"); - ComplexObject2 obj2 = new ComplexObject2(); - obj2.f1 = true; - obj2.f2 = new HashMap<>(ImmutableMap.of((byte) -1, 2)); - structRoundBack(fury, obj2, "test_serialize_simple_struct"); - } - - public void testSerializeComplexStruct() throws Exception { - Fury fury = - Fury.builder() - .withLanguage(Language.XLANG) - .withRefTracking(true) - .requireClassRegistration(false) - .build(); - fury.register(ComplexObject1.class, "test.ComplexObject1"); - fury.register(ComplexObject2.class, "test.ComplexObject2"); - ComplexObject2 obj2 = new ComplexObject2(); - obj2.f1 = true; - obj2.f2 = ImmutableMap.of((byte) -1, 2); - ComplexObject1 obj = new ComplexObject1(); - obj.f1 = obj2; - obj.f2 = "abc"; - obj.f3 = Arrays.asList("abc", "abc"); - obj.f4 = ImmutableMap.of((byte) 1, 2); - obj.f5 = Byte.MAX_VALUE; - obj.f6 = Short.MAX_VALUE; - obj.f7 = Integer.MAX_VALUE; - obj.f8 = Long.MAX_VALUE; - obj.f9 = 1.0f / 2; - obj.f10 = 1 / 3.0; - obj.f11 = new short[] {(short) 1, (short) 2}; - obj.f12 = ImmutableList.of((short) -1, (short) 4); - - structRoundBack(fury, obj, "test_serialize_complex_struct"); - } - - private void structRoundBack(Fury fury, Object obj, String testName) throws IOException { - byte[] serialized = fury.serialize(obj); - Assert.assertEquals(fury.deserialize(serialized), obj); - Path dataFile = Paths.get(testName); - System.out.println(dataFile.toAbsolutePath()); - Files.deleteIfExists(dataFile); - Files.write(dataFile, serialized); - dataFile.toFile().deleteOnExit(); - ImmutableList command = - ImmutableList.of( - PYTHON_EXECUTABLE, "-m", PYTHON_MODULE, testName, dataFile.toAbsolutePath().toString()); - Assert.assertTrue(executeCommand(command, 30)); - Assert.assertEquals(fury.deserialize(Files.readAllBytes(dataFile)), obj); - } - - @Test - public void testSerializeOpaqueObjectSimple() { - Fury fury = - Fury.builder() - .withLanguage(Language.XLANG) - .withRefTracking(true) - .requireClassRegistration(false) - .build(); - fury.register(ComplexObject2.class, "test.ComplexObject2"); - ComplexObject2 obj = new ComplexObject2(); - obj.f1 = Foo.create(); - byte[] serialized = fury.serialize(obj); - Assert.assertEquals(fury.deserialize(serialized), obj); - } - - @Test - public void testSerializeOpaqueObject() throws Exception { - Fury fury = - Fury.builder() - .withLanguage(Language.XLANG) - .withRefTracking(true) - .requireClassRegistration(false) - .build(); - fury.register(ComplexObject1.class, "test.ComplexObject1"); - // don't register ComplexObject2/Foo to make them serialize as opaque blobs. - ComplexObject1 obj = new ComplexObject1(); - obj.f1 = new ComplexObject2(); - ((ComplexObject2) obj.f1).f1 = true; - ((ComplexObject2) obj.f1).f2 = new HashMap<>(ImmutableMap.of((byte) -1, 2)); - obj.f2 = "abc"; - obj.f3 = Arrays.asList(obj.f1, Foo.create()); - byte[] serialized = fury.serialize(obj); - Assert.assertEquals(fury.deserialize(serialized), obj); - - Path dataFile = Files.createTempFile("test_serialize_opaque_object", "data"); - ImmutableList command = - ImmutableList.of( - PYTHON_EXECUTABLE, - "-m", - PYTHON_MODULE, - "test_serialize_opaque_object", - dataFile.toAbsolutePath().toString()); - Files.write(dataFile, serialized); - Assert.assertTrue(executeCommand(command, 30)); - // TODO support OpaqueObject itself serialization - // ComplexObject1 newObj = (ComplexObject1) fury.deserialize(Files.readAllBytes(dataFile)); - // assertEquals(newObj.f1.getClass(), OpaqueObjects.OpaqueObject.class); - // assertEquals(newObj.f2, obj.f2); - // assertNotNull(newObj.f3); - // assertTrue(newObj.f3.get(0) instanceof OpaqueObjects.OpaqueObject); - // assertTrue(newObj.f3.get(1) instanceof OpaqueObjects.OpaqueObject); - } - - private static class ComplexObject1Serializer extends Serializer { - - public ComplexObject1Serializer(Fury fury, Class cls) { - super(fury, cls); - } - - @Override - public void write(MemoryBuffer buffer, ComplexObject1 value) { - xwrite(buffer, value); - } - - @Override - public ComplexObject1 read(MemoryBuffer buffer) { - return xread(buffer); - } - - @Override - public short getXtypeId() { - return Fury.FURY_TYPE_TAG_ID; - } - - @Override - public String getCrossLanguageTypeTag() { - return "test.ComplexObject1"; - } - - @Override - public void xwrite(MemoryBuffer buffer, ComplexObject1 value) { - fury.xwriteRef(buffer, value.f1); - fury.xwriteRef(buffer, value.f2); - fury.xwriteRef(buffer, value.f3); - } - - @SuppressWarnings("unchecked") - @Override - public ComplexObject1 xread(MemoryBuffer buffer) { - ComplexObject1 obj = new ComplexObject1(); - fury.getRefResolver().reference(obj); - obj.f1 = fury.xreadRef(buffer); - obj.f2 = (String) fury.xreadRef(buffer); - obj.f3 = (List) fury.xreadRef(buffer); - return obj; - } - } - - @Test - public void testRegisterSerializer() throws Exception { - Fury fury = - Fury.builder() - .withLanguage(Language.XLANG) - .withRefTracking(true) - .requireClassRegistration(false) - .build(); - fury.registerSerializer(ComplexObject1.class, ComplexObject1Serializer.class); - ComplexObject1 obj = new ComplexObject1(); - obj.f1 = true; - obj.f2 = "abc"; - obj.f3 = Arrays.asList("abc", "abc"); - byte[] serialized = fury.serialize(obj); - Assert.assertEquals(fury.deserialize(serialized), obj); - Path dataFile = Files.createTempFile("test_register_serializer", "data"); - Files.write(dataFile, serialized); - ImmutableList command = - ImmutableList.of( - PYTHON_EXECUTABLE, - "-m", - PYTHON_MODULE, - "test_register_serializer", - dataFile.toAbsolutePath().toString()); - Assert.assertTrue(executeCommand(command, 30)); - Assert.assertEquals(fury.deserialize(Files.readAllBytes(dataFile)), obj); - } - - @Test - public void testOutOfBandBuffer() throws Exception { - Fury fury = - Fury.builder() - .withLanguage(Language.XLANG) - .withRefTracking(true) - .requireClassRegistration(false) - .build(); - AtomicInteger counter = new AtomicInteger(0); - List data = - IntStream.range(0, 10).mapToObj(i -> new byte[] {0, 1}).collect(Collectors.toList()); - List bufferObjects = new ArrayList<>(); - byte[] serialized = - fury.serialize( - data, - o -> { - if (counter.incrementAndGet() % 2 == 0) { - bufferObjects.add(o); - return false; - } else { - return true; - } - }); - List buffers = - bufferObjects.stream().map(BufferObject::toBuffer).collect(Collectors.toList()); - List newObj = (List) fury.deserialize(serialized, buffers); - for (int i = 0; i < data.size(); i++) { - Assert.assertEquals(data.get(i), newObj.get(i)); - } - - Path intBandDataFile = Files.createTempFile("test_oob_buffer_in_band", "data"); - Files.write(intBandDataFile, serialized); - Path outOfBandDataFile = Files.createTempFile("test_oob_buffer_out_of_band", "data"); - Files.deleteIfExists(outOfBandDataFile); - Files.createFile(outOfBandDataFile); - MemoryBuffer outOfBandBuffer = MemoryBuffer.newHeapBuffer(32); - outOfBandBuffer.writeInt32(buffers.size()); - for (int i = 0; i < buffers.size(); i++) { - outOfBandBuffer.writeInt32(bufferObjects.get(i).totalBytes()); - bufferObjects.get(i).writeTo(outOfBandBuffer); - } - Files.write(outOfBandDataFile, outOfBandBuffer.getBytes(0, outOfBandBuffer.writerIndex())); - ImmutableList command = - ImmutableList.of( - PYTHON_EXECUTABLE, - "-m", - PYTHON_MODULE, - "test_oob_buffer", - intBandDataFile.toAbsolutePath().toString(), - outOfBandDataFile.toAbsolutePath().toString()); - Assert.assertTrue(executeCommand(command, 30)); - - MemoryBuffer inBandBuffer = MemoryUtils.wrap(Files.readAllBytes(intBandDataFile)); - outOfBandBuffer = MemoryUtils.wrap(Files.readAllBytes(outOfBandDataFile)); - int numBuffers = outOfBandBuffer.readInt32(); - buffers = new ArrayList<>(); - for (int i = 0; i < numBuffers; i++) { - int len = outOfBandBuffer.readInt32(); - int readerIndex = outOfBandBuffer.readerIndex(); - buffers.add(outOfBandBuffer.slice(readerIndex, len)); - outOfBandBuffer.readerIndex(readerIndex + len); - } - newObj = (List) fury.deserialize(inBandBuffer, buffers); - Assert.assertNotNull(newObj); - for (int i = 0; i < data.size(); i++) { - Assert.assertEquals(data.get(i), newObj.get(i)); - } - } - - @Data - static class ArrayStruct { - ArrayField[] f1; - } - - @Data - static class ArrayField { - public String a; - } - - @Test - public void testStructArrayField() { - Fury fury = Fury.builder().withLanguage(Language.XLANG).requireClassRegistration(true).build(); - fury.register(ArrayStruct.class, "example.bar"); - fury.register(ArrayField.class, "example.foo"); - - ArrayField a = new ArrayField(); - a.a = "123"; - ArrayStruct struct = new ArrayStruct(); - struct.f1 = new ArrayField[] {a}; - Assert.assertEquals(serDe(fury, struct), struct); - } -} From 7e236a9c5c7dca2df182e090fd39f53f4d384fd2 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 3 May 2024 12:18:35 +0800 Subject: [PATCH 2/7] debug ci --- .github/workflows/ci.yml | 2 -- ci/run_ci.sh | 7 +++++-- .../test/java/org/apache/fury/CyclicTest.java | 19 +++++++++++++++++-- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ef81e1ce8..835da4d576 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,5 @@ jobs: with: java-version: ${{ matrix.java-version }} distribution: 'temurin' - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - name: Run CI with Maven run: ./ci/run_ci.sh java${{ matrix.java-version }} diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 30bbe44b49..2ed76a4052 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -142,8 +142,11 @@ jdk17_plus_tests() { java -version echo "Executing fury java tests" cd "$ROOT/java" - set +e - mvn -T10 --batch-mode --no-transfer-progress test install -pl '!fury-format,!fury-testsuite' + set -ex + for i in {1..100}; do + echo "====================> $i" + mvn -T10 --batch-mode --no-transfer-progress test install -pl '!fury-format,!fury-testsuite' + done testcode=$? if [[ $testcode -ne 0 ]]; then exit $testcode diff --git a/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java b/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java index ac3a4049e3..e68d37b625 100644 --- a/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java @@ -26,6 +26,7 @@ import org.apache.fury.config.CompatibleMode; import org.apache.fury.config.FuryBuilder; import org.apache.fury.config.Language; +import org.apache.fury.resolver.MetaContext; import org.apache.fury.test.bean.Cyclic; import org.apache.fury.test.bean.FinalCyclic; import org.testng.Assert; @@ -92,8 +93,22 @@ public void testBeanMetaShared(FuryBuilder builder) { for (Object[] objects : beans()) { Object notCyclic = objects[0]; Object cyclic = objects[1]; - Assert.assertEquals(notCyclic, serDeMetaShared(fury, notCyclic)); - Assert.assertEquals(cyclic, serDeMetaShared(fury, cyclic)); + { + MetaContext context = new MetaContext(); + fury.getSerializationContext().setMetaContext(context); + byte[] bytes = fury.serialize(notCyclic); + fury.getSerializationContext().setMetaContext(context); + Object notCyclic1 = fury.deserialize(bytes); + Assert.assertEquals(notCyclic, notCyclic1); + } + { + MetaContext context = new MetaContext(); + fury.getSerializationContext().setMetaContext(context); + byte[] bytes = fury.serialize(cyclic); + fury.getSerializationContext().setMetaContext(context); + Object c = fury.deserialize(bytes); + Assert.assertEquals(cyclic, c); + } Object[] arr = new Object[2]; arr[0] = arr; arr[1] = cyclic; From 74079f92f621fed08cb08e79abefc6d6abdc4be9 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 3 May 2024 12:39:35 +0800 Subject: [PATCH 3/7] test CyclicTest only --- ci/run_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 2ed76a4052..11610f3e97 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -145,7 +145,7 @@ jdk17_plus_tests() { set -ex for i in {1..100}; do echo "====================> $i" - mvn -T10 --batch-mode --no-transfer-progress test install -pl '!fury-format,!fury-testsuite' + mvn -T10 --batch-mode --no-transfer-progress test -Dtest=org.apache.fury.CyclicTest install -pl '!fury-format,!fury-testsuite' done testcode=$? if [[ $testcode -ne 0 ]]; then From 8ba578a6b6da4d1266178155751e823f4bcf7f8a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 3 May 2024 12:43:59 +0800 Subject: [PATCH 4/7] test CyclicTest only --- ci/run_ci.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 11610f3e97..3d3d3ce788 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -143,6 +143,8 @@ jdk17_plus_tests() { echo "Executing fury java tests" cd "$ROOT/java" set -ex + mvn -T10 install -DskipTests + cd fury-core for i in {1..100}; do echo "====================> $i" mvn -T10 --batch-mode --no-transfer-progress test -Dtest=org.apache.fury.CyclicTest install -pl '!fury-format,!fury-testsuite' From 660fa7b0fc6fd5881e2a022ad0db6b225499df45 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 3 May 2024 12:56:56 +0800 Subject: [PATCH 5/7] test CyclicTest only --- ci/run_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 3d3d3ce788..2ca3e12d95 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -147,7 +147,7 @@ jdk17_plus_tests() { cd fury-core for i in {1..100}; do echo "====================> $i" - mvn -T10 --batch-mode --no-transfer-progress test -Dtest=org.apache.fury.CyclicTest install -pl '!fury-format,!fury-testsuite' + mvn -T10 --batch-mode --no-transfer-progress test -Dtest=org.apache.fury.CyclicTest done testcode=$? if [[ $testcode -ne 0 ]]; then From 97a318acd8bc292ceb78dd981e073587ea204a68 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 3 May 2024 13:06:03 +0800 Subject: [PATCH 6/7] remove fury data provider --- java/fury-core/src/test/java/org/apache/fury/CyclicTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java b/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java index e68d37b625..b5f0b8cbc6 100644 --- a/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java @@ -65,7 +65,7 @@ public static Object[][] fury() { .toArray(Object[][]::new); } - @Test(dataProvider = "fury") + @Test public void testBean(FuryBuilder builder) { Fury fury = builder.withMetaContextShare(false).withRefTracking(true).build(); for (Object[] objects : beans()) { @@ -87,7 +87,7 @@ public void testBean(FuryBuilder builder) { } } - @Test(dataProvider = "fury") + @Test public void testBeanMetaShared(FuryBuilder builder) { Fury fury = builder.withMetaContextShare(true).withRefTracking(true).build(); for (Object[] objects : beans()) { From 7e7235d5999800cecfaca4e18cd7a115afc4358e Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 3 May 2024 13:31:11 +0800 Subject: [PATCH 7/7] disable async compilation --- java/fury-core/src/test/java/org/apache/fury/CyclicTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java b/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java index b5f0b8cbc6..e68d37b625 100644 --- a/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java @@ -65,7 +65,7 @@ public static Object[][] fury() { .toArray(Object[][]::new); } - @Test + @Test(dataProvider = "fury") public void testBean(FuryBuilder builder) { Fury fury = builder.withMetaContextShare(false).withRefTracking(true).build(); for (Object[] objects : beans()) { @@ -87,7 +87,7 @@ public void testBean(FuryBuilder builder) { } } - @Test + @Test(dataProvider = "fury") public void testBeanMetaShared(FuryBuilder builder) { Fury fury = builder.withMetaContextShare(true).withRefTracking(true).build(); for (Object[] objects : beans()) {