-
Notifications
You must be signed in to change notification settings - Fork 9
132 lines (115 loc) · 4.02 KB
/
ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
name: CI
on:
push:
branches:
- main
pull_request:
branches:
- main
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: true
jobs:
check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Format
run: cargo fmt --all -- --check
- name: Clippy with all features
run: cargo clippy --all-features --all-targets -- -D warnings
test-default:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup java env
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: "11"
- name: Setup hadoop env
shell: bash
run: |
curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner
echo "HADOOP_HOME=/home/runner/hadoop-3.3.5" >> $GITHUB_ENV
- name: Test
shell: bash
run: |
export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)
cargo test --features async_file -- --nocapture
env:
LD_LIBRARY_PATH: ${{ env.JAVA_HOME }}/lib/server:${{ env.HADOOP_HOME}}/lib/native
HDRS_TEST: on
HDRS_NAMENODE: default
HDRS_WORKDIR: /tmp/hdrs/
test-cluster:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Configure Hdfs
# namenode will use ports: 9870, 9000, 8020
# datanode will use ports: 9864
run: |
docker run -d \
--name namenode \
--network host \
-e CLUSTER_NAME=test \
-e WEBHDFS_CONF_dfs_webhdfs_enabled=true \
-e CORE_CONF_hadoop_http_staticuser_user=root \
-e HDFS_CONF_dfs_permissions_enabled=false \
bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
docker run -d \
--name datanode \
--network host \
-e CLUSTER_NAME=test \
-e WEBHDFS_CONF_dfs_webhdfs_enabled=true \
-e CORE_CONF_hadoop_http_staticuser_user=root \
-e HDFS_CONF_dfs_permissions_enabled=false \
bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
curl --retry 30 --retry-delay 1 --retry-connrefused http://localhost:9870
- name: Setup java env
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: "11"
- name: Setup hadoop env
shell: bash
run: |
curl -LsSf https://archive.apache.org/dist/hadoop/common/hadoop-3.1.3/hadoop-3.1.3.tar.gz | tar zxf - -C /home/runner
echo "HADOOP_HOME=/home/runner/hadoop-3.1.3" >> $GITHUB_ENV
- name: Test
shell: bash
run: |
export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)
cargo test --features async_file -- --nocapture
env:
LD_LIBRARY_PATH: ${{ env.JAVA_HOME }}/lib/server:${{ env.HADOOP_HOME }}/lib/native
HDRS_TEST: on
HDRS_INTEGRATED_TEST: on
HDRS_NAMENODE: hdfs://localhost:8020
HDRS_WORKDIR: /tmp/hdrs/
test-vendored:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup java env
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: "11"
- name: Setup hadoop env
shell: bash
run: |
curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner
echo "HADOOP_HOME=/home/runner/hadoop-3.3.5" >> $GITHUB_ENV
- name: Test
shell: bash
run: |
export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)
cargo test --features async_file,vendored -- --nocapture
env:
# If vendored has been enabled, we don't need to load native libs
LD_LIBRARY_PATH: ${{ env.JAVA_HOME }}/lib/server
HDRS_TEST: on
HDRS_NAMENODE: default
HDRS_WORKDIR: /tmp/hdrs/