diff --git a/Cargo.toml b/Cargo.toml index 1657a16..49c61a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,10 +19,14 @@ tokio-io = ["tokio"] futures-io = { version = "0.3.21", optional = true } # hdrs requires at least hadoop 2.3 to work. # hadoop 2.2 doesn't handle FileNotFound correctly. -hdfs-sys = { version = "0.2.0", features = ["hdfs_2_3"] } +# +# hdrs requires at least hadoop 2.6 to work. +# Older version of hadoop doesn't handle errno correctly. +hdfs-sys = { version = "0.2.0", features = ["hdfs_2_6"] } libc = "0.2.124" log = "0.4.16" tokio = { version = "1.18.0", optional = true } +errno = "0.2.8" [dev-dependencies] anyhow = "1.0.57" diff --git a/README.md b/README.md index c6354ad..2401288 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,12 @@ export LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${LD_LIBRARY_PATH} export CLASSPATH=$(find $HADOOP_HOME -iname "*.jar" | xargs echo | tr ' ' ':') ``` +## Version Requirement + +`hdrs` requires at least hadoop 2.3 to work: hadoop 2.2 doesn't handle FileNotFound correctly. + +`hdrs` requires at least hadoop 2.6 to work: Older version of hadoop doesn't handle errno correctly. In older versions, hadoop will set errno to `3` if input path is an empty dir. + ## Contributing Check out the [CONTRIBUTING.md](./CONTRIBUTING.md) guide for more details on getting started with contributing to this project. diff --git a/src/client.rs b/src/client.rs index 31f6fd8..54f2a36 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,6 +1,7 @@ use std::ffi::CString; use std::{env, fs, io}; +use errno::{set_errno, Errno}; use hdfs_sys::*; use log::debug; @@ -54,6 +55,8 @@ impl Client { pub fn connect(name_node: &str) -> io::Result { prepare_env()?; + set_errno(Errno(0)); + debug!("connect name node {}", name_node); let fs = unsafe { @@ -187,6 +190,8 @@ impl Client { /// assert_eq!(fi.unwrap_err().kind(), io::ErrorKind::NotFound) /// ``` pub fn metadata(&self, path: &str) -> io::Result { + set_errno(Errno(0)); + let hfi = unsafe { let p = CString::new(path)?; hdfsGetPathInfo(self.fs, p.as_ptr()) @@ -216,6 +221,8 @@ impl Client { /// let fis = fs.read_dir("/tmp/hello/"); /// ``` pub fn read_dir(&self, path: &str) -> io::Result { + set_errno(Errno(0)); + let mut entries = 0; let hfis = unsafe { let p = CString::new(path)?; diff --git a/tests/main.rs b/tests/main.rs index 8775526..f3d5136 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -39,6 +39,31 @@ fn test_mkdir() -> Result<()> { Ok(()) } +#[test] +fn test_read_dir() -> Result<()> { + let _ = env_logger::try_init(); + dotenv::from_filename(".env").ok(); + + if env::var("HDRS_TEST").unwrap_or_default() != "on" { + return Ok(()); + } + + let name_node = env::var("HDRS_NAMENODE")?; + let work_dir = env::var("HDRS_WORKDIR").unwrap_or_default(); + + let fs = Client::connect(&name_node)?; + + let path = format!("{work_dir}{}", uuid::Uuid::new_v4()); + + let _ = fs.create_dir(&path).expect("mkdir should succeed"); + debug!("read dir {}", path); + let readdir = fs.read_dir(&path).expect("readdir should succeed"); + debug!("readdir: {:?}", readdir); + assert_eq!(readdir.len(), 0); + + Ok(()) +} + #[test] fn test_file() -> Result<()> { use std::io::{Read, Seek, SeekFrom, Write};